Repository: BeehiveInnovations/pal-mcp-server Branch: main Commit: 7afc7c1cc96e Files: 360 Total size: 3.7 MB Directory structure: gitextract_qj0m35_8/ ├── .claude/ │ ├── commands/ │ │ └── fix-github-issue.md │ └── settings.json ├── .coveragerc ├── .dockerignore ├── .gitattributes ├── .github/ │ ├── FUNDING.yml │ ├── ISSUE_TEMPLATE/ │ │ ├── bug_report.yml │ │ ├── config.yml │ │ ├── documentation.yml │ │ ├── feature_request.yml │ │ └── tool_addition.yml │ ├── pull_request_template.md │ └── workflows/ │ ├── docker-pr.yml │ ├── docker-release.yml │ ├── semantic-pr.yml │ ├── semantic-release.yml │ └── test.yml ├── .gitignore ├── .pre-commit-config.yaml ├── AGENTS.md ├── CHANGELOG.md ├── CLAUDE.md ├── Dockerfile ├── LICENSE ├── README.md ├── SECURITY.md ├── claude_config_example.json ├── clink/ │ ├── __init__.py │ ├── agents/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── claude.py │ │ ├── codex.py │ │ └── gemini.py │ ├── constants.py │ ├── models.py │ ├── parsers/ │ │ ├── __init__.py │ │ ├── base.py │ │ ├── claude.py │ │ ├── codex.py │ │ └── gemini.py │ └── registry.py ├── code_quality_checks.ps1 ├── code_quality_checks.sh ├── communication_simulator_test.py ├── conf/ │ ├── __init__.py │ ├── azure_models.json │ ├── cli_clients/ │ │ ├── claude.json │ │ ├── codex.json │ │ └── gemini.json │ ├── custom_models.json │ ├── dial_models.json │ ├── gemini_models.json │ ├── openai_models.json │ ├── openrouter_models.json │ └── xai_models.json ├── config.py ├── docker/ │ ├── README.md │ └── scripts/ │ ├── build.ps1 │ ├── build.sh │ ├── deploy.ps1 │ ├── deploy.sh │ └── healthcheck.py ├── docker-compose.yml ├── docs/ │ ├── adding_providers.md │ ├── adding_tools.md │ ├── advanced-usage.md │ ├── ai-collaboration.md │ ├── ai_banter.md │ ├── azure_openai.md │ ├── configuration.md │ ├── context-revival.md │ ├── contributions.md │ ├── custom_models.md │ ├── docker-deployment.md │ ├── gemini-setup.md │ ├── getting-started.md │ ├── index.md │ ├── locale-configuration.md │ ├── logging.md │ ├── model_ranking.md │ ├── name-change.md │ ├── testing.md │ ├── tools/ │ │ ├── analyze.md │ │ ├── apilookup.md │ │ ├── challenge.md │ │ ├── chat.md │ │ ├── clink.md │ │ ├── codereview.md │ │ ├── consensus.md │ │ ├── debug.md │ │ ├── docgen.md │ │ ├── listmodels.md │ │ ├── planner.md │ │ ├── precommit.md │ │ ├── refactor.md │ │ ├── secaudit.md │ │ ├── testgen.md │ │ ├── thinkdeep.md │ │ ├── tracer.md │ │ └── version.md │ ├── troubleshooting.md │ ├── vcr-testing.md │ └── wsl-setup.md ├── examples/ │ ├── claude_config_macos.json │ └── claude_config_wsl.json ├── pal-mcp-server ├── providers/ │ ├── __init__.py │ ├── azure_openai.py │ ├── base.py │ ├── custom.py │ ├── dial.py │ ├── gemini.py │ ├── openai.py │ ├── openai_compatible.py │ ├── openrouter.py │ ├── registries/ │ │ ├── __init__.py │ │ ├── azure.py │ │ ├── base.py │ │ ├── custom.py │ │ ├── dial.py │ │ ├── gemini.py │ │ ├── openai.py │ │ ├── openrouter.py │ │ └── xai.py │ ├── registry.py │ ├── registry_provider_mixin.py │ ├── shared/ │ │ ├── __init__.py │ │ ├── model_capabilities.py │ │ ├── model_response.py │ │ ├── provider_type.py │ │ └── temperature.py │ └── xai.py ├── pyproject.toml ├── pytest.ini ├── requirements-dev.txt ├── requirements.txt ├── run-server.ps1 ├── run-server.sh ├── run_integration_tests.ps1 ├── run_integration_tests.sh ├── scripts/ │ └── sync_version.py ├── server.py ├── simulator_tests/ │ ├── __init__.py │ ├── base_test.py │ ├── conversation_base_test.py │ ├── log_utils.py │ ├── test_analyze_validation.py │ ├── test_basic_conversation.py │ ├── test_chat_simple_validation.py │ ├── test_codereview_validation.py │ ├── test_consensus_conversation.py │ ├── test_consensus_three_models.py │ ├── test_consensus_workflow_accurate.py │ ├── test_content_validation.py │ ├── test_conversation_chain_validation.py │ ├── test_cross_tool_comprehensive.py │ ├── test_cross_tool_continuation.py │ ├── test_debug_certain_confidence.py │ ├── test_debug_validation.py │ ├── test_line_number_validation.py │ ├── test_logs_validation.py │ ├── test_model_thinking_config.py │ ├── test_o3_model_selection.py │ ├── test_o3_pro_expensive.py │ ├── test_ollama_custom_url.py │ ├── test_openrouter_fallback.py │ ├── test_openrouter_models.py │ ├── test_per_tool_deduplication.py │ ├── test_planner_continuation_history.py │ ├── test_planner_validation.py │ ├── test_planner_validation_old.py │ ├── test_precommitworkflow_validation.py │ ├── test_prompt_size_limit_bug.py │ ├── test_refactor_validation.py │ ├── test_secaudit_validation.py │ ├── test_testgen_validation.py │ ├── test_thinkdeep_validation.py │ ├── test_token_allocation_validation.py │ ├── test_vision_capability.py │ └── test_xai_models.py ├── systemprompts/ │ ├── __init__.py │ ├── analyze_prompt.py │ ├── chat_prompt.py │ ├── clink/ │ │ ├── codex_codereviewer.txt │ │ ├── default.txt │ │ ├── default_codereviewer.txt │ │ └── default_planner.txt │ ├── codereview_prompt.py │ ├── consensus_prompt.py │ ├── debug_prompt.py │ ├── docgen_prompt.py │ ├── generate_code_prompt.py │ ├── planner_prompt.py │ ├── precommit_prompt.py │ ├── refactor_prompt.py │ ├── secaudit_prompt.py │ ├── testgen_prompt.py │ ├── thinkdeep_prompt.py │ └── tracer_prompt.py ├── tests/ │ ├── CASSETTE_MAINTENANCE.md │ ├── __init__.py │ ├── conftest.py │ ├── gemini_cassettes/ │ │ ├── chat_codegen/ │ │ │ └── gemini25_pro_calculator/ │ │ │ └── mldev.json │ │ ├── chat_cross/ │ │ │ └── step1_gemini25_flash_number/ │ │ │ └── mldev.json │ │ └── consensus/ │ │ └── step2_gemini25_flash_against/ │ │ └── mldev.json │ ├── http_transport_recorder.py │ ├── mock_helpers.py │ ├── openai_cassettes/ │ │ ├── chat_cross_step2_gpt5_reminder.json │ │ ├── chat_gpt5_continuation.json │ │ ├── chat_gpt5_moon_distance.json │ │ ├── consensus_step1_gpt51_for.json │ │ ├── consensus_step1_gpt52_for.json │ │ ├── consensus_step1_gpt5_for.json │ │ └── o3_pro_basic_math.json │ ├── pii_sanitizer.py │ ├── sanitize_cassettes.py │ ├── test_alias_target_restrictions.py │ ├── test_auto_mode.py │ ├── test_auto_mode_comprehensive.py │ ├── test_auto_mode_custom_provider_only.py │ ├── test_auto_mode_model_listing.py │ ├── test_auto_mode_provider_selection.py │ ├── test_auto_model_planner_fix.py │ ├── test_azure_openai_provider.py │ ├── test_buggy_behavior_prevention.py │ ├── test_cassette_semantic_matching.py │ ├── test_challenge.py │ ├── test_chat_codegen_integration.py │ ├── test_chat_cross_model_continuation.py │ ├── test_chat_openai_integration.py │ ├── test_chat_simple.py │ ├── test_clink_claude_agent.py │ ├── test_clink_claude_parser.py │ ├── test_clink_codex_agent.py │ ├── test_clink_gemini_agent.py │ ├── test_clink_gemini_parser.py │ ├── test_clink_integration.py │ ├── test_clink_parsers.py │ ├── test_clink_tool.py │ ├── test_collaboration.py │ ├── test_config.py │ ├── test_consensus.py │ ├── test_consensus_integration.py │ ├── test_consensus_schema.py │ ├── test_conversation_continuation_integration.py │ ├── test_conversation_field_mapping.py │ ├── test_conversation_file_features.py │ ├── test_conversation_memory.py │ ├── test_conversation_missing_files.py │ ├── test_custom_openai_temperature_fix.py │ ├── test_custom_provider.py │ ├── test_debug.py │ ├── test_deploy_scripts.py │ ├── test_dial_provider.py │ ├── test_directory_expansion_tracking.py │ ├── test_disabled_tools.py │ ├── test_docker_claude_desktop_integration.py │ ├── test_docker_config_complete.py │ ├── test_docker_healthcheck.py │ ├── test_docker_implementation.py │ ├── test_docker_mcp_validation.py │ ├── test_docker_security.py │ ├── test_docker_volume_persistence.py │ ├── test_file_protection.py │ ├── test_gemini_token_usage.py │ ├── test_image_support_integration.py │ ├── test_image_validation.py │ ├── test_integration_utf8.py │ ├── test_intelligent_fallback.py │ ├── test_issue_245_simple.py │ ├── test_large_prompt_handling.py │ ├── test_line_numbers_integration.py │ ├── test_listmodels.py │ ├── test_listmodels_restrictions.py │ ├── test_mcp_error_handling.py │ ├── test_model_enumeration.py │ ├── test_model_metadata_continuation.py │ ├── test_model_resolution_bug.py │ ├── test_model_restrictions.py │ ├── test_o3_pro_output_text_fix.py │ ├── test_o3_temperature_fix_simple.py │ ├── test_openai_compatible_token_usage.py │ ├── test_openai_provider.py │ ├── test_openrouter_provider.py │ ├── test_openrouter_registry.py │ ├── test_openrouter_store_parameter.py │ ├── test_parse_model_option.py │ ├── test_path_traversal_security.py │ ├── test_per_tool_model_defaults.py │ ├── test_pii_sanitizer.py │ ├── test_pip_detection_fix.py │ ├── test_planner.py │ ├── test_precommit_workflow.py │ ├── test_prompt_regression.py │ ├── test_prompt_size_limit_bug_fix.py │ ├── test_provider_retry_logic.py │ ├── test_provider_routing_bugs.py │ ├── test_provider_utf8.py │ ├── test_providers.py │ ├── test_rate_limit_patterns.py │ ├── test_refactor.py │ ├── test_secaudit.py │ ├── test_server.py │ ├── test_supported_models_aliases.py │ ├── test_thinking_modes.py │ ├── test_tools.py │ ├── test_tracer.py │ ├── test_utf8_localization.py │ ├── test_utils.py │ ├── test_uvx_resource_packaging.py │ ├── test_uvx_support.py │ ├── test_workflow_file_embedding.py │ ├── test_workflow_metadata.py │ ├── test_workflow_prompt_size_validation_simple.py │ ├── test_workflow_utf8.py │ ├── test_xai_provider.py │ └── transport_helpers.py ├── tools/ │ ├── __init__.py │ ├── analyze.py │ ├── apilookup.py │ ├── challenge.py │ ├── chat.py │ ├── clink.py │ ├── codereview.py │ ├── consensus.py │ ├── debug.py │ ├── docgen.py │ ├── listmodels.py │ ├── models.py │ ├── planner.py │ ├── precommit.py │ ├── refactor.py │ ├── secaudit.py │ ├── shared/ │ │ ├── __init__.py │ │ ├── base_models.py │ │ ├── base_tool.py │ │ ├── exceptions.py │ │ └── schema_builders.py │ ├── simple/ │ │ ├── __init__.py │ │ └── base.py │ ├── testgen.py │ ├── thinkdeep.py │ ├── tracer.py │ ├── version.py │ └── workflow/ │ ├── __init__.py │ ├── base.py │ ├── schema_builders.py │ └── workflow_mixin.py └── utils/ ├── __init__.py ├── client_info.py ├── conversation_memory.py ├── env.py ├── file_types.py ├── file_utils.py ├── image_utils.py ├── model_context.py ├── model_restrictions.py ├── security_config.py ├── storage_backend.py └── token_utils.py ================================================ FILE CONTENTS ================================================ ================================================ FILE: .claude/commands/fix-github-issue.md ================================================ Please analyze and fix the GitHub issue: $ARGUMENTS. Follow these steps: 1. Use `gh issue view` to get the issue details 2. Understand the problem described in the issue 3. Search the codebase for relevant files 4. Implement the necessary changes to fix the issue 5. Write and run tests to verify the fix 6. Ensure code passes linting and type checking 7. Create a descriptive commit message 8. Push and create a PR Remember to use the GitHub CLI (`gh`) for all GitHub-related tasks. ================================================ FILE: .claude/settings.json ================================================ { "permissions": { "allow": [ ], "deny": [] } } ================================================ FILE: .coveragerc ================================================ [run] source = gemini_server omit = */tests/* */venv/* */__pycache__/* */site-packages/* [report] exclude_lines = pragma: no cover def __repr__ if self.debug: if settings.DEBUG raise AssertionError raise NotImplementedError if 0: if __name__ == .__main__.: if TYPE_CHECKING: class .*\bProtocol\): @(abc\.)?abstractmethod [html] directory = htmlcov ================================================ FILE: .dockerignore ================================================ # Git .git .gitignore # Python __pycache__/ *.py[cod] *$py.class *.so .Python env/ venv/ .venv/ .pal_venv/ ENV/ env.bak/ venv.bak/ # IDE .vscode/ .idea/ *.swp *.swo # OS .DS_Store Thumbs.db # Logs logs/*.log* *.log # Docker Dockerfile* docker-compose* .dockerignore # Documentation docs/ README.md *.md # Tests tests/ simulator_tests/ test_simulation_files/ pytest.ini # Development .env .env.local examples/ code_quality_checks.sh run_integration_tests.sh # Security - Sensitive files *.key *.pem *.p12 *.pfx *.crt *.csr secrets/ private/ ================================================ FILE: .gitattributes ================================================ # Ensure shell scripts always have LF line endings on checkout *.sh text eol=lf *.bash text eol=lf # Python files *.py text eol=lf # Shell script without extension run-server text eol=lf code_quality_checks text eol=lf run_integration_tests text eol=lf # General text files *.md text *.txt text *.yml text *.yaml text *.json text *.xml text # Binary files *.png binary *.jpg binary *.jpeg binary *.gif binary *.ico binary *.pdf binary ================================================ FILE: .github/FUNDING.yml ================================================ # These are supported funding model platforms github: [guidedways] ================================================ FILE: .github/ISSUE_TEMPLATE/bug_report.yml ================================================ name: 🐞 Bug Report description: Create a report to help us improve labels: ["bug", "needs-triage"] body: - type: markdown attributes: value: | Thank you for taking the time to file a bug report! Please provide as much detail as possible to help us reproduce and fix the issue. - type: input id: version attributes: label: Project Version description: "Which version are you using? (To see version: ./run-server.sh -v)" placeholder: "e.g., 9.4.1" validations: required: true - type: textarea id: description attributes: label: Bug Description description: A clear and concise description of what the bug is. placeholder: "When I run the `codereview` nothing happens" validations: required: true - type: textarea id: logs attributes: label: Relevant Log Output description: "Please copy and paste any relevant log output. Logs are stored under the `logs` folder in the pal folder. You an also use `./run-server.sh -f` to see logs" render: shell - type: dropdown id: environment attributes: label: Operating System description: What operating system are you running the Docker client on? options: - macOS - Windows - Linux validations: required: true - type: checkboxes id: no-duplicate-issues attributes: label: Sanity Checks description: "Before submitting, please confirm the following:" options: - label: I have searched the existing issues and this is not a duplicate. required: true - label: I am using `GEMINI_API_KEY` required: true - label: I am using `OPENAI_API_KEY` required: true - label: I am using `OPENROUTER_API_KEY` required: true - label: I am using `CUSTOM_API_URL` required: true ================================================ FILE: .github/ISSUE_TEMPLATE/config.yml ================================================ blank_issues_enabled: false contact_links: - name: 💬 General Discussion url: https://github.com/BeehiveInnovations/pal-mcp-server/discussions about: Ask questions, share ideas, or discuss usage patterns with the community - name: 📚 Documentation url: https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/README.md about: Check the README for setup instructions and usage examples - name: 🤝 Contributing Guide url: https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/CONTRIBUTING.md about: Learn how to contribute to the project ================================================ FILE: .github/ISSUE_TEMPLATE/documentation.yml ================================================ name: 📖 Documentation Improvement description: Report an issue or suggest an improvement for the documentation labels: ["documentation", "good first issue"] body: - type: input id: location attributes: label: Documentation Location description: "Which file or page has the issue? (e.g., README.md, CONTRIBUTING.md, CLAUDE.md)" placeholder: "e.g., README.md" validations: required: true - type: dropdown id: issue-type attributes: label: Type of Documentation Issue description: What kind of documentation improvement is this? options: - Typo or grammar error - Unclear or confusing explanation - Outdated information - Missing information - Code example doesn't work - Installation/setup instructions unclear - Tool usage examples need improvement - Other validations: required: true - type: textarea id: problem attributes: label: What is wrong with the documentation? description: "Please describe the problem. Be specific about what is unclear, incorrect, or missing." placeholder: "README is missing some details" validations: required: true - type: textarea id: suggestion attributes: label: Suggested Improvement description: "How can we make it better? If you can, please provide the exact text or changes you'd like to see." placeholder: "Please improve...." - type: dropdown id: audience attributes: label: Target Audience description: Which audience would benefit most from this improvement? options: - New users (first-time setup) - Developers (contributing to the project) - Advanced users (complex workflows) - All users validations: required: true ================================================ FILE: .github/ISSUE_TEMPLATE/feature_request.yml ================================================ name: ✨ Feature Request description: Suggest an idea for this project labels: ["enhancement", "needs-triage"] body: - type: textarea id: problem-description attributes: label: What problem is this feature trying to solve? description: "A clear and concise description of the problem or user need. Why is this change needed?" placeholder: "Currently, I can only use one Gemini tool at a time. I want to be able to chain multiple tools together (e.g., analyze -> codereview -> thinkdeep) in a single workflow." validations: required: true - type: textarea id: proposed-solution attributes: label: Describe the solution you'd like description: A clear and concise description of what you want to happen. How would it work from a user's perspective? placeholder: "I'd like to be able to specify a workflow like 'analyze src/ then codereview the findings then use thinkdeep to suggest improvements' in a single command or configuration." validations: required: true - type: textarea id: alternatives attributes: label: Describe alternatives you've considered description: A clear and concise description of any alternative solutions or features you've considered. placeholder: "I considered manually running each tool sequentially, but automatic workflow chaining would be more efficient and ensure context is preserved between steps." - type: dropdown id: feature-type attributes: label: Feature Category description: What type of enhancement is this? options: - New tool (chat, codereview, debug, etc.) - Workflow improvement - Integration enhancement - Performance optimization - User experience improvement - Documentation enhancement - Other validations: required: true - type: checkboxes id: contribution attributes: label: Contribution options: - label: I am willing to submit a Pull Request to implement this feature. ================================================ FILE: .github/ISSUE_TEMPLATE/tool_addition.yml ================================================ name: 🛠️ New Gemini Tool Proposal description: Propose a new PAL MCP tool (e.g., `summarize`, `fixer`, `refactor`) labels: ["enhancement", "new-tool"] body: - type: input id: tool-name attributes: label: Proposed Tool Name description: "What would the tool be called? (e.g., `summarize`, `docgen`, `refactor`)" placeholder: "e.g., `docgen`" validations: required: true - type: textarea id: purpose attributes: label: What is the primary purpose of this tool? description: "Explain the tool's core function and the value it provides to developers using Claude + PAL." placeholder: "This tool will automatically generate comprehensive documentation from code, extracting class and function signatures, docstrings, and creating usage examples." validations: required: true - type: textarea id: example-usage attributes: label: Example Usage in Claude Desktop description: "Show how a user would invoke this tool through Claude and what the expected output would look like." placeholder: | **User prompt to Claude:** "Use pal to generate documentation for my entire src/ directory" **Expected behavior:** - Analyze all Python files in src/ - Extract classes, functions, and their docstrings - Generate structured markdown documentation - Include usage examples where possible - Return organized documentation with table of contents render: markdown validations: required: true - type: dropdown id: tool-category attributes: label: Tool Category description: What category does this tool fit into? options: - Code Analysis (like analyze) - Code Quality (like codereview) - Code Generation/Refactoring - Documentation Generation - Testing Support - Debugging Support (like debug) - Workflow Automation - Architecture Planning (like thinkdeep) - Other validations: required: true - type: textarea id: system-prompt attributes: label: Proposed System Prompt (Optional) description: "If you have ideas for how pal should be prompted for this tool, share them here." placeholder: | You are an expert technical documentation generator. Your task is to create comprehensive, user-friendly documentation from source code... - type: checkboxes id: contribution attributes: label: Contribution options: - label: I am willing to submit a Pull Request to implement this new tool. - label: I have checked that this tool doesn't overlap significantly with existing tools (analyze, codereview, debug, thinkdeep, chat). ================================================ FILE: .github/pull_request_template.md ================================================ ## PR Title Format **Please ensure your PR title follows [Conventional Commits](https://www.conventionalcommits.org/) format:** ### Version Bumping Types (trigger semantic release): - `feat: ` - New features → **MINOR** version bump (1.1.0 → 1.2.0) - `fix: ` - Bug fixes → **PATCH** version bump (1.1.0 → 1.1.1) - `perf: ` - Performance improvements → **PATCH** version bump (1.1.0 → 1.1.1) ### Breaking Changes (trigger MAJOR version bump): For breaking changes, use any commit type above with `BREAKING CHANGE:` in the commit body or `!` after the type: - `feat!: ` → **MAJOR** version bump (1.1.0 → 2.0.0) - `fix!: ` → **MAJOR** version bump (1.1.0 → 2.0.0) ### Non-Versioning Types (no release): - `build: ` - Build system changes - `chore: ` - Maintenance tasks - `ci: ` - CI/CD changes - `docs: ` - Documentation only - `refactor: ` - Code refactoring (no functional changes) - `style: ` - Code style/formatting changes - `test: ` - Test additions/changes ### Docker Build Triggering: Docker builds are **independent** of versioning and trigger based on: **Automatic**: When PRs modify relevant files: - Python files (`*.py`), `requirements*.txt`, `pyproject.toml` - Docker files (`Dockerfile`, `docker-compose.yml`, `.dockerignore`) **Manual**: Add the `docker-build` label to force builds for any PR. ## Description Please provide a clear and concise description of what this PR does. ## Changes Made - [ ] List the specific changes made - [ ] Include any breaking changes - [ ] Note any dependencies added/removed ## Testing **Please review our [Testing Guide](../docs/testing.md) before submitting.** ### Run all linting and tests (required): ```bash # Activate virtual environment first source venv/bin/activate # Run comprehensive code quality checks (recommended) ./code_quality_checks.sh # If you made tool changes, also run simulator tests python communication_simulator_test.py ``` - [ ] All linting passes (ruff, black, isort) - [ ] All unit tests pass - [ ] **For new features**: Unit tests added in `tests/` - [ ] **For tool changes**: Simulator tests added in `simulator_tests/` - [ ] **For bug fixes**: Tests added to prevent regression - [ ] Simulator tests pass (if applicable) - [ ] Manual testing completed with realistic scenarios ## Related Issues Fixes #(issue number) ## Checklist - [ ] PR title follows the format guidelines above - [ ] **Activated venv and ran code quality checks: `source venv/bin/activate && ./code_quality_checks.sh`** - [ ] Self-review completed - [ ] **Tests added for ALL changes** (see Testing section above) - [ ] Documentation updated as needed - [ ] All unit tests passing - [ ] Relevant simulator tests passing (if tool changes) - [ ] Ready for review ## Additional Notes Any additional information that reviewers should know. ================================================ FILE: .github/workflows/docker-pr.yml ================================================ name: PR Docker Build on: pull_request: types: [opened, synchronize, reopened, labeled, unlabeled] paths: - '**.py' - 'requirements*.txt' - 'pyproject.toml' - 'Dockerfile' - 'docker-compose.yml' - '.dockerignore' permissions: contents: read packages: write pull-requests: write jobs: docker: name: Build Docker Image runs-on: ubuntu-latest if: | github.event.action == 'opened' || github.event.action == 'synchronize' || github.event.action == 'reopened' || contains(github.event.pull_request.labels.*.name, 'docker-build') steps: - name: Checkout uses: actions/checkout@v4 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry if: github.event.pull_request.head.repo.full_name == github.repository uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata id: meta uses: docker/metadata-action@v5 with: images: ghcr.io/${{ github.repository }} tags: | # PR-specific tag for testing type=raw,value=pr-${{ github.event.number }}-${{ github.sha }} type=raw,value=pr-${{ github.event.number }} - name: Build and push Docker image (internal PRs) if: github.event.pull_request.head.repo.full_name == github.repository uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - name: Build Docker image (fork PRs) if: github.event.pull_request.head.repo.full_name != github.repository uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 push: false tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - name: Add Docker build comment (internal PRs) if: github.event.pull_request.head.repo.full_name == github.repository uses: marocchino/sticky-pull-request-comment@d2ad0de260ae8b0235ce059e63f2949ba9e05943 # v2.9.3 with: header: docker-build message: | ## 🐳 Docker Build Complete **PR**: #${{ github.event.number }} | **Commit**: `${{ github.sha }}` ``` ${{ steps.meta.outputs.tags }} ``` **Test:** `docker pull ghcr.io/${{ github.repository }}:pr-${{ github.event.number }}` **Claude config:** ```json { "mcpServers": { "pal": { "command": "docker", "args": ["run", "--rm", "-i", "-e", "GEMINI_API_KEY", "ghcr.io/${{ github.repository }}:pr-${{ github.event.number }}"], "env": { "GEMINI_API_KEY": "your-key" } } } } ``` 💡 Add `docker-build` label to manually trigger builds - name: Update job summary (internal PRs) if: github.event.pull_request.head.repo.full_name == github.repository run: | { echo "## 🐳 Docker Build Complete" echo "**PR**: #${{ github.event.number }} | **Commit**: ${{ github.sha }}" echo '```' echo "${{ steps.meta.outputs.tags }}" echo '```' } >> $GITHUB_STEP_SUMMARY - name: Update job summary (fork PRs) if: github.event.pull_request.head.repo.full_name != github.repository run: | { echo "## 🐳 Docker Build Complete (Build Only)" echo "**PR**: #${{ github.event.number }} | **Commit**: ${{ github.sha }}" echo "✅ Multi-platform Docker build successful" echo "Note: Fork PRs only build (no push) for security" } >> $GITHUB_STEP_SUMMARY ================================================ FILE: .github/workflows/docker-release.yml ================================================ name: Docker Release Build on: release: types: [published] workflow_dispatch: inputs: tag: description: 'Tag to build (leave empty for latest release)' required: false type: string permissions: contents: read packages: write jobs: docker: name: Build and Push Docker Image runs-on: ubuntu-latest steps: - name: Checkout uses: actions/checkout@v4 with: # If triggered by workflow_dispatch with a tag, checkout that tag ref: ${{ inputs.tag || github.event.release.tag_name }} - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 with: registry: ghcr.io username: ${{ github.actor }} password: ${{ secrets.GITHUB_TOKEN }} - name: Extract metadata id: meta uses: docker/metadata-action@v5 with: images: ghcr.io/${{ github.repository }} tags: | # Tag with the release version type=semver,pattern={{version}},value=${{ inputs.tag || github.event.release.tag_name }} type=semver,pattern={{major}}.{{minor}},value=${{ inputs.tag || github.event.release.tag_name }} type=semver,pattern={{major}},value=${{ inputs.tag || github.event.release.tag_name }} # Also tag as latest for the most recent release type=raw,value=latest,enable={{is_default_branch}} - name: Build and push Docker image uses: docker/build-push-action@v5 with: context: . platforms: linux/amd64,linux/arm64 push: true tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha cache-to: type=gha,mode=max - name: Update release with Docker info if: github.event_name == 'release' run: | RELEASE_TAG="${{ github.event.release.tag_name }}" DOCKER_TAGS=$(echo "${{ steps.meta.outputs.tags }}" | tr '\n' ' ') # Add Docker information to the release gh release edit "$RELEASE_TAG" --notes-file - << EOF ${{ github.event.release.body }} --- ## 🐳 Docker Images This release is available as Docker images: $(echo "$DOCKER_TAGS" | sed 's/ghcr.io/- `ghcr.io/g' | sed 's/ /`\n/g') **Quick start with Docker:** \`\`\`bash docker pull ghcr.io/${{ github.repository }}:$RELEASE_TAG \`\`\` **Claude Desktop configuration:** \`\`\`json { "mcpServers": { "pal-mcp-server": { "command": "docker", "args": [ "run", "--rm", "-i", "-e", "GEMINI_API_KEY", "ghcr.io/${{ github.repository }}:$RELEASE_TAG" ], "env": { "GEMINI_API_KEY": "your-api-key-here" } } } } \`\`\` EOF env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Create deployment summary run: | echo "## 🐳 Docker Release Build Complete" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**Release**: ${{ inputs.tag || github.event.release.tag_name }}" >> $GITHUB_STEP_SUMMARY echo "**Images built:**" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY echo "${{ steps.meta.outputs.tags }}" >> $GITHUB_STEP_SUMMARY echo "\`\`\`" >> $GITHUB_STEP_SUMMARY ================================================ FILE: .github/workflows/semantic-pr.yml ================================================ --- name: Semantic PR on: pull_request: types: [opened, edited, synchronize] concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true permissions: contents: read pull-requests: write jobs: semantic-pr: name: Validate PR runs-on: ubuntu-latest timeout-minutes: 5 steps: - name: Check PR Title id: lint-pr-title uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3 env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Add PR error comment uses: marocchino/sticky-pull-request-comment@d2ad0de260ae8b0235ce059e63f2949ba9e05943 # v2.9.3 if: always() && (steps.lint-pr-title.outputs.error_message != null) with: header: pr-title-lint-error message: | We require pull request titles to follow the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) and it looks like your proposed title needs to be adjusted. Details: ``` ${{ steps.lint-pr-title.outputs.error_message }} ``` - name: Delete PR error comment uses: marocchino/sticky-pull-request-comment@d2ad0de260ae8b0235ce059e63f2949ba9e05943 # v2.9.3 if: ${{ steps.lint-pr-title.outputs.error_message == null }} with: header: pr-title-lint-error delete: true ================================================ FILE: .github/workflows/semantic-release.yml ================================================ name: Semantic Release on: push: branches: - main permissions: contents: write issues: write pull-requests: write jobs: release: runs-on: ubuntu-latest concurrency: release steps: - name: Checkout uses: actions/checkout@v4 with: fetch-depth: 0 token: ${{ secrets.GITHUB_TOKEN }} persist-credentials: true - name: Setup Python uses: actions/setup-python@v4 with: python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip pip install python-semantic-release - name: Verify tests pass run: | pip install -r requirements.txt pip install -r requirements-dev.txt python -m pytest tests/ -v --ignore=simulator_tests/ -m "not integration" - name: Run semantic release env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | git config --global user.name "github-actions[bot]" git config --global user.email "github-actions[bot]@users.noreply.github.com" semantic-release version semantic-release publish - name: Sync version to config.py run: | pip install toml python scripts/sync_version.py if git diff --quiet config.py; then echo "No version changes in config.py" else git add config.py git commit -m "chore: sync version to config.py [skip ci]" git push fi - name: Upload build artifacts to release if: hashFiles('dist/*') != '' run: | # Get the latest release tag LATEST_TAG=$(gh release list --limit 1 --json tagName --jq '.[0].tagName') if [ ! -z "$LATEST_TAG" ]; then echo "Uploading artifacts to release $LATEST_TAG" gh release upload "$LATEST_TAG" dist/* --clobber fi env: GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} ================================================ FILE: .github/workflows/test.yml ================================================ name: Tests on: pull_request: branches: [main] jobs: test: runs-on: ubuntu-latest strategy: matrix: python-version: ["3.10", "3.11", "3.12"] steps: - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} uses: actions/setup-python@v4 with: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements.txt pip install -r requirements-dev.txt - name: Run unit tests run: | # Run only unit tests (exclude simulation tests and integration tests) # Integration tests require local-llama which isn't available in CI python -m pytest tests/ -v --ignore=simulator_tests/ -m "not integration" env: # Ensure no API key is accidentally used in CI GEMINI_API_KEY: "" OPENAI_API_KEY: "" lint: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v4 with: python-version: "3.11" - name: Install dependencies run: | python -m pip install --upgrade pip pip install -r requirements-dev.txt - name: Run black formatter check run: black --check . --exclude="test_simulation_files/" - name: Run ruff linter run: ruff check . --exclude test_simulation_files ================================================ FILE: .gitignore ================================================ # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] *$py.class # C extensions *.so # Distribution / packaging .Python build/ develop-eggs/ dist/ downloads/ eggs/ .eggs/ lib/ lib64/ parts/ sdist/ var/ wheels/ share/python-wheels/ *.egg-info/ .installed.cfg *.egg MANIFEST # PyInstaller *.manifest *.spec # Installer logs pip-log.txt pip-delete-this-directory.txt # Unit test / coverage reports htmlcov/ .tox/ .nox/ .coverage .coverage.* .cache nosetests.xml coverage.xml *.cover *.py,cover .hypothesis/ .pytest_cache/ cover/ # Translations *.mo *.pot # Django stuff: *.log local_settings.py db.sqlite3 db.sqlite3-journal # Flask stuff: instance/ .webassets-cache # Scrapy stuff: .scrapy # Sphinx documentation docs/_build/ # PyBuilder .pybuilder/ target/ # Jupyter Notebook .ipynb_checkpoints # IPython profile_default/ ipython_config.py # pyenv .python-version # pipenv Pipfile.lock # poetry poetry.lock # pdm .pdm.toml .pdm-python pdm.lock # PEP 582 __pypackages__/ # Celery stuff celerybeat-schedule celerybeat.pid # SageMath parsed files *.sage.py # Environments .env .env~ .venv env/ venv/ ENV/ env.bak/ venv.bak/ # Spyder project settings .spyderproject .spyproject # Rope project settings .ropeproject # mkdocs documentation /site # mypy .mypy_cache/ .dmypy.json dmypy.json # Pyre type checker .pyre/ # pytype static type analyzer .pytype/ # Cython debug symbols cython_debug/ # PyCharm .idea/ # VS Code .vscode/ # macOS .DS_Store # API Keys and secrets *.key *.pem .env.local .env.*.local # Test outputs test_output/ *.test.log .coverage htmlcov/ coverage.xml .pytest_cache/ # Test simulation artifacts (dynamically created during testing) test_simulation_files/.claude/ # Temporary test directories test-setup/ # Scratch feature documentation files FEATURE_*.md # Temporary files /tmp/ # Local user instructions CLAUDE.local.md # Claude Code personal settings .claude/settings.local.json # Standalone mode files .pal_venv/ .docker_cleaned logs/ *.backup *.backup-*.json /.desktop_configured /worktrees/ test_simulation_files/ .mcp.json ================================================ FILE: .pre-commit-config.yaml ================================================ --- default_stages: [pre-commit, pre-push] repos: - repo: https://github.com/psf/black rev: 25.1.0 hooks: - id: black - repo: https://github.com/pycqa/isort rev: 6.0.1 hooks: - id: isort args: ["--profile", "black"] - repo: https://github.com/astral-sh/ruff-pre-commit rev: v0.12.8 hooks: - id: ruff args: [--fix] # Configuration for specific tools default_language_version: python: python3 # Exclude patterns exclude: | (?x)^( \.git/| \.venv/| venv/| \.pal_venv/| __pycache__/| \.pytest_cache/| logs/| dist/| build/| test_simulation_files/ ) ================================================ FILE: AGENTS.md ================================================ # Repository Guidelines See `requirements.txt` and `requirements-dev.txt` Also read CLAUDE.md and CLAUDE.local.md if available. ## Project Structure & Module Organization PAL MCP Server centers on `server.py`, which exposes MCP entrypoints and coordinates multi-model workflows. Feature-specific tools live in `tools/`, provider integrations in `providers/`, and shared helpers in `utils/`. Prompt and system context assets stay in `systemprompts/`, while configuration templates and automation scripts live under `conf/`, `scripts/`, and `docker/`. Unit tests sit in `tests/`; simulator-driven scenarios and log utilities are in `simulator_tests/` with the `communication_simulator_test.py` harness. Authoritative documentation and samples live in `docs/`, and runtime diagnostics are rotated in `logs/`. ## Build, Test, and Development Commands - `source .pal_venv/bin/activate` – activate the managed Python environment. - `./run-server.sh` – install dependencies, refresh `.env`, and launch the MCP server locally. - `./code_quality_checks.sh` – run Ruff autofix, Black, isort, and the default pytest suite. - `python communication_simulator_test.py --quick` – smoke-test orchestration across tools and providers. - `./run_integration_tests.sh [--with-simulator]` – exercise provider-dependent flows against remote or Ollama models. Run code quality checks: ```bash .pal_venv/bin/activate && ./code_quality_checks.sh ``` For example, this is how we run an individual / all tests: ```bash .pal_venv/bin/activate && pytest tests/test_auto_mode_model_listing.py -q .pal_venv/bin/activate && pytest -q ``` ## Coding Style & Naming Conventions Target Python 3.9+ with Black and isort using a 120-character line limit; Ruff enforces pycodestyle, pyflakes, bugbear, comprehension, and pyupgrade rules. Prefer explicit type hints, snake_case modules, and imperative commit-time docstrings. Extend workflows by defining hook or abstract methods instead of checking `hasattr()`/`getattr()`—inheritance-backed contracts keep behavior discoverable and testable. ## Testing Guidelines Mirror production modules inside `tests/` and name tests `test_` or `Test` classes. Run `python -m pytest tests/ -v -m "not integration"` before every commit, adding `--cov=. --cov-report=html` for coverage-sensitive changes. Use `python communication_simulator_test.py --verbose` or `--individual ` to validate cross-agent flows, and reserve `./run_integration_tests.sh` for provider or transport modifications. Capture relevant excerpts from `logs/mcp_server.log` or `logs/mcp_activity.log` when documenting failures. ## Commit & Pull Request Guidelines Follow Conventional Commits: `type(scope): summary`, where `type` is one of `feat`, `fix`, `docs`, `style`, `refactor`, `perf`, `test`, `build`, `ci`, or `chore`. Keep commits focused, referencing issues or simulator cases when helpful. Pull requests should outline intent, list validation commands executed, flag configuration or tool toggles, and attach screenshots or log snippets when user-visible behavior changes. ## GitHub CLI Commands The GitHub CLI (`gh`) streamlines issue and PR management directly from the terminal. ### Viewing Issues ```bash # View issue details in current repository gh issue view # View issue from specific repository gh issue view --repo owner/repo-name # View issue with all comments gh issue view --comments # Get issue data as JSON for scripting gh issue view --json title,body,author,state,labels,comments # Open issue in web browser gh issue view --web ``` ### Managing Issues ```bash # List all open issues gh issue list # List issues with filters gh issue list --label bug --state open # Create a new issue gh issue create --title "Issue title" --body "Description" # Close an issue gh issue close # Reopen an issue gh issue reopen ``` ### Pull Request Operations ```bash # View PR details gh pr view # List pull requests gh pr list # Create a PR from current branch gh pr create --title "PR title" --body "Description" # Check out a PR locally gh pr checkout # Merge a PR gh pr merge ``` Install GitHub CLI: `brew install gh` (macOS) or visit https://cli.github.com for other platforms. ## Security & Configuration Tips Store API keys and provider URLs in `.env` or your MCP client config; never commit secrets or generated log artifacts. Use `run-server.sh` to regenerate environments and verify connectivity after dependency changes. When adding providers or tools, sanitize prompts and responses, document required environment variables in `docs/`, and update `claude_config_example.json` if new capabilities ship by default. ================================================ FILE: CHANGELOG.md ================================================ # CHANGELOG ## v9.8.2 (2025-12-15) ### Bug Fixes - Allow home subdirectories through is_dangerous_path() ([`e5548ac`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e5548acb984ca4f8b2ae8381f879a0285094257f)) - Path traversal vulnerability - use prefix matching in is_dangerous_path() ([`9ed15f4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9ed15f405a9462b4db7aa44ca2d989e092c008e4)) - Use Path.is_relative_to() for cross-platform dangerous path detection ([`91ffb51`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/91ffb51564e5655ec91111938039ed81e0d8e4c6)) - **security**: Handle macOS symlinked system dirs ([`ba08308`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ba08308a23d1c1491099c5d0eae548077bd88f9f)) ### Chores - Sync version to config.py [skip ci] ([`c492735`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c4927358720277efa0373b339bd8e06ee06498d0)) ## v9.8.1 (2025-12-15) ### Bug Fixes - **providers**: Omit store parameter for OpenRouter responses endpoint ([`1f8b58d`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1f8b58d607c2809b9fa78860718a69207cb66e32)) ### Chores - Sync version to config.py [skip ci] ([`69a42a7`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/69a42a71d19d66f1d94d51fa27db29323e3d9a63)) ### Refactoring - **tests**: Address code review feedback ([`0c3e63c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/0c3e63c0c7f1556f4b6686f9c6f30e4bb4a48c7c)) - **tests**: Remove unused setUp method ([`b6a8d68`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b6a8d682d920c2283724b588818bc1162a865d74)) ## v9.8.0 (2025-12-15) ### Chores - Sync version to config.py [skip ci] ([`cb97a89`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/cb97a891dec6ab7c56b8b35c277ab3680af384d9)) ### Features - Add Claude Opus 4.5 model via OpenRouter ([`813ce5c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/813ce5c9f7db2910eb12d8c84d3d99f464c430ed)) ### Testing - Add comprehensive test coverage for Opus 4.5 aliases ([`cf63fd2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/cf63fd25440d599f2ec006bb8cfda5b8a6f61524)) ## v9.7.0 (2025-12-15) ### Chores - Sync version to config.py [skip ci] ([`aa85644`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/aa85644c9b15893443107c3a62ec58cd7b9dc532)) ### Features - Re-enable web search for clink codex using correct --enable flag ([`e7b9f3a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e7b9f3a5d7e06c690c82b9fd13a93310bcf388ed)) ## v9.6.0 (2025-12-15) ### Chores - Sync version to config.py [skip ci] ([`94ff26c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/94ff26c673a64087eb29f8f54c1828f1157c594a)) ### Features - Support native installed Claude CLI detection ([`adc6231`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/adc6231b98886f0bc35cb04d04d948eba2f0f058)) ## v9.5.0 (2025-12-11) ### Bug Fixes - Grok test ([`39c7721`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/39c77215e5d6892269e523ff25b706dd5671c042)) ### Chores - Sync version to config.py [skip ci] ([`5c3dd75`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/5c3dd75ca6b259f590bfd5078ea8e2f684e52de4)) - Sync version to config.py [skip ci] ([`605633b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/605633b2a2b044bbc5e41f2994dde27409a5b9b4)) ### Documentation - Cleanup ([`74f26e8`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/74f26e82e7a9c8a0214deef1cb18a3b2fa074050)) - Cleanup ([`2b22174`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2b221746fee6f7749d8aed8d07a85e428ac8e00f)) - Update subheading ([`591287c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/591287cb2f442a1fa34cd1139e3a0ad887388e5b)) ### Features - GPT-5.2 support ([`8b16405`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/8b16405f0609e232ff808361dc2a4d8ec258b0f3)) - Grok-4.1 support https://github.com/BeehiveInnovations/pal-mcp-server/issues/339 ([`514c9c5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/514c9c58fcc91933348d2188ed8c82bbe98132f2)) ## v9.4.2 (2025-12-04) ### Bug Fixes - Rebranding, see [docs/name-change.md](docs/name-change.md) for details ([`b2dc849`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b2dc84992d70839b29b611178b3871f4922b747f)) ### Chores - Sync version to config.py [skip ci] ([`bcfacce`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bcfaccecd490859fe189f45df4cf5b8e102d7874)) ## v9.4.1 (2025-11-21) ### Bug Fixes - Regression https://github.com/BeehiveInnovations/pal-mcp-server/issues/338 ([`aceddb6`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/aceddb655fc36918108b3da1f926bdd4e94875a2)) ### Chores - Sync version to config.py [skip ci] ([`c4461a4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c4461a466fab9c647b0a5035328c4d0f3e28f647)) ## v9.4.0 (2025-11-18) ### Bug Fixes - Failing test for gemini 3.0 pro open router ([`19a2a89`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/19a2a89b12c5dec53aea21a4244aff7796a5e049)) ### Chores - Sync version to config.py [skip ci] ([`d3de61f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d3de61f8787ab60261d09f2c7f362c50d2093799)) ### Features - Gemini 3.0 Pro Preview for Open Router ([`bbfdfac`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bbfdfac511668e8ae60f9b9b5d41eb9ab55d74cf)) ### Refactoring - Enable search on codex CLI ([`1579d9f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1579d9f806a653bb04c9c73ab304cdd0e78fbdfa)) ## v9.3.1 (2025-11-18) ### Chores - Sync version to config.py [skip ci] ([`d256098`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d2560983402abf084608f7750f05407a8d3e20a0)) ## v9.3.0 (2025-11-18) ### Chores - Sync version to config.py [skip ci] ([`3748d47`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3748d47faba7d871f2dd379f2c8646aa8cd3c6e9)) ## v9.2.2 (2025-11-18) ### Bug Fixes - **build**: Include clink resources in package ([`e9ac1ce`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e9ac1ce3354fbb124a72190702618f94266b8459)) ### Chores - Sync version to config.py [skip ci] ([`749bc73`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/749bc7307949fa0b0e026bfcfbd546d7619eba8b)) ## v9.2.1 (2025-11-18) ### Bug Fixes - **server**: Iterate provider instances during shutdown ([`d40fc83`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d40fc83d7549293372f3d20cc599a79ec355acef)) ### Chores - Sync version to config.py [skip ci] ([`84f6c4f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/84f6c4fb241257b611f4b954c22a6b9340007a73)) ## v9.2.0 (2025-11-18) ### Chores - Sync version to config.py [skip ci] ([`7a1de64`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7a1de6477aae88bfe7a2f677faf0794169651354)) ### Documentation - Streamline advanced usage guide by reorganizing table of contents for improved navigation ([`698d391`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/698d391b26a0dd565eada8bfa6e67e549ce1dd20)) - Update .env.example to include new GPT-5.1 model options and clarify existing model descriptions ([`dbbfef2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/dbbfef292c67ed54f90f7612c9c14d4095bd6c45)) - Update advanced usage and configuration to include new GPT-5.1 models and enhance tool parameters ([`807c9df`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/807c9df70e3b54031ec6beea10f3975455b36dfb)) ### Features - Add new GPT-5.1 models to configuration files and update model selection logic in OpenAI provider ([`8e9aa23`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/8e9aa2304d5e9ea9a9f8dc2a13a27a1ced6b1608)) - Enhance model support by adding GPT-5.1 to .gitignore and updating cassette maintenance documentation for dual-model testing ([`f713d8a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f713d8a354a37c32a806c98994e6f949ecd64237)) ## v9.1.4 (2025-11-18) ### Bug Fixes - Replaced deprecated Codex web search configuration ([`2ec64ba`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2ec64ba7489acc586846b25eedf94a4f05d5bd2d)) ### Chores - Sync version to config.py [skip ci] ([`4d3d177`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4d3d177d91370097ca7ac4f922fa3a8b69ce3250)) ## v9.1.3 (2025-10-22) ### Bug Fixes - Reduced token usage, removed parameters from schema that CLIs never seem to use ([`3e27319`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3e27319e60b0287df918856b58b2bbf042c948a8)) - Telemetry option no longer available in gemini 0.11 ([`2a8dff0`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2a8dff0cc8a3f33111533cdb971d654637ed0578)) ### Chores - Sync version to config.py [skip ci] ([`9e163f9`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9e163f9dc0654fc28961c9897b7c787a2b96e57d)) - Sync version to config.py [skip ci] ([`557e443`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/557e443a63ffd733fb41faaa8696f6f4bb2c2fd1)) ### Refactoring - Improved precommit system prompt ([`3efff60`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3efff6056e322ee1531d7bed5601038c129a8b29)) ## v9.1.2 (2025-10-21) ### Bug Fixes - Configure codex with a longer timeout ([`d2773f4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d2773f488af28986632846652874de9ff633049c)) - Handle claude's array style JSON https://github.com/BeehiveInnovations/pal-mcp-server/issues/295 ([`d5790a9`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d5790a9bfef719f03d17f2d719f1882e55d13b3b)) ### Chores - Sync version to config.py [skip ci] ([`04132f1`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/04132f1459f1e086afd8e3d456f671b63338f846)) ## v9.1.1 (2025-10-17) ### Bug Fixes - Failing test ([`aed3e3e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/aed3e3ee80c440ac8ab0d4abbf235b84df723d18)) - Handler for parsing multiple generated code blocks ([`f4c20d2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f4c20d2a20e1c57d8b10e8f508e07e2a8d72f94a)) - Improved error reporting; codex cli would at times fail to figure out how to handle plain-text / JSON errors ([`95e69a7`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/95e69a7cb234305dcd37dcdd2f22be715922e9a8)) ### Chores - Sync version to config.py [skip ci] ([`942757a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/942757a360a74c021b2a1aa63e394f18f5abcecd)) ## v9.1.0 (2025-10-17) ### Chores - Sync version to config.py [skip ci] ([`3ee0c8f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3ee0c8f555cb51b975700290919c2a8e2ada8cc4)) ### Features - Enhance review prompts to emphasize static analysis ([`36e66e2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/36e66e2e9a44a73a466545d4d3477ecb2cb3e669)) ## v9.0.4 (2025-10-17) ### Chores - Sync version to config.py [skip ci] ([`8c6f653`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/8c6f6532d843f7f1b283ce9b6472e5ba991efe16)) ## v9.0.3 (2025-10-16) ### Bug Fixes - Remove duplicate -o json flag in gemini CLI config ([`3b2eff5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3b2eff58ac0e2388045a7442c63f56ce259b54ba)) ### Chores - Sync version to config.py [skip ci] ([`b205d71`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b205d7159b674ce47ebc11af7255d1e3556fff93)) ## v9.0.2 (2025-10-15) ### Bug Fixes - Update Claude CLI commands to new mcp syntax ([`a2189cb`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/a2189cb88a295ebad6268b9b08c893cd65bc1d89)) ### Chores - Sync version to config.py [skip ci] ([`d08cdc6`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d08cdc6691e0f68917f2824945905b7256e0e568)) ## v9.0.1 (2025-10-14) ### Bug Fixes - Add JSON output flag to gemini CLI configuration ([`eb3dff8`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/eb3dff845828f60ff2659586883af622b8b035eb)) ### Chores - Sync version to config.py [skip ci] ([`b9408aa`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b9408aae8860d43b1da0ba67f9db98db7e4de2cf)) ## v9.0.0 (2025-10-08) ### Chores - Sync version to config.py [skip ci] ([`23c9b35`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/23c9b35d5226b07b59a4c4b3d7833ba81b019ea8)) ### Features - Claude Code as a CLI agent now supported. Mix and match: spawn claude code from within claude code, or claude code from within codex. ([`4cfaa0b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4cfaa0b6060769adfbd785a072526a5368421a73)) ## v8.0.2 (2025-10-08) ### Bug Fixes - Restore run-server quote trimming regex ([`1de4542`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1de454224c105891137134e2a25c2ee4f00dba45)) ### Chores - Sync version to config.py [skip ci] ([`728fb43`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/728fb439b929c9dc37646b24537ae043208fda7d)) ## v8.0.1 (2025-10-08) ### Bug Fixes - Resolve executable path for cross-platform compatibility in CLI agent ([`f98046c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f98046c2fccaa7f9a24665a0d705a98006461da5)) ### Chores - Sync version to config.py [skip ci] ([`52245b9`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/52245b91eaa5d720f8c3b21ead55248dd8e8bd57)) ### Testing - Fix clink agent tests to mock shutil.which() for executable resolution ([`4370be3`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4370be33b4b69a40456527213bcd62321a925a57)) ## v8.0.0 (2025-10-07) ### Chores - Sync version to config.py [skip ci] ([`4c34541`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4c3454121c3c678cdfe8ea03fa77f4dd414df9bc)) ## v7.8.1 (2025-10-07) ### Bug Fixes - Updated model description to fix test ([`04f7ce5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/04f7ce5b03804564263f53a765931edba9c320cd)) ### Chores - Sync version to config.py [skip ci] ([`c27e81d`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c27e81d6d2f22978816f798a161a869d1ab5f025)) ### Refactoring - Moved registries into a separate module and code cleanup ([`7c36b92`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7c36b9255a13007a10af4fadefc21aadfce482b0)) ## v7.8.0 (2025-10-07) ### Chores - Sync version to config.py [skip ci] ([`3e5fa96`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3e5fa96c981bbd7b844a9887a518ffe266b78e9b)) ### Documentation - Consensus video ([`2352684`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/23526841922a73c68094e5205e19af04a1f6c8cc)) - Formatting ([`7d7c74b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7d7c74b5a38b7d1adf132b8e28034017df7aa852)) - Link to videos from main page ([`e8ef193`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e8ef193daba393b55a3beaaba49721bb9182378a)) - Update README.md ([`7b13543`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7b13543824fc0af729daf753ecdddba9ee7d9f1e)) ### Features - All native providers now read from catalog files like OpenRouter / Custom configs. Allows for greater control over the capabilities ([`2a706d5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2a706d5720c0bf97b71c3e0fc95c15f78015bedf)) - Provider cleanup ([`9268dda`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9268ddad2a07306351765b47098134512739f49f)) ### Refactoring - New base class for model registry / loading ([`02d13da`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/02d13da897016d7491b4a10a1195983385d66654)) ## v7.7.0 (2025-10-07) ### Chores - Sync version to config.py [skip ci] ([`70ae62a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/70ae62a2cd663c3abcabddd1be1bc6ed9512d7df)) ### Documentation - Video ([`ed5dda7`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ed5dda7c5a9439c2835cc69d76e6377169ad048a)) ### Features - More aliases ([`5f0aaf5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/5f0aaf5f69c9d188d817b5ffbf6738c61da40ec7)) ## v7.6.0 (2025-10-07) ### Chores - Sync version to config.py [skip ci] ([`c1c75ba`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c1c75ba304c2840329650c46273e87eab9b05906)) - Sync version to config.py [skip ci] ([`0fa9b66`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/0fa9b6658099c8e0d79fda0c7d2347f62d0e6137)) ### Documentation - Info about AI client timeouts ([`3ddfed5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3ddfed5ef09000791e1c94b041c43dc273ed53a8)) ### Features - Add support for openai/gpt-5-pro model ([`abed075`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/abed075b2eaa99e9618202f47ff921094baae952)) ## v7.5.2 (2025-10-06) ### Bug Fixes - Handle 429 response https://github.com/BeehiveInnovations/pal-mcp-server/issues/273 ([`cbe1d79`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/cbe1d7993276bd014b495cbd2d0ece1f5d7583d9)) ### Chores - Sync version to config.py [skip ci] ([`74fdd36`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/74fdd36de92d34681fcc5a2f772c3d05634f0a55)) ## v7.5.1 (2025-10-06) ### Chores - Sync version to config.py [skip ci] ([`004e379`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/004e379cf2f1853829dccb15fa72ec18d282f1a4)) ## v7.5.0 (2025-10-06) ### Chores - Sync version to config.py [skip ci] ([`71e7cd5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/71e7cd55b1f4955a6d718fddc0de419414d133b6)) ### Documentation - Video ([`775e4d5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/775e4d50b826858095c5f2a61a07fc01c4a00816)) - Videos ([`bb2066c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bb2066c909f6581ba40fc5ddef3870954ae553ab)) ### Features - Support for GPT-5-Pro highest reasoning model https://github.com/BeehiveInnovations/pal-mcp-server/issues/275 ([`a65485a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/a65485a1e52fc79739000426295a27d096f4c9d8)) ## v7.4.0 (2025-10-06) ### Chores - Sync version to config.py [skip ci] ([`76bf98e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/76bf98e5cd972dabd3c79b25fcb9b9a717b23f6d)) ### Features - Improved prompt ([`b1e9963`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b1e9963991a41dff082ec1dce5691c318f105e6d)) ## v7.3.0 (2025-10-06) ### Chores - Sync version to config.py [skip ci] ([`e7920d0`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e7920d0ed16c0e6de9d1ccaa0b58d3fb5cbd7f2f)) ### Documentation - Fixed typo ([`3ab0aa8`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3ab0aa8314ad5992bcb00de549a0fab2e522751d)) - Fixed typo ([`c17ce3c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c17ce3cf958d488b97fa7127942542ab514b58bd)) - Update apilookup.md ([`1918679`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/19186794edac4fce5523e671310aecff4cbfdc81)) - Update README.md ([`23c6c78`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/23c6c78bf152ede6e7b5f7b7770b12a8442845a3)) ### Features - Codex supports web-search natively but needs to be turned on, run-server script asks if the user would like this done ([`97ba7e4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/97ba7e44ce7e3fd874759514ed2f0738033fc801)) ## v7.2.0 (2025-10-06) ### Chores - Sync version to config.py [skip ci] ([`1854b1e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1854b1e26b705cda0dc3f4d733647f1454aa0352)) ### Documentation - Updated ([`bb57f71`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bb57f719666ab6a586d835688ff8086282a5a0dc)) ### Features - New tool to perform apilookup (latest APIs / SDKs / language features etc) https://github.com/BeehiveInnovations/pal-mcp-server/issues/204 ([`5bea595`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/5bea59540f58b3c45044828c10f131aed104dd1c)) ### Refactoring - De-duplicate roles to avoid explosion when more CLIs get added ([`c42e9e9`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c42e9e9c34d7ae4732e2e4fbed579b681a6d170d)) ## v7.1.1 (2025-10-06) ### Bug Fixes - Clink missing in toml ([`1ff77fa`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1ff77faa800ad6c2dde49cad98dfa72035fe1c81)) ### Chores - Sync version to config.py [skip ci] ([`e02e78d`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e02e78d903b35f4c01b8039f4157e97b38d3ec7b)) ### Documentation - Example for codex cli ([`344c42b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/344c42bcbfb543bfd05cbc27fd5b419c76b77954)) - Example for codex cli ([`c3044de`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c3044de7424e638dde5c8ec49adb6c3c7c5a60b2)) - Update README.md ([`2e719ae`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2e719ae35e7979f7b83bd910867e79863a7f9ceb)) ## v7.1.0 (2025-10-05) ### Chores - Sync version to config.py [skip ci] ([`d54bfdd`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d54bfdd49797d076ec9cade44c56292a8089c744)) ### Features - Support for codex as external CLI ([`561e4aa`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/561e4aaaa8a89eb89c03985b9e7720cc98ef666c)) ## v7.0.2 (2025-10-05) ### Chores - Sync version to config.py [skip ci] ([`f2142a2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f2142a22ec50abc54b464eedd6b8239d20c509be)) ## v7.0.1 (2025-10-05) ### Bug Fixes - --yolo needed for running shell commands, documentation added ([`15ae3f2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/15ae3f24babccf42f43be5028bf8c60c05a6beaf)) ### Chores - Sync version to config.py [skip ci] ([`bc4a27b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bc4a27b18a4a3f45afb22178e61ea0be4d6a273c)) ### Documentation - Updated intro ([`fb668c3`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/fb668c39b5f6e3dd37f7027f953f6004f258f2bf)) ## v7.0.0 (2025-10-05) ### Chores - Sync version to config.py [skip ci] ([`0d46976`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/0d46976a8aa85254e4dbe06f5e71161cd3b13938)) - Sync version to config.py [skip ci] ([`8296bf8`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/8296bf871c39597a904c70e7d98c72fcb4dc5a84)) ### Documentation - Instructions for OpenCode ([`bd66622`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bd666227c8f7557483f7e24fb8544fc0456600dc)) - Updated intro ([`615873c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/615873c3db2ecf5ce6475caa3445e1da9a2517bd)) ### Features - Huge update - Link another CLI (such as `gemini` directly from with Claude Code / Codex). https://github.com/BeehiveInnovations/pal-mcp-server/issues/208 ([`a2ccb48`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/a2ccb48e9a5080a75dbfd483b5f09fc719c887e5)) ### Refactoring - Fixed test ([`9c99b9b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9c99b9b35219f54db8d7be0958d4390a106631ae)) - Include file modification dates too ([`47973e9`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/47973e945efa2cdbdb8f3404d467d7f1abc62b0a)) ## v6.1.0 (2025-10-04) ### Chores - Sync version to config.py [skip ci] ([`18095d7`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/18095d7d398e4bf3d24c57a52c81ac619acb1b89)) ### Documentation - Updated intro ([`aa65394`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/aa6539472c4ddf1c3c1bac446fdee03e75e1cb50)) ### Features - Support for Qwen Code ([`fe9968b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/fe9968b633d0312b82426e9ebddfe1d6515be3c5)) ## v6.0.0 (2025-10-04) ### Chores - Sync version to config.py [skip ci] ([`ae8749a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ae8749ab37bdaa7e225b5219820adeb74ca9a552)) ### Documentation - Updated ([`e91ed2a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e91ed2a924b1702edf9e1417479ac0dee0ca1553)) ### Features - Azure OpenAI / Azure AI Foundry support. Models should be defined in conf/azure_models.json (or a custom path). See .env.example for environment variables or see readme. https://github.com/BeehiveInnovations/pal-mcp-server/issues/265 ([`ff9a07a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ff9a07a37adf7a24aa87c63b3ba9db88bdff467b)) - Breaking change - OpenRouter models are now read from conf/openrouter_models.json while Custom / Self-hosted models are read from conf/custom_models.json ([`ff9a07a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ff9a07a37adf7a24aa87c63b3ba9db88bdff467b)) - OpenAI/compatible models (such as Azure OpenAI) can declare if they use the response API instead via `use_openai_responses_api` ([`3824d13`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3824d131618683572e9e8fffa6b25ccfabf4cf50)) - OpenRouter / Custom Models / Azure can separately also use custom config paths now (see .env.example ) ([`ff9a07a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ff9a07a37adf7a24aa87c63b3ba9db88bdff467b)) ### Refactoring - Breaking change: `is_custom` property has been removed from model_capabilities.py (and thus custom_models.json) given each models are now read from separate configuration files ([`ff9a07a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ff9a07a37adf7a24aa87c63b3ba9db88bdff467b)) - Model registry class made abstract, OpenRouter / Custom Provider / Azure OpenAI now subclass these ([`ff9a07a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ff9a07a37adf7a24aa87c63b3ba9db88bdff467b)) ## v5.22.0 (2025-10-04) ### Bug Fixes - CI test ([`bc93b53`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bc93b5343bbd8657b95ab47c00a2cb99a68a009f)) - Listmodels to always honor restricted models ([`4015e91`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4015e917ed32ae374ec6493b74993fcb34f4a971)) ### Chores - Sync version to config.py [skip ci] ([`054e34e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/054e34e31ca5bee5a11c0e3e6537f58e8897c79c)) - Sync version to config.py [skip ci] ([`c0334d7`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c0334d77922f1b05e3fd755851da112567fb9ae6)) ### Features - Centralized environment handling, ensures PAL_MCP_FORCE_ENV_OVERRIDE is honored correctly ([`2c534ac`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2c534ac06e4c6078b96781dfb55c5759b982afe8)) ### Refactoring - Don't retry on 429 ([`d184024`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d18402482087f52b7bd07755c9304ed00ed20592)) - Improved retry logic and moved core logic to base class ([`f955100`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f955100f3a82973ccd987607e1d8a1bbe07828c8)) - Removed subclass override when the base class should be resolving the model name ([`06d7701`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/06d7701cc3ee09732ab713fa9c7c004199154483)) ## v5.21.0 (2025-10-03) ### Chores - Sync version to config.py [skip ci] ([`ddb20a6`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ddb20a6cdb8cdeee27c0aacb0b9c794283b5774c)) ## v5.20.1 (2025-10-03) ### Chores - Sync version to config.py [skip ci] ([`03addcf`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/03addcfa2d3aed5086fe4c94e8b9ae56229a93ae)) ## v5.20.0 (2025-10-03) ### Chores - Sync version to config.py [skip ci] ([`539bc72`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/539bc72f1ca2a2cadcccad02de1fd5fc22cd0415)) ## v5.19.0 (2025-10-03) ### Bug Fixes - Add GPT-5-Codex to Responses API routing and simplify comments ([`82b021d`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/82b021d75acc791e68c7afb35f6492f68cf02bec)) ### Chores - Sync version to config.py [skip ci] ([`8e32ef3`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/8e32ef33e3ce7ab2a9d7eb5c90fe5b93b12d5c26)) ### Documentation - Bumped defaults ([`95d98a9`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/95d98a9bc0a5bafadccb9f6d1e4eda97a0dd2ce7)) ### Features - Add GPT-5-Codex support with Responses API integration ([`f265342`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f2653427ca829368e7145325d20a98df3ee6d6b4)) ### Testing - Cross tool memory recall, testing continuation via cassette recording ([`88493bd`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/88493bd357c6a12477c3160813100dae1bc46493)) ## v5.18.3 (2025-10-03) ### Bug Fixes - External model name now recorded properly in responses ([`d55130a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d55130a430401e106cd86f3e830b3d756472b7ff)) ### Chores - Sync version to config.py [skip ci] ([`5714e20`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/5714e2016405f7607b44d78f85081c7ccee706e5)) ### Documentation - Updated docs ([`b4e5090`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b4e50901ba60c88137a29d00ecf99718582856d3)) ### Refactoring - Generic name for the CLI agent ([`e9b6947`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e9b69476cd922c12931d62ccc3be9082bbbf6014)) - Generic name for the CLI agent ([`7a6fa0e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7a6fa0e77a8c4a682dc11c9bbb16bdaf86d9edf4)) - Generic name for the CLI agent ([`b692da2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b692da2a82facce7455b8f2ec0108e1db84c07c3)) - Generic name for the CLI agent ([`f76ebbf`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f76ebbf280cc78ffcfe17cb4590aeaa231db8aa1)) - Generic name for the CLI agent ([`c05913a`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c05913a09e53e195b9a108647c09c061ced19d17)) - Generic name for the CLI agent ([`0dfaa63`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/0dfaa6312ed95ac3d1ae0032334ae1286871b15e)) ### Testing - Fixed integration tests, removed magicmock ([`87ccb6b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/87ccb6b25ba32a3cb9c4cc64fc0e96294f492c04)) ## v5.18.2 (2025-10-02) ### Bug Fixes - Https://github.com/BeehiveInnovations/pal-mcp-server/issues/194 ([`8b3a286`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/8b3a2867fb83eccb3a8e8467e7e3fc5b8ebe1d0c)) ### Chores - Sync version to config.py [skip ci] ([`bf2196c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bf2196cdd58ae8d8d93597f2be69c798265d678f)) ## v5.18.1 (2025-10-02) ### Chores - Sync version to config.py [skip ci] ([`e434a26`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e434a2614af82efd15de4dd94b2c30559c91414e)) ## v5.18.0 (2025-10-02) ### Chores - Sync version to config.py [skip ci] ([`e78fe35`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e78fe35a1b64cc0ed89664440ef7c7b94495d7dc)) ### Features - Added `intelligence_score` to the model capabilities schema; a 1-20 number that can be specified to influence the sort order of models presented to the CLI in `auto selection` mode ([`6cab9e5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/6cab9e56fc5373da5c11d4545bcb85371d4803a4)) ## v5.17.4 (2025-10-02) ### Chores - Sync version to config.py [skip ci] ([`a6c9b92`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/a6c9b9212c77852d9e9a8780f4bc3e53b3bfed2f)) ## v5.17.3 (2025-10-02) ### Chores - Sync version to config.py [skip ci] ([`722f6f8`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/722f6f86ae228206ce0094d109a3b20499d4e11a)) ## v5.17.2 (2025-10-02) ### Chores - Sync version to config.py [skip ci] ([`e47a7e8`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e47a7e89d5bfad0bb0150cb3207f1a37dc91b170)) ## v5.17.1 (2025-10-02) ### Bug Fixes - Baseclass should return MODEL_CAPABILITIES ([`82a03ce`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/82a03ce63f28fece17bfc1d70bdb75aadec4c6bb)) ### Chores - Sync version to config.py [skip ci] ([`7ce66bd`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7ce66bd9508865cef64dc30936e86e37c1a306d0)) ### Documentation - Document custom timeout values ([`218fbdf`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/218fbdf49cb90f2353f58bbaef567519dd876634)) ### Refactoring - Clean temperature inference ([`9c11ecc`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9c11ecc4bf37562aa08dc3ecfa70f380e0ead357)) - Cleanup ([`6ec2033`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/6ec2033f34c74ad139036de83a34cf6d374db77b)) - Cleanup provider base class; cleanup shared responsibilities; cleanup public contract ([`693b84d`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/693b84db2b87271ac809abcf02100eee7405720b)) - Cleanup token counting ([`7fe9fc4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7fe9fc49f8e3cd92be4c45a6645d5d4ab3014091)) - Code cleanup ([`bb138e2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/bb138e2fb552f837b0f9f466027580e1feb26f7c)) - Code cleanup ([`182aa62`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/182aa627dfba6c578089f83444882cdd2635a7e3)) - Moved image related code out of base provider into a separate utility ([`14a35af`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/14a35afa1d25408e62b968d9846be7bffaede327)) - Moved temperature method from base provider to model capabilities ([`6d237d0`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/6d237d09709f757a042baf655f47eb4ddfc078ad)) - Moved temperature method from base provider to model capabilities ([`f461cb4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f461cb451953f882bbde096a9ecf0584deb1dde8)) - Removed hard coded checks, use model capabilities instead ([`250545e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/250545e34f8d4f8026bfebb3171f3c2bc40f4692)) - Removed hook from base class, turned into helper static method instead ([`2b10adc`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2b10adcaf2b8741f0da5de84cc3483eae742a014)) - Removed method from provider, should use model capabilities instead ([`a254ff2`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/a254ff2220ba00ec30f5110c69a4841419917382)) - Renaming to reflect underlying type ([`1dc25f6`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1dc25f6c3d4cdbf01f041cc424e3b5235c23175b)) ## v5.17.0 (2025-10-02) ### Bug Fixes - Use types.HttpOptions from module imports instead of local import ([`956e8a6`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/956e8a6927837f5c7f031a0db1dd0b0b5483c626)) ### Chores - Sync version to config.py [skip ci] ([`0836213`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/0836213071d0037d8a6d2e64d34ab5df79b8e684)) ### Code Style - Apply Black formatting to use double quotes ([`33ea896`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/33ea896c511764904bf2b6b22df823928f88a148)) ### Features - Add custom Gemini endpoint support ([`462bce0`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/462bce002e2141b342260969588e69f55f8bb46a)) ### Refactoring - Simplify Gemini provider initialization using kwargs dict ([`023940b`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/023940be3e38a7eedbc8bf8404a4a5afc50f8398)) ## v5.16.0 (2025-10-01) ### Bug Fixes - Resolve logging timing and import organization issues ([`d34c299`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d34c299f02a233af4f17bdcc848219bf07799723)) ### Chores - Sync version to config.py [skip ci] ([`b6c4bca`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b6c4bca158e4cee1ae4abd08b7e55216ebffba2d)) ### Code Style - Fix ruff import sorting issue ([`4493a69`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4493a693332e0532d04ad3634de2a2f5b1249b64)) ### Features - Add configurable environment variable override system ([`93ce698`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/93ce6987b6e7d8678ffa5ac51f5106a7a21ce67b)) ## v5.15.0 (2025-10-01) ### Chores - Sync version to config.py [skip ci] ([`b0fe956`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b0fe956f8a50240507e0fc911f0800634c15e9f7)) ### Features - Depending on the number of tools in use, this change should save ~50% of overall tokens used. fixes https://github.com/BeehiveInnovations/pal-mcp-server/issues/255 but also refactored individual tools to instead encourage the agent to use the listmodels tool if needed. ([`d9449c7`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d9449c7bb607caff3f0454f210ddfc36256c738a)) ### Performance Improvements - Tweaks to schema descriptions, aiming to reduce token usage without performance degradation ([`cc8a4df`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/cc8a4dfd21b6f3dae4972a833b619e53c964693b)) ### Refactoring - Trimmed some prompts ([`f69ff03`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/f69ff03c4d10e606a1dfed2a167f3ba2e2236ba8)) ## v5.14.1 (2025-10-01) ### Bug Fixes - Https://github.com/BeehiveInnovations/pal-mcp-server/issues/258 ([`696b45f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/696b45f25e80faccb67034254cf9a8fc4c643dbd)) ### Chores - Sync version to config.py [skip ci] ([`692016c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/692016c6205ed0a0c3d9e830482d88231aca2e31)) ## v5.14.0 (2025-10-01) ### Chores - Sync version to config.py [skip ci] ([`c0f822f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c0f822ffa23292d668f7b5dd3cb62e3f23fb29af)) ### Features - Add Claude Sonnet 4.5 and update alias configuration ([`95c4822`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/95c4822af2dc55f59c0e4ed9454673d6ca964731)) ### Testing - Update tests to match new Claude Sonnet 4.5 alias configuration ([`7efb409`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7efb4094d4eb7db006340d3d9240b9113ac25cd3)) ## v5.13.0 (2025-10-01) ### Bug Fixes - Add sonnet alias for Claude Sonnet 4.1 to match opus/haiku pattern ([`dc96344`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/dc96344db043e087ee4f8bf264a79c51dc2e0b7a)) - Missing "optenai/" in name ([`7371ed6`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/7371ed6487b7d90a1b225a67dca2a38c1a52f2ad)) ### Chores - Sync version to config.py [skip ci] ([`b8479fc`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/b8479fc638083d6caa4bad6205e3d3fcab830aca)) ### Features - Add comprehensive GPT-5 series model support ([`4930824`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/493082405237e66a2f033481a5f8bf8293b0d553)) ## v5.12.1 (2025-10-01) ### Bug Fixes - Resolve consensus tool model_context parameter missing issue ([`9044b63`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9044b63809113047fe678d659e4fcd175f58e87a)) ### Chores - Sync version to config.py [skip ci] ([`e3ebf4e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/e3ebf4e94eba63acdc4df5a0b0493e44e3343dd1)) ### Code Style - Fix trailing whitespace in consensus.py ([`0760b31`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/0760b31f8a6d03c4bea3fd2a94dfbbfab0ad5079)) ### Refactoring - Optimize ModelContext creation in consensus tool ([`30a8952`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/30a8952fbccd22bebebd14eb2c8005404b79bcd6)) ## v5.12.0 (2025-10-01) ### Bug Fixes - Removed use_websearch; this parameter was confusing Codex. It started using this to prompt the external model to perform searches! web-search is enabled by Claude / Codex etc by default and the external agent can ask claude to search on its behalf. ([`cff6d89`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/cff6d8998f64b73265c4e31b2352462d6afe377f)) ### Chores - Sync version to config.py [skip ci] ([`28cabe0`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/28cabe0833661b0bab56d4227781ee2da332b00c)) ### Features - Implement semantic cassette matching for o3 models ([`70fa088`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/70fa088c32ac4e6153d5e7b30a3e32022be2f908)) ## v5.11.2 (2025-10-01) ### Chores - Sync version to config.py [skip ci] ([`4d6f1b4`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4d6f1b41005dee428c955e33f04f8f9f6259e662)) ## v5.11.1 (2025-10-01) ### Bug Fixes - Remove duplicate OpenAI models from listmodels output ([`c29e762`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/c29e7623ace257eb45396cdf8c19e1659e29edb9)) ### Chores - Sync version to config.py [skip ci] ([`1209064`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/12090646ee83f2368311d595d87ae947e46ddacd)) ### Testing - Update OpenAI provider alias tests to match new format ([`d13700c`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d13700c14c7ee3d092302837cb1726d17bab1ab8)) ## v5.11.0 (2025-08-26) ### Chores - Sync version to config.py [skip ci] ([`9735469`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/973546990f2c45afa93f1aa6de33ff461ecf1a83)) ### Features - Codex CLI support ([`ce56d16`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/ce56d16240ddcc476145a512561efe5c66438f0d)) ## v5.10.3 (2025-08-24) ### Bug Fixes - Address test failures and PR feedback ([`6bd9d67`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/6bd9d6709acfb584ab30a0a4d6891cabdb6d3ccf)) - Resolve temperature handling issues for O3/custom models ([#245](https://github.com/BeehiveInnovations/pal-mcp-server/pull/245), [`3b4fd88`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/3b4fd88d7e9a3f09fea616a10cb3e9d6c1a0d63b)) ### Chores - Sync version to config.py [skip ci] ([`d6e6808`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d6e6808be525192ab8388c0f01bc1bbd016fc23a)) ## v5.10.2 (2025-08-24) ### Bug Fixes - Another fix for https://github.com/BeehiveInnovations/pal-mcp-server/issues/251 ([`a07036e`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/a07036e6805042895109c00f921c58a09caaa319)) ### Chores - Sync version to config.py [skip ci] ([`9da5c37`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/9da5c37809cbde19d0c7ffed273ae93ca883a016)) ## v5.10.0 (2025-08-22) ### Chores - Sync version to config.py [skip ci] ([`1254205`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/12542054a214022d3f515e53367f5bf3a77fb289)) ### Features - Refactored and tweaked model descriptions / schema to use fewer tokens at launch (average reduction per field description: 60-80%) without sacrificing tool effectiveness ([`4b202f5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4b202f5d1d24cea1394adab26a976188f847bd09)) ## v5.9.0 (2025-08-21) ### Documentation - Update instructions for precommit ([`90821b5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/90821b51ff653475d9fb1bc70b57951d963e8841)) ### Features - Refactored and improved codereview in line with precommit. Reviews are now either external (default) or internal. Takes away anxiety and loss of tokens when Claude incorrectly decides to be 'confident' about its own changes and bungle things up. ([`80d21e5`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/80d21e57c0246762c0a306ede5b93d6aeb2315d8)) ### Refactoring - Minor prompt tweaks ([`d30c212`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/d30c212029c05b767d99b5391c1dd4cee78ef336)) ## v5.8.6 (2025-08-20) ### Bug Fixes - Escape backslashes in TOML regex pattern ([`1c973af`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/1c973afb002650b9bbee8a831b756bef848915a1)) - Establish version 5.8.6 and add version sync automation ([`90a4195`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/90a419538128b54fbd30da4b8a8088ac59f8c691)) - Restore proper version 5.8.6 ([`340b58f`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/340b58f2e790b84c3736aa96df7f6f5f2d6a13c9)) ### Chores - Sync version to config.py [skip ci] ([`4f82f65`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/4f82f6500502b7b6ba41875a560c41f6a63b683b)) ## v1.1.0 (2025-08-20) ### Features - Improvements to precommit ([`2966dcf`](https://github.com/BeehiveInnovations/pal-mcp-server/commit/2966dcf2682feb7eef4073738d0c225a44ce0533)) ## v1.0.0 (2025-08-20) - Initial Release ================================================ FILE: CLAUDE.md ================================================ # Claude Development Guide for PAL MCP Server This file contains essential commands and workflows for developing and maintaining the PAL MCP Server when working with Claude. Use these instructions to efficiently run quality checks, manage the server, check logs, and run tests. ## Quick Reference Commands ### Code Quality Checks Before making any changes or submitting PRs, always run the comprehensive quality checks: ```bash # Activate virtual environment first source venv/bin/activate # Run all quality checks (linting, formatting, tests) ./code_quality_checks.sh ``` This script automatically runs: - Ruff linting with auto-fix - Black code formatting - Import sorting with isort - Complete unit test suite (excluding integration tests) - Verification that all checks pass 100% **Run Integration Tests (requires API keys):** ```bash # Run integration tests that make real API calls ./run_integration_tests.sh # Run integration tests + simulator tests ./run_integration_tests.sh --with-simulator ``` ### Server Management #### Setup/Update the Server ```bash # Run setup script (handles everything) ./run-server.sh ``` This script will: - Set up Python virtual environment - Install all dependencies - Create/update .env file - Configure MCP with Claude - Verify API keys #### View Logs ```bash # Follow logs in real-time ./run-server.sh -f # Or manually view logs tail -f logs/mcp_server.log ``` ### Log Management #### View Server Logs ```bash # View last 500 lines of server logs tail -n 500 logs/mcp_server.log # Follow logs in real-time tail -f logs/mcp_server.log # View specific number of lines tail -n 100 logs/mcp_server.log # Search logs for specific patterns grep "ERROR" logs/mcp_server.log grep "tool_name" logs/mcp_activity.log ``` #### Monitor Tool Executions Only ```bash # View tool activity log (focused on tool calls and completions) tail -n 100 logs/mcp_activity.log # Follow tool activity in real-time tail -f logs/mcp_activity.log # Use simple tail commands to monitor logs tail -f logs/mcp_activity.log | grep -E "(TOOL_CALL|TOOL_COMPLETED|ERROR|WARNING)" ``` #### Available Log Files **Current log files (with proper rotation):** ```bash # Main server log (all activity including debug info) - 20MB max, 10 backups tail -f logs/mcp_server.log # Tool activity only (TOOL_CALL, TOOL_COMPLETED, etc.) - 20MB max, 5 backups tail -f logs/mcp_activity.log ``` **For programmatic log analysis (used by tests):** ```python # Import the LogUtils class from simulator tests from simulator_tests.log_utils import LogUtils # Get recent logs recent_logs = LogUtils.get_recent_server_logs(lines=500) # Check for errors errors = LogUtils.check_server_logs_for_errors() # Search for specific patterns matches = LogUtils.search_logs_for_pattern("TOOL_CALL.*debug") ``` ### Testing Simulation tests are available to test the MCP server in a 'live' scenario, using your configured API keys to ensure the models are working and the server is able to communicate back and forth. **IMPORTANT**: After any code changes, restart your Claude session for the changes to take effect. #### Run All Simulator Tests ```bash # Run the complete test suite python communication_simulator_test.py # Run tests with verbose output python communication_simulator_test.py --verbose ``` #### Quick Test Mode (Recommended for Time-Limited Testing) ```bash # Run quick test mode - 6 essential tests that provide maximum functionality coverage python communication_simulator_test.py --quick # Run quick test mode with verbose output python communication_simulator_test.py --quick --verbose ``` **Quick mode runs these 6 essential tests:** - `cross_tool_continuation` - Cross-tool conversation memory testing (chat, thinkdeep, codereview, analyze, debug) - `conversation_chain_validation` - Core conversation threading and memory validation - `consensus_workflow_accurate` - Consensus tool with flash model and stance testing - `codereview_validation` - CodeReview tool with flash model and multi-step workflows - `planner_validation` - Planner tool with flash model and complex planning workflows - `token_allocation_validation` - Token allocation and conversation history buildup testing **Why these 6 tests:** They cover the core functionality including conversation memory (`utils/conversation_memory.py`), chat tool functionality, file processing and deduplication, model selection (flash/flashlite/o3), and cross-tool conversation workflows. These tests validate the most critical parts of the system in minimal time. **Note:** Some workflow tools (analyze, codereview, planner, consensus, etc.) require specific workflow parameters and may need individual testing rather than quick mode testing. #### Run Individual Simulator Tests (For Detailed Testing) ```bash # List all available tests python communication_simulator_test.py --list-tests # RECOMMENDED: Run tests individually for better isolation and debugging python communication_simulator_test.py --individual basic_conversation python communication_simulator_test.py --individual content_validation python communication_simulator_test.py --individual cross_tool_continuation python communication_simulator_test.py --individual memory_validation # Run multiple specific tests python communication_simulator_test.py --tests basic_conversation content_validation # Run individual test with verbose output for debugging python communication_simulator_test.py --individual memory_validation --verbose ``` Available simulator tests include: - `basic_conversation` - Basic conversation flow with chat tool - `content_validation` - Content validation and duplicate detection - `per_tool_deduplication` - File deduplication for individual tools - `cross_tool_continuation` - Cross-tool conversation continuation scenarios - `cross_tool_comprehensive` - Comprehensive cross-tool file deduplication and continuation - `line_number_validation` - Line number handling validation across tools - `memory_validation` - Conversation memory validation - `model_thinking_config` - Model-specific thinking configuration behavior - `o3_model_selection` - O3 model selection and usage validation - `ollama_custom_url` - Ollama custom URL endpoint functionality - `openrouter_fallback` - OpenRouter fallback behavior when only provider - `openrouter_models` - OpenRouter model functionality and alias mapping - `token_allocation_validation` - Token allocation and conversation history validation - `testgen_validation` - TestGen tool validation with specific test function - `refactor_validation` - Refactor tool validation with codesmells - `conversation_chain_validation` - Conversation chain and threading validation - `consensus_stance` - Consensus tool validation with stance steering (for/against/neutral) **Note**: All simulator tests should be run individually for optimal testing and better error isolation. #### Run Unit Tests Only ```bash # Run all unit tests (excluding integration tests that require API keys) python -m pytest tests/ -v -m "not integration" # Run specific test file python -m pytest tests/test_refactor.py -v # Run specific test function python -m pytest tests/test_refactor.py::TestRefactorTool::test_format_response -v # Run tests with coverage python -m pytest tests/ --cov=. --cov-report=html -m "not integration" ``` #### Run Integration Tests (Uses Free Local Models) **Setup Requirements:** ```bash # 1. Install Ollama (if not already installed) # Visit https://ollama.ai or use brew install ollama # 2. Start Ollama service ollama serve # 3. Pull a model (e.g., llama3.2) ollama pull llama3.2 # 4. Set environment variable for custom provider export CUSTOM_API_URL="http://localhost:11434" ``` **Run Integration Tests:** ```bash # Run integration tests that make real API calls to local models python -m pytest tests/ -v -m "integration" # Run specific integration test python -m pytest tests/test_prompt_regression.py::TestPromptIntegration::test_chat_normal_prompt -v # Run all tests (unit + integration) python -m pytest tests/ -v ``` **Note**: Integration tests use the local-llama model via Ollama, which is completely FREE to run unlimited times. Requires `CUSTOM_API_URL` environment variable set to your local Ollama endpoint. They can be run safely in CI/CD but are excluded from code quality checks to keep them fast. ### Development Workflow #### Before Making Changes 1. Ensure virtual environment is activated: `source .pal_venv/bin/activate` 2. Run quality checks: `./code_quality_checks.sh` 3. Check logs to ensure server is healthy: `tail -n 50 logs/mcp_server.log` #### After Making Changes 1. Run quality checks again: `./code_quality_checks.sh` 2. Run integration tests locally: `./run_integration_tests.sh` 3. Run quick test mode for fast validation: `python communication_simulator_test.py --quick` 4. Run relevant specific simulator tests if needed: `python communication_simulator_test.py --individual ` 5. Check logs for any issues: `tail -n 100 logs/mcp_server.log` 6. Restart Claude session to use updated code #### Before Committing/PR 1. Final quality check: `./code_quality_checks.sh` 2. Run integration tests: `./run_integration_tests.sh` 3. Run quick test mode: `python communication_simulator_test.py --quick` 4. Run full simulator test suite (optional): `./run_integration_tests.sh --with-simulator` 5. Verify all tests pass 100% ### Common Troubleshooting #### Server Issues ```bash # Check if Python environment is set up correctly ./run-server.sh # View recent errors grep "ERROR" logs/mcp_server.log | tail -20 # Check virtual environment which python # Should show: .../pal-mcp-server/.pal_venv/bin/python ``` #### Test Failures ```bash # First try quick test mode to see if it's a general issue python communication_simulator_test.py --quick --verbose # Run individual failing test with verbose output python communication_simulator_test.py --individual --verbose # Check server logs during test execution tail -f logs/mcp_server.log # Run tests with debug output LOG_LEVEL=DEBUG python communication_simulator_test.py --individual ``` #### Linting Issues ```bash # Auto-fix most linting issues ruff check . --fix black . isort . # Check what would be changed without applying ruff check . black --check . isort --check-only . ``` ### File Structure Context - `./code_quality_checks.sh` - Comprehensive quality check script - `./run-server.sh` - Server setup and management - `communication_simulator_test.py` - End-to-end testing framework - `simulator_tests/` - Individual test modules - `tests/` - Unit test suite - `tools/` - MCP tool implementations - `providers/` - AI provider implementations - `systemprompts/` - System prompt definitions - `logs/` - Server log files ### Environment Requirements - Python 3.9+ with virtual environment - All dependencies from `requirements.txt` installed - Proper API keys configured in `.env` file This guide provides everything needed to efficiently work with the PAL MCP Server codebase using Claude. Always run quality checks before and after making changes to ensure code integrity. ================================================ FILE: Dockerfile ================================================ # =========================================== # STAGE 1: Build dependencies # =========================================== FROM python:3.11-slim AS builder # Install system dependencies for building RUN apt-get update && apt-get install -y \ build-essential \ curl \ && rm -rf /var/lib/apt/lists/* # Set working directory WORKDIR /app # Copy requirements files COPY requirements.txt ./ # Create virtual environment and install dependencies RUN python -m venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" # Install Python dependencies RUN pip install --no-cache-dir --upgrade pip setuptools wheel && \ pip install --no-cache-dir -r requirements.txt # =========================================== # STAGE 2: Runtime image # =========================================== FROM python:3.11-slim AS runtime # Add metadata labels for traceability LABEL maintainer="PAL MCP Server Team" LABEL version="1.0.0" LABEL description="PAL MCP Server - AI-powered Model Context Protocol server" LABEL org.opencontainers.image.title="pal-mcp-server" LABEL org.opencontainers.image.description="AI-powered Model Context Protocol server with multi-provider support" LABEL org.opencontainers.image.version="1.0.0" LABEL org.opencontainers.image.source="https://github.com/BeehiveInnovations/pal-mcp-server" LABEL org.opencontainers.image.documentation="https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/README.md" LABEL org.opencontainers.image.licenses="Apache 2.0 License" # Create non-root user for security RUN groupadd -r paluser && useradd -r -g paluser paluser # Install minimal runtime dependencies RUN apt-get update && apt-get install -y \ ca-certificates \ procps \ && rm -rf /var/lib/apt/lists/* \ && apt-get clean # Copy virtual environment from builder COPY --from=builder /opt/venv /opt/venv ENV PATH="/opt/venv/bin:$PATH" # Set working directory WORKDIR /app # Copy application code COPY --chown=paluser:paluser . . # Create logs directory with proper permissions RUN mkdir -p logs && chown -R paluser:paluser logs # Create tmp directory for container operations RUN mkdir -p tmp && chown -R paluser:paluser tmp # Copy health check script COPY --chown=paluser:paluser docker/scripts/healthcheck.py /usr/local/bin/healthcheck.py RUN chmod +x /usr/local/bin/healthcheck.py # Switch to non-root user USER paluser # Health check configuration HEALTHCHECK --interval=30s --timeout=10s --start-period=5s --retries=3 \ CMD python /usr/local/bin/healthcheck.py # Set environment variables ENV PYTHONUNBUFFERED=1 ENV PYTHONPATH=/app # Default command CMD ["python", "server.py"] ================================================ FILE: LICENSE ================================================ Apache License Version 2.0, January 2004 http://www.apache.org/licenses/ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION 1. Definitions. "License" shall mean the terms and conditions for use, reproduction, and distribution as defined by Sections 1 through 9 of this document. "Licensor" shall mean the copyright owner or entity granting the License. "Legal Entity" shall mean the union of the acting entity and all other entities that control, are controlled by, or are under common control with that entity. For the purposes of this definition, "control" means (i) the power, direct or indirect, to cause the direction or management of such entity, whether by contract or otherwise, or (ii) ownership of fifty percent (50%) or more of the outstanding shares, or (iii) beneficial ownership of such entity. "You" (or "Your") shall mean an individual or Legal Entity exercising permissions granted by this License. "Source" shall mean the preferred form for making modifications, including but not limited to software source code, documentation source, and configuration files. "Object" shall mean any form resulting from mechanical transformation or translation of a Source form, including but not limited to compiled object code, generated documentation, and conversions to other media types. "Work" shall mean the work of authorship covered by this License, whether in source or binary form, which is made available under the License, as indicated by a copyright notice that is included in or attached to the work. (The copyright notice requirement does not apply to derivative works of the License holder.) "Derivative Works" shall mean any work, whether in Source or Object form, that is based upon (or derived from) the Work and for which the editorial revisions, annotations, elaborations, or other modifications represent, as a whole, an original work of authorship. For the purposes of this License, Derivative Works shall not include works that remain separable from, or merely link (or bind by name) to the interfaces of, the Work and derivative works thereof. "Contribution" shall mean any work of authorship, including the original version of the Work and any modifications or additions to that Work or Derivative Works thereof, that is intentionally submitted to Licensor for inclusion in the Work by the copyright owner or by an individual or Legal Entity authorized to submit on behalf of the copyright owner. For the purposes of this definition, "submitted" means any form of electronic, verbal, or written communication sent to the Licensor or its representatives, including but not limited to communication on electronic mailing lists, source code control systems, and issue tracking systems that are managed by, or on behalf of, the Licensor for the purpose of discussing and improving the Work, but excluding communication that is conspicuously marked or otherwise designated in writing by the copyright owner as "Not a Contribution." "Contributor" shall mean Licensor and any individual or Legal Entity on behalf of whom a Contribution has been received by Licensor and subsequently incorporated within the Work. 2. Grant of Copyright License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable copyright license to use, reproduce, modify, distribute, and otherwise transfer the Work as part of a Derivative Work. 3. Grant of Patent License. Subject to the terms and conditions of this License, each Contributor hereby grants to You a perpetual, worldwide, non-exclusive, no-charge, royalty-free, irrevocable (except as stated in this section) patent license to make, have made, use, offer to sell, sell, import, and otherwise transfer the Work, where such license applies only to those patent claims licensable by such Contributor that are necessarily infringed by their Contribution(s) alone or by combination of their Contribution(s) with the Work to which such Contribution(s) was submitted. If You institute patent litigation against any entity (including a cross-claim or counterclaim in a lawsuit) alleging that the Work or a Contribution incorporated within the Work constitutes direct or contributory patent infringement, then any patent licenses granted to You under this License for that Work shall terminate as of the date such litigation is filed. 4. Redistribution. You may reproduce and distribute copies of the Work or Derivative Works thereof in any medium, with or without modifications, and in Source or Object form, provided that You meet the following conditions: (a) You must give any other recipients of the Work or Derivative Works a copy of this License; and (b) You must cause any modified files to carry prominent notices stating that You changed the files; and (c) You must retain, in the Source form of any Derivative Works that You distribute, all copyright, patent, trademark, and attribution notices from the Source form of the Work, excluding those notices that do not pertain to any part of the Derivative Works; and (d) If the Work includes a "NOTICE" file as part of its distribution, then any Derivative Works that You distribute must include a readable copy of the attribution notices contained within such NOTICE file, excluding those notices that do not pertain to any part of the Derivative Works, in at least one of the following places: within a NOTICE file distributed as part of the Derivative Works; within the Source form or documentation, if provided along with the Derivative Works; or, within a display generated by the Derivative Works, if and wherever such third-party notices normally appear. The contents of the NOTICE file are for informational purposes only and do not modify the License. You may add Your own attribution notices within Derivative Works that You distribute, alongside or as an addendum to the NOTICE text from the Work, provided that such additional attribution notices cannot be construed as modifying the License. You may add Your own copyright notice to Your modifications and may provide additional or different license terms and conditions for use, reproduction, or distribution of Your modifications, or for any such Derivative Works as a whole, provided Your use, reproduction, and distribution of the Work otherwise complies with the conditions stated in this License. 5. Submission of Contributions. Unless You explicitly state otherwise, any Contribution intentionally submitted for inclusion in the Work by You to the Licensor shall be under the terms and conditions of this License, without any additional terms or conditions. Notwithstanding the above, nothing herein shall supersede or modify the terms of any separate license agreement you may have executed with Licensor regarding such Contributions. 6. Trademarks. This License does not grant permission to use the trade names, trademarks, service marks, or product names of the Licensor, except as required for reasonable and customary use in describing the origin of the Work and reproducing the content of the NOTICE file. 7. Disclaimer of Warranty. Unless required by applicable law or agreed to in writing, Licensor provides the Work (and each Contributor provides its Contributions) on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied, including, without limitation, any warranties or conditions of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A PARTICULAR PURPOSE. You are solely responsible for determining the appropriateness of using or redistributing the Work and assume any risks associated with Your exercise of permissions under this License. 8. Limitation of Liability. In no event and under no legal theory, whether in tort (including negligence), contract, or otherwise, unless required by applicable law (such as deliberate and grossly negligent acts) or agreed to in writing, shall any Contributor be liable to You for damages, including any direct, indirect, special, incidental, or consequential damages of any character arising as a result of this License or out of the use or inability to use the Work (including but not limited to damages for loss of goodwill, work stoppage, computer failure or malfunction, or any and all other commercial damages or losses), even if such Contributor has been advised of the possibility of such damages. 9. Accepting Warranty or Support. You may choose to offer, and to charge a fee for, warranty, support, indemnity or other liability obligations and/or rights consistent with this License. However, in accepting such obligations, You may act only on Your own behalf and on Your sole responsibility, not on behalf of any other Contributor, and only if You agree to indemnify, defend, and hold each Contributor harmless for any liability incurred by, or claims asserted against, such Contributor by reason of your accepting any such warranty or support. END OF TERMS AND CONDITIONS APPENDIX: How to apply the Apache License to your work. To apply the Apache License to your work, attach the following boilerplate notice, with the fields enclosed by brackets "[]" replaced with your own identifying information. (Don't include the brackets!) The text should be enclosed in comments for the particular file format. An identification line is also useful. Copyright 2025 Beehive Innovations https://github.com/BeehiveInnovations Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. ================================================ FILE: README.md ================================================ # PAL MCP: Many Workflows. One Context.
Your AI's PAL – a Provider Abstraction Layer
Formerly known as Zen MCP [PAL in action](https://github.com/user-attachments/assets/0d26061e-5f21-4ab1-b7d0-f883ddc2c3da) 👉 **[Watch more examples](#-watch-tools-in-action)** ### Your CLI + Multiple Models = Your AI Dev Team **Use the 🤖 CLI you love:** [Claude Code](https://www.anthropic.com/claude-code) · [Gemini CLI](https://github.com/google-gemini/gemini-cli) · [Codex CLI](https://github.com/openai/codex) · [Qwen Code CLI](https://qwenlm.github.io/qwen-code-docs/) · [Cursor](https://cursor.com) · _and more_ **With multiple models within a single prompt:** Gemini · OpenAI · Anthropic · Grok · Azure · Ollama · OpenRouter · DIAL · On-Device Model
--- ## 🆕 Now with CLI-to-CLI Bridge The new **[`clink`](docs/tools/clink.md)** (CLI + Link) tool connects external AI CLIs directly into your workflow: - **Connect external CLIs** like [Gemini CLI](https://github.com/google-gemini/gemini-cli), [Codex CLI](https://github.com/openai/codex), and [Claude Code](https://www.anthropic.com/claude-code) directly into your workflow - **CLI Subagents** - Launch isolated CLI instances from _within_ your current CLI! Claude Code can spawn Codex subagents, Codex can spawn Gemini CLI subagents, etc. Offload heavy tasks (code reviews, bug hunting) to fresh contexts while your main session's context window remains unpolluted. Each subagent returns only final results. - **Context Isolation** - Run separate investigations without polluting your primary workspace - **Role Specialization** - Spawn `planner`, `codereviewer`, or custom role agents with specialized system prompts - **Full CLI Capabilities** - Web search, file inspection, MCP tool access, latest documentation lookups - **Seamless Continuity** - Sub-CLIs participate as first-class members with full conversation context between tools ```bash # Codex spawns Codex subagent for isolated code review in fresh context clink with codex codereviewer to audit auth module for security issues # Subagent reviews in isolation, returns final report without cluttering your context as codex reads each file and walks the directory structure # Consensus from different AI models → Implementation handoff with full context preservation between tools Use consensus with gpt-5 and gemini-pro to decide: dark mode or offline support next Continue with clink gemini - implement the recommended feature # Gemini receives full debate context and starts coding immediately ``` 👉 **[Learn more about clink](docs/tools/clink.md)** --- ## Why PAL MCP? **Why rely on one AI model when you can orchestrate them all?** A Model Context Protocol server that supercharges tools like [Claude Code](https://www.anthropic.com/claude-code), [Codex CLI](https://developers.openai.com/codex/cli), and IDE clients such as [Cursor](https://cursor.com) or the [Claude Dev VS Code extension](https://marketplace.visualstudio.com/items?itemName=Anthropic.claude-vscode). **PAL MCP connects your favorite AI tool to multiple AI models** for enhanced code analysis, problem-solving, and collaborative development. ### True AI Collaboration with Conversation Continuity PAL supports **conversation threading** so your CLI can **discuss ideas with multiple AI models, exchange reasoning, get second opinions, and even run collaborative debates between models** to help you reach deeper insights and better solutions. Your CLI always stays in control but gets perspectives from the best AI for each subtask. Context carries forward seamlessly across tools and models, enabling complex workflows like: code reviews with multiple models → automated planning → implementation → pre-commit validation. > **You're in control.** Your CLI of choice orchestrates the AI team, but you decide the workflow. Craft powerful prompts that bring in Gemini Pro, GPT 5, Flash, or local offline models exactly when needed.
Reasons to Use PAL MCP A typical workflow with Claude Code as an example: 1. **Multi-Model Orchestration** - Claude coordinates with Gemini Pro, O3, GPT-5, and 50+ other models to get the best analysis for each task 2. **Context Revival Magic** - Even after Claude's context resets, continue conversations seamlessly by having other models "remind" Claude of the discussion 3. **Guided Workflows** - Enforces systematic investigation phases that prevent rushed analysis and ensure thorough code examination 4. **Extended Context Windows** - Break Claude's limits by delegating to Gemini (1M tokens) or O3 (200K tokens) for massive codebases 5. **True Conversation Continuity** - Full context flows across tools and models - Gemini remembers what O3 said 10 steps ago 6. **Model-Specific Strengths** - Extended thinking with Gemini Pro, blazing speed with Flash, strong reasoning with O3, privacy with local Ollama 7. **Professional Code Reviews** - Multi-pass analysis with severity levels, actionable feedback, and consensus from multiple AI experts 8. **Smart Debugging Assistant** - Systematic root cause analysis with hypothesis tracking and confidence levels 9. **Automatic Model Selection** - Claude intelligently picks the right model for each subtask (or you can specify) 10. **Vision Capabilities** - Analyze screenshots, diagrams, and visual content with vision-enabled models 11. **Local Model Support** - Run Llama, Mistral, or other models locally for complete privacy and zero API costs 12. **Bypass MCP Token Limits** - Automatically works around MCP's 25K limit for large prompts and responses **The Killer Feature:** When Claude's context resets, just ask to "continue with O3" - the other model's response magically revives Claude's understanding without re-ingesting documents! #### Example: Multi-Model Code Review Workflow 1. `Perform a codereview using gemini pro and o3 and use planner to generate a detailed plan, implement the fixes and do a final precommit check by continuing from the previous codereview` 2. This triggers a [`codereview`](docs/tools/codereview.md) workflow where Claude walks the code, looking for all kinds of issues 3. After multiple passes, collects relevant code and makes note of issues along the way 4. Maintains a `confidence` level between `exploring`, `low`, `medium`, `high` and `certain` to track how confidently it's been able to find and identify issues 5. Generates a detailed list of critical -> low issues 6. Shares the relevant files, findings, etc with **Gemini Pro** to perform a deep dive for a second [`codereview`](docs/tools/codereview.md) 7. Comes back with a response and next does the same with o3, adding to the prompt if a new discovery comes to light 8. When done, Claude takes in all the feedback and combines a single list of all critical -> low issues, including good patterns in your code. The final list includes new findings or revisions in case Claude misunderstood or missed something crucial and one of the other models pointed this out 9. It then uses the [`planner`](docs/tools/planner.md) workflow to break the work down into simpler steps if a major refactor is required 10. Claude then performs the actual work of fixing highlighted issues 11. When done, Claude returns to Gemini Pro for a [`precommit`](docs/tools/precommit.md) review All within a single conversation thread! Gemini Pro in step 11 _knows_ what was recommended by O3 in step 7! Taking that context and review into consideration to aid with its final pre-commit review. **Think of it as Claude Code _for_ Claude Code.** This MCP isn't magic. It's just **super-glue**. > **Remember:** Claude stays in full control — but **YOU** call the shots. > PAL is designed to have Claude engage other models only when needed — and to follow through with meaningful back-and-forth. > **You're** the one who crafts the powerful prompt that makes Claude bring in Gemini, Flash, O3 — or fly solo. > You're the guide. The prompter. The puppeteer. > #### You are the AI - **Actually Intelligent**.
#### Recommended AI Stack
For Claude Code Users For best results when using [Claude Code](https://claude.ai/code): - **Sonnet 4.5** - All agentic work and orchestration - **Gemini 3.0 Pro** OR **GPT-5.2 / Pro** - Deep thinking, additional code reviews, debugging and validations, pre-commit analysis
For Codex Users For best results when using [Codex CLI](https://developers.openai.com/codex/cli): - **GPT-5.2 Codex Medium** - All agentic work and orchestration - **Gemini 3.0 Pro** OR **GPT-5.2-Pro** - Deep thinking, additional code reviews, debugging and validations, pre-commit analysis
## Quick Start (5 minutes) **Prerequisites:** Python 3.10+, Git, [uv installed](https://docs.astral.sh/uv/getting-started/installation/) **1. Get API Keys** (choose one or more): - **[OpenRouter](https://openrouter.ai/)** - Access multiple models with one API - **[Gemini](https://makersuite.google.com/app/apikey)** - Google's latest models - **[OpenAI](https://platform.openai.com/api-keys)** - O3, GPT-5 series - **[Azure OpenAI](https://learn.microsoft.com/azure/ai-services/openai/)** - Enterprise deployments of GPT-4o, GPT-4.1, GPT-5 family - **[X.AI](https://console.x.ai/)** - Grok models - **[DIAL](https://dialx.ai/)** - Vendor-agnostic model access - **[Ollama](https://ollama.ai/)** - Local models (free) **2. Install** (choose one): **Option A: Clone and Automatic Setup** (recommended) ```bash git clone https://github.com/BeehiveInnovations/pal-mcp-server.git cd pal-mcp-server # Handles everything: setup, config, API keys from system environment. # Auto-configures Claude Desktop, Claude Code, Gemini CLI, Codex CLI, Qwen CLI # Enable / disable additional settings in .env ./run-server.sh ``` **Option B: Instant Setup with [uvx](https://docs.astral.sh/uv/getting-started/installation/)** ```json // Add to ~/.claude/settings.json or .mcp.json // Don't forget to add your API keys under env { "mcpServers": { "pal": { "command": "bash", "args": ["-c", "for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \"$p\" ] && exec \"$p\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1"], "env": { "PATH": "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:~/.local/bin", "GEMINI_API_KEY": "your-key-here", "DISABLED_TOOLS": "analyze,refactor,testgen,secaudit,docgen,tracer", "DEFAULT_MODEL": "auto" } } } } ``` **3. Start Using!** ``` "Use pal to analyze this code for security issues with gemini pro" "Debug this error with o3 and then get flash to suggest optimizations" "Plan the migration strategy with pal, get consensus from multiple models" "clink with cli_name=\"gemini\" role=\"planner\" to draft a phased rollout plan" ``` 👉 **[Complete Setup Guide](docs/getting-started.md)** with detailed installation, configuration for Gemini / Codex / Qwen, and troubleshooting 👉 **[Cursor & VS Code Setup](docs/getting-started.md#ide-clients)** for IDE integration instructions 📺 **[Watch tools in action](#-watch-tools-in-action)** to see real-world examples ## Provider Configuration PAL activates any provider that has credentials in your `.env`. See `.env.example` for deeper customization. ## Core Tools > **Note:** Each tool comes with its own multi-step workflow, parameters, and descriptions that consume valuable context window space even when not in use. To optimize performance, some tools are disabled by default. See [Tool Configuration](#tool-configuration) below to enable them. **Collaboration & Planning** *(Enabled by default)* - **[`clink`](docs/tools/clink.md)** - Bridge requests to external AI CLIs (Gemini planner, codereviewer, etc.) - **[`chat`](docs/tools/chat.md)** - Brainstorm ideas, get second opinions, validate approaches. With capable models (GPT-5.2 Pro, Gemini 3.0 Pro), generates complete code / implementation - **[`thinkdeep`](docs/tools/thinkdeep.md)** - Extended reasoning, edge case analysis, alternative perspectives - **[`planner`](docs/tools/planner.md)** - Break down complex projects into structured, actionable plans - **[`consensus`](docs/tools/consensus.md)** - Get expert opinions from multiple AI models with stance steering **Code Analysis & Quality** - **[`debug`](docs/tools/debug.md)** - Systematic investigation and root cause analysis - **[`precommit`](docs/tools/precommit.md)** - Validate changes before committing, prevent regressions - **[`codereview`](docs/tools/codereview.md)** - Professional reviews with severity levels and actionable feedback - **[`analyze`](docs/tools/analyze.md)** *(disabled by default - [enable](#tool-configuration))* - Understand architecture, patterns, dependencies across entire codebases **Development Tools** *(Disabled by default - [enable](#tool-configuration))* - **[`refactor`](docs/tools/refactor.md)** - Intelligent code refactoring with decomposition focus - **[`testgen`](docs/tools/testgen.md)** - Comprehensive test generation with edge cases - **[`secaudit`](docs/tools/secaudit.md)** - Security audits with OWASP Top 10 analysis - **[`docgen`](docs/tools/docgen.md)** - Generate documentation with complexity analysis **Utilities** - **[`apilookup`](docs/tools/apilookup.md)** - Forces current-year API/SDK documentation lookups in a sub-process (saves tokens within the current context window), prevents outdated training data responses - **[`challenge`](docs/tools/challenge.md)** - Prevent "You're absolutely right!" responses with critical analysis - **[`tracer`](docs/tools/tracer.md)** *(disabled by default - [enable](#tool-configuration))* - Static analysis prompts for call-flow mapping
👉 Tool Configuration ### Default Configuration To optimize context window usage, only essential tools are enabled by default: **Enabled by default:** - `chat`, `thinkdeep`, `planner`, `consensus` - Core collaboration tools - `codereview`, `precommit`, `debug` - Essential code quality tools - `apilookup` - Rapid API/SDK information lookup - `challenge` - Critical thinking utility **Disabled by default:** - `analyze`, `refactor`, `testgen`, `secaudit`, `docgen`, `tracer` ### Enabling Additional Tools To enable additional tools, remove them from the `DISABLED_TOOLS` list: **Option 1: Edit your .env file** ```bash # Default configuration (from .env.example) DISABLED_TOOLS=analyze,refactor,testgen,secaudit,docgen,tracer # To enable specific tools, remove them from the list # Example: Enable analyze tool DISABLED_TOOLS=refactor,testgen,secaudit,docgen,tracer # To enable ALL tools DISABLED_TOOLS= ``` **Option 2: Configure in MCP settings** ```json // In ~/.claude/settings.json or .mcp.json { "mcpServers": { "pal": { "env": { // Tool configuration "DISABLED_TOOLS": "refactor,testgen,secaudit,docgen,tracer", "DEFAULT_MODEL": "pro", "DEFAULT_THINKING_MODE_THINKDEEP": "high", // API configuration "GEMINI_API_KEY": "your-gemini-key", "OPENAI_API_KEY": "your-openai-key", "OPENROUTER_API_KEY": "your-openrouter-key", // Logging and performance "LOG_LEVEL": "INFO", "CONVERSATION_TIMEOUT_HOURS": "6", "MAX_CONVERSATION_TURNS": "50" } } } } ``` **Option 3: Enable all tools** ```json // Remove or empty the DISABLED_TOOLS to enable everything { "mcpServers": { "pal": { "env": { "DISABLED_TOOLS": "" } } } } ``` **Note:** - Essential tools (`version`, `listmodels`) cannot be disabled - After changing tool configuration, restart your Claude session for changes to take effect - Each tool adds to context window usage, so only enable what you need
## 📺 Watch Tools In Action
Chat Tool - Collaborative decision making and multi-turn conversations **Picking Redis vs Memcached:** [Chat Redis or Memcached_web.webm](https://github.com/user-attachments/assets/41076cfe-dd49-4dfc-82f5-d7461b34705d) **Multi-turn conversation with continuation:** [Chat With Gemini_web.webm](https://github.com/user-attachments/assets/37bd57ca-e8a6-42f7-b5fb-11de271e95db)
Consensus Tool - Multi-model debate and decision making **Multi-model consensus debate:** [PAL Consensus Debate](https://github.com/user-attachments/assets/76a23dd5-887a-4382-9cf0-642f5cf6219e)
PreCommit Tool - Comprehensive change validation **Pre-commit validation workflow:**
API Lookup Tool - Current vs outdated API documentation **Without PAL - outdated APIs:** [API without PAL](https://github.com/user-attachments/assets/01a79dc9-ad16-4264-9ce1-76a56c3580ee) **With PAL - current APIs:** [API with PAL](https://github.com/user-attachments/assets/5c847326-4b66-41f7-8f30-f380453dce22)
Challenge Tool - Critical thinking vs reflexive agreement **Without PAL:** ![without_pal@2x](https://github.com/user-attachments/assets/64f3c9fb-7ca9-4876-b687-25e847edfd87) **With PAL:** ![with_pal@2x](https://github.com/user-attachments/assets/9d72f444-ba53-4ab1-83e5-250062c6ee70)
## Key Features **AI Orchestration** - **Auto model selection** - Claude picks the right AI for each task - **Multi-model workflows** - Chain different models in single conversations - **Conversation continuity** - Context preserved across tools and models - **[Context revival](docs/context-revival.md)** - Continue conversations even after context resets **Model Support** - **Multiple providers** - Gemini, OpenAI, Azure, X.AI, OpenRouter, DIAL, Ollama - **Latest models** - GPT-5, Gemini 3.0 Pro, O3, Grok-4, local Llama - **[Thinking modes](docs/advanced-usage.md#thinking-modes)** - Control reasoning depth vs cost - **Vision support** - Analyze images, diagrams, screenshots **Developer Experience** - **Guided workflows** - Systematic investigation prevents rushed analysis - **Smart file handling** - Auto-expand directories, manage token limits - **Web search integration** - Access current documentation and best practices - **[Large prompt support](docs/advanced-usage.md#working-with-large-prompts)** - Bypass MCP's 25K token limit ## Example Workflows **Multi-model Code Review:** ``` "Perform a codereview using gemini pro and o3, then use planner to create a fix strategy" ``` → Claude reviews code systematically → Consults Gemini Pro → Gets O3's perspective → Creates unified action plan **Collaborative Debugging:** ``` "Debug this race condition with max thinking mode, then validate the fix with precommit" ``` → Deep investigation → Expert analysis → Solution implementation → Pre-commit validation **Architecture Planning:** ``` "Plan our microservices migration, get consensus from pro and o3 on the approach" ``` → Structured planning → Multiple expert opinions → Consensus building → Implementation roadmap 👉 **[Advanced Usage Guide](docs/advanced-usage.md)** for complex workflows, model configuration, and power-user features ## Quick Links **📖 Documentation** - [Docs Overview](docs/index.md) - High-level map of major guides - [Getting Started](docs/getting-started.md) - Complete setup guide - [Tools Reference](docs/tools/) - All tools with examples - [Advanced Usage](docs/advanced-usage.md) - Power user features - [Configuration](docs/configuration.md) - Environment variables, restrictions - [Adding Providers](docs/adding_providers.md) - Provider-specific setup (OpenAI, Azure, custom gateways) - [Model Ranking Guide](docs/model_ranking.md) - How intelligence scores drive auto-mode suggestions **🔧 Setup & Support** - [WSL Setup](docs/wsl-setup.md) - Windows users - [Troubleshooting](docs/troubleshooting.md) - Common issues - [Contributing](docs/contributions.md) - Code standards, PR process ## License Apache 2.0 License - see [LICENSE](LICENSE) file for details. ## Acknowledgments Built with the power of **Multi-Model AI** collaboration 🤝 - **A**ctual **I**ntelligence by real Humans - [MCP (Model Context Protocol)](https://modelcontextprotocol.com) - [Codex CLI](https://developers.openai.com/codex/cli) - [Claude Code](https://claude.ai/code) - [Gemini](https://ai.google.dev/) - [OpenAI](https://openai.com/) - [Azure OpenAI](https://learn.microsoft.com/azure/ai-services/openai/) ### Star History [![Star History Chart](https://api.star-history.com/svg?repos=BeehiveInnovations/pal-mcp-server&type=Date)](https://www.star-history.com/#BeehiveInnovations/pal-mcp-server&Date) ================================================ FILE: SECURITY.md ================================================ # Security Policy ## Supported Versions | Version | Supported | | ------- | ------------------ | | 9.x.x | :white_check_mark: | | < 9.0 | :x: | ## Important Disclaimer PAL MCP is an open-source Model Context Protocol (MCP) server that acts as middleware between AI clients (Claude Code, Codex CLI, Cursor, etc.) and various AI model providers. **Please understand the following:** - **No Warranty**: This software is provided "AS IS" under the Apache 2.0 License, without warranties of any kind. See the [LICENSE](LICENSE) file for full terms. - **User Responsibility**: The AI client (not PAL MCP) controls tool invocations and workflows. Users are responsible for reviewing AI-generated outputs and actions. - **API Key Security**: You are responsible for securing your own API keys. Never commit keys to version control or share them publicly. - **Third-Party Services**: PAL MCP connects to external AI providers (Google, OpenAI, Azure, etc.). Their terms of service and privacy policies apply to data sent through this server. ## Reporting a Vulnerability **Please do not report security vulnerabilities through public GitHub issues.** ### Preferred Method Use [GitHub Security Advisories](https://github.com/BeehiveInnovations/pal-mcp-server/security/advisories/new) to report vulnerabilities privately. ### What to Include - Description of the vulnerability - Steps to reproduce - Affected versions - Potential impact - Suggested fix (optional) ### What to Expect - We will acknowledge your report and assess the issue - Critical issues will be prioritized - We'll keep you informed of progress as work proceeds We cannot commit to specific response timelines, but we take security seriously. ### After Resolution We welcome security researchers to submit a pull request with the fix. This is an open-source project and we appreciate community contributions to improve security. ## Disclosure Policy We practice coordinated disclosure. Please allow reasonable time to address issues before public disclosure. We'll work with you on timing. ## Scope ### In Scope - Authentication/authorization bypasses - Injection vulnerabilities (command injection, prompt injection with security impact) - Information disclosure (API keys, sensitive data leakage) - Denial of service vulnerabilities in the MCP server itself - Dependency vulnerabilities with exploitable impact ### Out of Scope - Issues in upstream AI providers (report to Google, OpenAI, etc. directly) - Issues in AI client software (report to Anthropic, OpenAI, Cursor, etc.) - AI model behavior or outputs (this is controlled by the AI client and model providers) - Social engineering attacks - Rate limiting or resource exhaustion on third-party APIs ## Security Best Practices for Users 1. **Protect API Keys**: Store keys in `.env` files (gitignored) or environment variables 2. **Review AI Actions**: Always review AI-suggested code changes before applying 3. **Use Local Models**: For sensitive codebases, consider using Ollama with local models 4. **Network Security**: When self-hosting, ensure appropriate network controls 5. **Keep Updated**: Regularly update to the latest version for security fixes ## Recognition We appreciate responsible disclosure and will credit security researchers in release notes (unless you prefer anonymity). ================================================ FILE: claude_config_example.json ================================================ { "comment": "Example Claude Desktop configuration for PAL MCP Server", "comment2": "Run './run-server.sh -c' to get the exact configuration for your system", "comment3": "For platform-specific examples, see the examples/ directory", "mcpServers": { "pal": { "command": "/path/to/pal-mcp-server/.pal_venv/bin/python", "args": ["/path/to/pal-mcp-server/server.py"] } } } ================================================ FILE: clink/__init__.py ================================================ """Public helpers for clink components.""" from __future__ import annotations from .registry import ClinkRegistry, get_registry __all__ = ["ClinkRegistry", "get_registry"] ================================================ FILE: clink/agents/__init__.py ================================================ """Agent factory for clink CLI integrations.""" from __future__ import annotations from clink.models import ResolvedCLIClient from .base import AgentOutput, BaseCLIAgent, CLIAgentError from .claude import ClaudeAgent from .codex import CodexAgent from .gemini import GeminiAgent _AGENTS: dict[str, type[BaseCLIAgent]] = { "gemini": GeminiAgent, "codex": CodexAgent, "claude": ClaudeAgent, } def create_agent(client: ResolvedCLIClient) -> BaseCLIAgent: agent_key = (client.runner or client.name).lower() agent_cls = _AGENTS.get(agent_key, BaseCLIAgent) return agent_cls(client) __all__ = [ "AgentOutput", "BaseCLIAgent", "CLIAgentError", "create_agent", ] ================================================ FILE: clink/agents/base.py ================================================ """Execute configured CLI agents for the clink tool and parse output.""" from __future__ import annotations import asyncio import logging import os import shlex import shutil import tempfile import time from collections.abc import Sequence from dataclasses import dataclass from pathlib import Path from clink.constants import DEFAULT_STREAM_LIMIT from clink.models import ResolvedCLIClient, ResolvedCLIRole from clink.parsers import BaseParser, ParsedCLIResponse, ParserError, get_parser logger = logging.getLogger("clink.agent") @dataclass class AgentOutput: """Container returned by CLI agents after successful execution.""" parsed: ParsedCLIResponse sanitized_command: list[str] returncode: int stdout: str stderr: str duration_seconds: float parser_name: str output_file_content: str | None = None class CLIAgentError(RuntimeError): """Raised when a CLI agent fails (non-zero exit, timeout, parse errors).""" def __init__(self, message: str, *, returncode: int | None = None, stdout: str = "", stderr: str = "") -> None: super().__init__(message) self.returncode = returncode self.stdout = stdout self.stderr = stderr class BaseCLIAgent: """Execute a configured CLI command and parse its output.""" def __init__(self, client: ResolvedCLIClient): self.client = client self._parser: BaseParser = get_parser(client.parser) self._logger = logging.getLogger(f"clink.runner.{client.name}") async def run( self, *, role: ResolvedCLIRole, prompt: str, system_prompt: str | None = None, files: Sequence[str], images: Sequence[str], ) -> AgentOutput: # Files and images are already embedded into the prompt by the tool; they are # accepted here only to keep parity with SimpleTool callers. _ = (files, images) # The runner simply executes the configured CLI command for the selected role. command = self._build_command(role=role, system_prompt=system_prompt) env = self._build_environment() # Resolve executable path for cross-platform compatibility (especially Windows) executable_name = command[0] resolved_executable = shutil.which(executable_name) if resolved_executable is None: raise CLIAgentError( f"Executable '{executable_name}' not found in PATH for CLI '{self.client.name}'. " f"Ensure the command is installed and accessible." ) command[0] = resolved_executable sanitized_command = list(command) cwd = str(self.client.working_dir) if self.client.working_dir else None limit = DEFAULT_STREAM_LIMIT stdout_text = "" stderr_text = "" output_file_content: str | None = None start_time = time.monotonic() output_file_path: Path | None = None command_with_output_flag = list(command) if self.client.output_to_file: fd, tmp_path = tempfile.mkstemp(prefix="clink-", suffix=".json") os.close(fd) output_file_path = Path(tmp_path) flag_template = self.client.output_to_file.flag_template try: rendered_flag = flag_template.format(path=str(output_file_path)) except KeyError as exc: # pragma: no cover - defensive raise CLIAgentError(f"Invalid output flag template '{flag_template}': missing placeholder {exc}") command_with_output_flag.extend(shlex.split(rendered_flag)) sanitized_command = list(command_with_output_flag) self._logger.debug("Executing CLI command: %s", " ".join(sanitized_command)) if cwd: self._logger.debug("Working directory: %s", cwd) try: process = await asyncio.create_subprocess_exec( *command_with_output_flag, stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE, stderr=asyncio.subprocess.PIPE, cwd=cwd, limit=limit, env=env, ) except FileNotFoundError as exc: raise CLIAgentError(f"Executable not found for CLI '{self.client.name}': {exc}") from exc try: stdout_bytes, stderr_bytes = await asyncio.wait_for( process.communicate(prompt.encode("utf-8")), timeout=self.client.timeout_seconds, ) except asyncio.TimeoutError as exc: process.kill() await process.communicate() raise CLIAgentError( f"CLI '{self.client.name}' timed out after {self.client.timeout_seconds} seconds", returncode=None, ) from exc duration = time.monotonic() - start_time return_code = process.returncode stdout_text = stdout_bytes.decode("utf-8", errors="replace") stderr_text = stderr_bytes.decode("utf-8", errors="replace") if output_file_path and output_file_path.exists(): output_file_content = output_file_path.read_text(encoding="utf-8", errors="replace") if self.client.output_to_file and self.client.output_to_file.cleanup: try: output_file_path.unlink() except OSError: # pragma: no cover - best effort cleanup pass if output_file_content and not stdout_text.strip(): stdout_text = output_file_content if return_code != 0: recovered = self._recover_from_error( returncode=return_code, stdout=stdout_text, stderr=stderr_text, sanitized_command=sanitized_command, duration_seconds=duration, output_file_content=output_file_content, ) if recovered is not None: return recovered if return_code != 0: raise CLIAgentError( f"CLI '{self.client.name}' exited with status {return_code}", returncode=return_code, stdout=stdout_text, stderr=stderr_text, ) try: parsed = self._parser.parse(stdout_text, stderr_text) except ParserError as exc: raise CLIAgentError( f"Failed to parse output from CLI '{self.client.name}': {exc}", returncode=return_code, stdout=stdout_text, stderr=stderr_text, ) from exc return AgentOutput( parsed=parsed, sanitized_command=sanitized_command, returncode=return_code, stdout=stdout_text, stderr=stderr_text, duration_seconds=duration, parser_name=self._parser.name, output_file_content=output_file_content, ) def _build_command(self, *, role: ResolvedCLIRole, system_prompt: str | None) -> list[str]: base = list(self.client.executable) base.extend(self.client.internal_args) base.extend(self.client.config_args) base.extend(role.role_args) return base def _build_environment(self) -> dict[str, str]: env = os.environ.copy() env.update(self.client.env) return env # ------------------------------------------------------------------ # Error recovery hooks # ------------------------------------------------------------------ def _recover_from_error( self, *, returncode: int, stdout: str, stderr: str, sanitized_command: list[str], duration_seconds: float, output_file_content: str | None, ) -> AgentOutput | None: """Hook for subclasses to convert CLI errors into successful outputs. Return an AgentOutput to treat the failure as success, or None to signal that normal error handling should proceed. """ return None ================================================ FILE: clink/agents/claude.py ================================================ """Claude-specific CLI agent hooks.""" from __future__ import annotations from clink.models import ResolvedCLIRole from clink.parsers.base import ParserError from .base import AgentOutput, BaseCLIAgent class ClaudeAgent(BaseCLIAgent): """Claude CLI agent with system-prompt injection support.""" def _build_command(self, *, role: ResolvedCLIRole, system_prompt: str | None) -> list[str]: command = list(self.client.executable) command.extend(self.client.internal_args) command.extend(self.client.config_args) if system_prompt and "--append-system-prompt" not in self.client.config_args: command.extend(["--append-system-prompt", system_prompt]) command.extend(role.role_args) return command def _recover_from_error( self, *, returncode: int, stdout: str, stderr: str, sanitized_command: list[str], duration_seconds: float, output_file_content: str | None, ) -> AgentOutput | None: try: parsed = self._parser.parse(stdout, stderr) except ParserError: return None return AgentOutput( parsed=parsed, sanitized_command=sanitized_command, returncode=returncode, stdout=stdout, stderr=stderr, duration_seconds=duration_seconds, parser_name=self._parser.name, output_file_content=output_file_content, ) ================================================ FILE: clink/agents/codex.py ================================================ """Codex-specific CLI agent hooks.""" from __future__ import annotations from clink.models import ResolvedCLIClient from clink.parsers.base import ParserError from .base import AgentOutput, BaseCLIAgent class CodexAgent(BaseCLIAgent): """Codex CLI agent with JSONL recovery support.""" def __init__(self, client: ResolvedCLIClient): super().__init__(client) def _recover_from_error( self, *, returncode: int, stdout: str, stderr: str, sanitized_command: list[str], duration_seconds: float, output_file_content: str | None, ) -> AgentOutput | None: try: parsed = self._parser.parse(stdout, stderr) except ParserError: return None return AgentOutput( parsed=parsed, sanitized_command=sanitized_command, returncode=returncode, stdout=stdout, stderr=stderr, duration_seconds=duration_seconds, parser_name=self._parser.name, output_file_content=output_file_content, ) ================================================ FILE: clink/agents/gemini.py ================================================ """Gemini-specific CLI agent hooks.""" from __future__ import annotations import json from typing import Any from clink.models import ResolvedCLIClient from clink.parsers.base import ParsedCLIResponse from .base import AgentOutput, BaseCLIAgent class GeminiAgent(BaseCLIAgent): """Gemini-specific behaviour.""" def __init__(self, client: ResolvedCLIClient): super().__init__(client) def _recover_from_error( self, *, returncode: int, stdout: str, stderr: str, sanitized_command: list[str], duration_seconds: float, output_file_content: str | None, ) -> AgentOutput | None: combined = "\n".join(part for part in (stderr, stdout) if part) if not combined: return None brace_index = combined.find("{") if brace_index == -1: return None json_candidate = combined[brace_index:] try: payload: dict[str, Any] = json.loads(json_candidate) except json.JSONDecodeError: return None error_block = payload.get("error") if not isinstance(error_block, dict): return None code = error_block.get("code") err_type = error_block.get("type") detail_message = error_block.get("message") prologue = combined[:brace_index].strip() lines: list[str] = [] if prologue and (not detail_message or prologue not in detail_message): lines.append(prologue) if detail_message: lines.append(detail_message) header = "Gemini CLI reported a tool failure" if code: header = f"{header} ({code})" elif err_type: header = f"{header} ({err_type})" content_lines = [header.rstrip(".") + "."] content_lines.extend(lines) message = "\n".join(content_lines).strip() metadata = { "cli_error_recovered": True, "cli_error_code": code, "cli_error_type": err_type, "cli_error_payload": payload, } parsed = ParsedCLIResponse(content=message or header, metadata=metadata) return AgentOutput( parsed=parsed, sanitized_command=sanitized_command, returncode=returncode, stdout=stdout, stderr=stderr, duration_seconds=duration_seconds, parser_name=self._parser.name, output_file_content=output_file_content, ) ================================================ FILE: clink/constants.py ================================================ """Internal defaults and constants for clink.""" from __future__ import annotations from dataclasses import dataclass, field from pathlib import Path DEFAULT_TIMEOUT_SECONDS = 1800 DEFAULT_STREAM_LIMIT = 10 * 1024 * 1024 # 10MB per stream PROJECT_ROOT = Path(__file__).resolve().parent.parent BUILTIN_PROMPTS_DIR = PROJECT_ROOT / "systemprompts" / "clink" CONFIG_DIR = PROJECT_ROOT / "conf" / "cli_clients" USER_CONFIG_DIR = Path.home() / ".pal" / "cli_clients" @dataclass(frozen=True) class CLIInternalDefaults: """Internal defaults applied to a CLI client during registry load.""" parser: str additional_args: list[str] = field(default_factory=list) env: dict[str, str] = field(default_factory=dict) default_role_prompt: str | None = None timeout_seconds: int = DEFAULT_TIMEOUT_SECONDS runner: str | None = None INTERNAL_DEFAULTS: dict[str, CLIInternalDefaults] = { "gemini": CLIInternalDefaults( parser="gemini_json", additional_args=["-o", "json"], default_role_prompt="systemprompts/clink/default.txt", runner="gemini", ), "codex": CLIInternalDefaults( parser="codex_jsonl", additional_args=["exec"], default_role_prompt="systemprompts/clink/default.txt", runner="codex", ), "claude": CLIInternalDefaults( parser="claude_json", additional_args=["--print", "--output-format", "json"], default_role_prompt="systemprompts/clink/default.txt", runner="claude", ), } ================================================ FILE: clink/models.py ================================================ """Pydantic models for clink configuration and runtime structures.""" from __future__ import annotations from pathlib import Path from typing import Any from pydantic import BaseModel, Field, PositiveInt, field_validator class OutputCaptureConfig(BaseModel): """Optional configuration for CLIs that write output to disk.""" flag_template: str = Field(..., description="Template used to inject the output path, e.g. '--output {path}'.") cleanup: bool = Field( default=True, description="Whether the temporary file should be removed after reading.", ) class CLIRoleConfig(BaseModel): """Role-specific configuration loaded from JSON manifests.""" prompt_path: str | None = Field( default=None, description="Path to the prompt file that seeds this role.", ) role_args: list[str] = Field(default_factory=list) description: str | None = Field(default=None) @field_validator("role_args", mode="before") @classmethod def _ensure_list(cls, value: Any) -> list[str]: if value is None: return [] if isinstance(value, list): return [str(item) for item in value] if isinstance(value, str): return [value] raise TypeError("role_args must be a list of strings or a single string") class CLIClientConfig(BaseModel): """Raw CLI client configuration before internal defaults are applied.""" name: str command: str | None = None working_dir: str | None = None additional_args: list[str] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) timeout_seconds: PositiveInt | None = Field(default=None) roles: dict[str, CLIRoleConfig] = Field(default_factory=dict) output_to_file: OutputCaptureConfig | None = None @field_validator("additional_args", mode="before") @classmethod def _ensure_args_list(cls, value: Any) -> list[str]: if value is None: return [] if isinstance(value, list): return [str(item) for item in value] if isinstance(value, str): return [value] raise TypeError("additional_args must be a list of strings or a single string") class ResolvedCLIRole(BaseModel): """Runtime representation of a CLI role with resolved prompt path.""" name: str prompt_path: Path role_args: list[str] = Field(default_factory=list) description: str | None = None class ResolvedCLIClient(BaseModel): """Runtime configuration after merging defaults and validating paths.""" name: str executable: list[str] working_dir: Path | None internal_args: list[str] = Field(default_factory=list) config_args: list[str] = Field(default_factory=list) env: dict[str, str] = Field(default_factory=dict) timeout_seconds: int parser: str runner: str | None = None roles: dict[str, ResolvedCLIRole] output_to_file: OutputCaptureConfig | None = None def list_roles(self) -> list[str]: return list(self.roles.keys()) def get_role(self, role_name: str | None) -> ResolvedCLIRole: key = role_name or "default" if key not in self.roles: available = ", ".join(sorted(self.roles.keys())) raise KeyError(f"Role '{role_name}' not configured for CLI '{self.name}'. Available roles: {available}") return self.roles[key] ================================================ FILE: clink/parsers/__init__.py ================================================ """Parser registry for clink.""" from __future__ import annotations from .base import BaseParser, ParsedCLIResponse, ParserError from .claude import ClaudeJSONParser from .codex import CodexJSONLParser from .gemini import GeminiJSONParser _PARSER_CLASSES: dict[str, type[BaseParser]] = { CodexJSONLParser.name: CodexJSONLParser, GeminiJSONParser.name: GeminiJSONParser, ClaudeJSONParser.name: ClaudeJSONParser, } def get_parser(name: str) -> BaseParser: normalized = (name or "").lower() if normalized not in _PARSER_CLASSES: raise ParserError(f"No parser registered for '{name}'") parser_cls = _PARSER_CLASSES[normalized] return parser_cls() __all__ = [ "BaseParser", "ParsedCLIResponse", "ParserError", "get_parser", ] ================================================ FILE: clink/parsers/base.py ================================================ """Parser interfaces for clink runner outputs.""" from __future__ import annotations from dataclasses import dataclass from typing import Any @dataclass class ParsedCLIResponse: """Result of parsing CLI stdout/stderr.""" content: str metadata: dict[str, Any] class ParserError(RuntimeError): """Raised when CLI output cannot be parsed into a structured response.""" class BaseParser: """Base interface for CLI output parsers.""" name: str = "base" def parse(self, stdout: str, stderr: str) -> ParsedCLIResponse: raise NotImplementedError("Parsers must implement parse()") ================================================ FILE: clink/parsers/claude.py ================================================ """Parser for Claude CLI JSON output.""" from __future__ import annotations import json from typing import Any from .base import BaseParser, ParsedCLIResponse, ParserError class ClaudeJSONParser(BaseParser): """Parse stdout produced by `claude --output-format json`.""" name = "claude_json" def parse(self, stdout: str, stderr: str) -> ParsedCLIResponse: if not stdout.strip(): raise ParserError("Claude CLI returned empty stdout while JSON output was expected") try: loaded = json.loads(stdout) except json.JSONDecodeError as exc: # pragma: no cover - defensive logging raise ParserError(f"Failed to decode Claude CLI JSON output: {exc}") from exc events: list[dict[str, Any]] | None = None assistant_entry: dict[str, Any] | None = None if isinstance(loaded, dict): payload: dict[str, Any] = loaded elif isinstance(loaded, list): events = [item for item in loaded if isinstance(item, dict)] result_entry = next( (item for item in events if item.get("type") == "result" or "result" in item), None, ) assistant_entry = next( (item for item in reversed(events) if item.get("type") == "assistant"), None, ) payload = result_entry or assistant_entry or (events[-1] if events else {}) if not payload: raise ParserError("Claude CLI JSON array did not contain any parsable objects") else: raise ParserError("Claude CLI returned unexpected JSON payload") metadata = self._build_metadata(payload, stderr) if events is not None: metadata["raw_events"] = events metadata["raw"] = loaded result = payload.get("result") content: str = "" if isinstance(result, str): content = result.strip() elif isinstance(result, list): # Some CLI flows may emit a list of strings; join them conservatively. joined = [part.strip() for part in result if isinstance(part, str) and part.strip()] content = "\n".join(joined) if content: return ParsedCLIResponse(content=content, metadata=metadata) message = self._extract_message(payload) if message is None and assistant_entry and assistant_entry is not payload: message = self._extract_message(assistant_entry) if message: return ParsedCLIResponse(content=message, metadata=metadata) stderr_text = stderr.strip() if stderr_text: metadata.setdefault("stderr", stderr_text) return ParsedCLIResponse( content="Claude CLI returned no textual result. Raw stderr was preserved for troubleshooting.", metadata=metadata, ) raise ParserError("Claude CLI response did not contain a textual result") def _build_metadata(self, payload: dict[str, Any], stderr: str) -> dict[str, Any]: metadata: dict[str, Any] = { "raw": payload, "is_error": bool(payload.get("is_error")), } type_field = payload.get("type") if isinstance(type_field, str): metadata["type"] = type_field subtype_field = payload.get("subtype") if isinstance(subtype_field, str): metadata["subtype"] = subtype_field duration_ms = payload.get("duration_ms") if isinstance(duration_ms, (int, float)): metadata["duration_ms"] = duration_ms api_duration = payload.get("duration_api_ms") if isinstance(api_duration, (int, float)): metadata["duration_api_ms"] = api_duration usage = payload.get("usage") if isinstance(usage, dict): metadata["usage"] = usage model_usage = payload.get("modelUsage") if isinstance(model_usage, dict) and model_usage: metadata["model_usage"] = model_usage first_model = next(iter(model_usage.keys())) metadata["model_used"] = first_model permission_denials = payload.get("permission_denials") if isinstance(permission_denials, list) and permission_denials: metadata["permission_denials"] = permission_denials session_id = payload.get("session_id") if isinstance(session_id, str) and session_id: metadata["session_id"] = session_id uuid_field = payload.get("uuid") if isinstance(uuid_field, str) and uuid_field: metadata["uuid"] = uuid_field stderr_text = stderr.strip() if stderr_text: metadata.setdefault("stderr", stderr_text) return metadata def _extract_message(self, payload: dict[str, Any]) -> str | None: message = payload.get("message") if isinstance(message, str) and message.strip(): return message.strip() error_field = payload.get("error") if isinstance(error_field, dict): error_message = error_field.get("message") if isinstance(error_message, str) and error_message.strip(): return error_message.strip() return None ================================================ FILE: clink/parsers/codex.py ================================================ """Parser for Codex CLI JSONL output.""" from __future__ import annotations import json from typing import Any from .base import BaseParser, ParsedCLIResponse, ParserError class CodexJSONLParser(BaseParser): """Parse stdout emitted by `codex exec --json`.""" name = "codex_jsonl" def parse(self, stdout: str, stderr: str) -> ParsedCLIResponse: lines = [line.strip() for line in (stdout or "").splitlines() if line.strip()] events: list[dict[str, Any]] = [] agent_messages: list[str] = [] errors: list[str] = [] usage: dict[str, Any] | None = None for line in lines: if not line.startswith("{"): continue try: event = json.loads(line) except json.JSONDecodeError: continue events.append(event) event_type = event.get("type") if event_type == "item.completed": item = event.get("item") or {} if item.get("type") == "agent_message": text = item.get("text") if isinstance(text, str) and text.strip(): agent_messages.append(text.strip()) elif event_type == "error": message = event.get("message") if isinstance(message, str) and message.strip(): errors.append(message.strip()) elif event_type == "turn.completed": turn_usage = event.get("usage") if isinstance(turn_usage, dict): usage = turn_usage if not agent_messages and errors: agent_messages.extend(errors) if not agent_messages: raise ParserError("Codex CLI JSONL output did not include an agent_message item") content = "\n\n".join(agent_messages).strip() metadata: dict[str, Any] = {"events": events} if errors: metadata["errors"] = errors if usage: metadata["usage"] = usage if stderr and stderr.strip(): metadata["stderr"] = stderr.strip() return ParsedCLIResponse(content=content, metadata=metadata) ================================================ FILE: clink/parsers/gemini.py ================================================ """Parser for Gemini CLI JSON output.""" from __future__ import annotations import json from typing import Any from .base import BaseParser, ParsedCLIResponse, ParserError class GeminiJSONParser(BaseParser): """Parse stdout produced by `gemini -o json`.""" name = "gemini_json" def parse(self, stdout: str, stderr: str) -> ParsedCLIResponse: if not stdout.strip(): raise ParserError("Gemini CLI returned empty stdout while JSON output was expected") try: payload: dict[str, Any] = json.loads(stdout) except json.JSONDecodeError as exc: # pragma: no cover - defensive logging raise ParserError(f"Failed to decode Gemini CLI JSON output: {exc}") from exc response = payload.get("response") response_text = response.strip() if isinstance(response, str) else "" metadata: dict[str, Any] = {"raw": payload} stats = payload.get("stats") if isinstance(stats, dict): metadata["stats"] = stats models = stats.get("models") if isinstance(models, dict) and models: model_name = next(iter(models.keys())) metadata["model_used"] = model_name model_stats = models.get(model_name) or {} tokens = model_stats.get("tokens") if isinstance(tokens, dict): metadata["token_usage"] = tokens api_stats = model_stats.get("api") if isinstance(api_stats, dict): metadata["latency_ms"] = api_stats.get("totalLatencyMs") if response_text: if stderr and stderr.strip(): metadata["stderr"] = stderr.strip() return ParsedCLIResponse(content=response_text, metadata=metadata) fallback_message, extra_metadata = self._build_fallback_message(payload, stderr) if fallback_message: metadata.update(extra_metadata) if stderr and stderr.strip(): metadata["stderr"] = stderr.strip() return ParsedCLIResponse(content=fallback_message, metadata=metadata) raise ParserError("Gemini CLI response is missing a textual 'response' field") def _build_fallback_message(self, payload: dict[str, Any], stderr: str) -> tuple[str | None, dict[str, Any]]: """Derive a human friendly message when Gemini returns empty content.""" stderr_text = stderr.strip() if stderr else "" stderr_lower = stderr_text.lower() extra_metadata: dict[str, Any] = {"empty_response": True} if "429" in stderr_lower or "rate limit" in stderr_lower: extra_metadata["rate_limit_status"] = 429 message = ( "Gemini request returned no content because the API reported a 429 rate limit. " "Retry after reducing the request size or waiting for quota to replenish." ) return message, extra_metadata stats = payload.get("stats") if isinstance(stats, dict): models = stats.get("models") if isinstance(models, dict) and models: first_model = next(iter(models.values())) if isinstance(first_model, dict): api_stats = first_model.get("api") if isinstance(api_stats, dict): total_errors = api_stats.get("totalErrors") total_requests = api_stats.get("totalRequests") if isinstance(total_errors, int) and total_errors > 0: extra_metadata["api_total_errors"] = total_errors if isinstance(total_requests, int): extra_metadata["api_total_requests"] = total_requests message = ( "Gemini CLI returned no textual output. The API reported " f"{total_errors} error(s); see stderr for details." ) return message, extra_metadata if stderr_text: message = "Gemini CLI returned no textual output. Raw stderr was preserved for troubleshooting." return message, extra_metadata return None, extra_metadata ================================================ FILE: clink/registry.py ================================================ """Configuration registry for clink CLI integrations.""" from __future__ import annotations import json import logging import shlex from collections.abc import Iterable from pathlib import Path from clink.constants import ( CONFIG_DIR, DEFAULT_TIMEOUT_SECONDS, INTERNAL_DEFAULTS, PROJECT_ROOT, USER_CONFIG_DIR, CLIInternalDefaults, ) from clink.models import ( CLIClientConfig, CLIRoleConfig, ResolvedCLIClient, ResolvedCLIRole, ) from utils.env import get_env from utils.file_utils import read_json_file logger = logging.getLogger("clink.registry") CONFIG_ENV_VAR = "CLI_CLIENTS_CONFIG_PATH" class RegistryLoadError(RuntimeError): """Raised when configuration files are invalid or missing critical data.""" class ClinkRegistry: """Loads CLI client definitions and exposes them for schema generation/runtime use.""" def __init__(self) -> None: self._clients: dict[str, ResolvedCLIClient] = {} self._load() def _load(self) -> None: self._clients.clear() for config_path in self._iter_config_files(): try: data = read_json_file(str(config_path)) except json.JSONDecodeError as exc: raise RegistryLoadError(f"Invalid JSON in {config_path}: {exc}") from exc if not data: logger.debug("Skipping empty configuration file: %s", config_path) continue config = CLIClientConfig.model_validate(data) resolved = self._resolve_config(config, source_path=config_path) key = resolved.name.lower() if key in self._clients: logger.info("Overriding CLI configuration for '%s' from %s", resolved.name, config_path) else: logger.debug("Loaded CLI configuration for '%s' from %s", resolved.name, config_path) self._clients[key] = resolved if not self._clients: raise RegistryLoadError( "No CLI clients configured. Ensure conf/cli_clients contains at least one definition or set " f"{CONFIG_ENV_VAR}." ) def reload(self) -> None: """Reload configurations from disk.""" self._load() def list_clients(self) -> list[str]: return sorted(client.name for client in self._clients.values()) def list_roles(self, cli_name: str) -> list[str]: config = self.get_client(cli_name) return sorted(config.roles.keys()) def get_client(self, cli_name: str) -> ResolvedCLIClient: key = cli_name.lower() if key not in self._clients: available = ", ".join(self.list_clients()) raise KeyError(f"CLI '{cli_name}' is not configured. Available clients: {available}") return self._clients[key] # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ def _iter_config_files(self) -> Iterable[Path]: search_paths: list[Path] = [] # 1. Built-in configs search_paths.append(CONFIG_DIR) # 2. CLI_CLIENTS_CONFIG_PATH environment override (file or directory) env_path_raw = get_env(CONFIG_ENV_VAR) if env_path_raw: env_path = Path(env_path_raw).expanduser() search_paths.append(env_path) # 3. User overrides in ~/.pal/cli_clients search_paths.append(USER_CONFIG_DIR) seen: set[Path] = set() for base in search_paths: if not base: continue if base in seen: continue seen.add(base) if base.is_file() and base.suffix.lower() == ".json": yield base continue if base.is_dir(): for path in sorted(base.glob("*.json")): if path.is_file(): yield path else: logger.debug("Configuration path does not exist: %s", base) def _resolve_config(self, raw: CLIClientConfig, *, source_path: Path) -> ResolvedCLIClient: if not raw.name: raise RegistryLoadError(f"CLI configuration at {source_path} is missing a 'name' field") normalized_name = raw.name.strip() internal_defaults = INTERNAL_DEFAULTS.get(normalized_name.lower()) if internal_defaults is None: raise RegistryLoadError(f"CLI '{raw.name}' is not supported by clink") executable = self._resolve_executable(raw, internal_defaults, source_path) internal_args = list(internal_defaults.additional_args) if internal_defaults else [] config_args = list(raw.additional_args) timeout_seconds = raw.timeout_seconds or ( internal_defaults.timeout_seconds if internal_defaults else DEFAULT_TIMEOUT_SECONDS ) parser_name = internal_defaults.parser if not parser_name: raise RegistryLoadError( f"CLI '{raw.name}' must define a parser either in configuration or internal defaults" ) runner_name = internal_defaults.runner if internal_defaults else None env = self._merge_env(raw, internal_defaults) working_dir = self._resolve_optional_path(raw.working_dir, source_path.parent) roles = self._resolve_roles(raw, internal_defaults, source_path) output_to_file = raw.output_to_file return ResolvedCLIClient( name=normalized_name, executable=executable, internal_args=internal_args, config_args=config_args, env=env, timeout_seconds=int(timeout_seconds), parser=parser_name, runner=runner_name, roles=roles, output_to_file=output_to_file, working_dir=working_dir, ) def _resolve_executable( self, raw: CLIClientConfig, internal_defaults: CLIInternalDefaults | None, source_path: Path, ) -> list[str]: command = raw.command if not command: raise RegistryLoadError(f"CLI '{raw.name}' must specify a 'command' in configuration") return shlex.split(command) def _merge_env( self, raw: CLIClientConfig, internal_defaults: CLIInternalDefaults | None, ) -> dict[str, str]: merged: dict[str, str] = {} if internal_defaults and internal_defaults.env: merged.update(internal_defaults.env) merged.update(raw.env) return merged def _resolve_roles( self, raw: CLIClientConfig, internal_defaults: CLIInternalDefaults | None, source_path: Path, ) -> dict[str, ResolvedCLIRole]: roles: dict[str, CLIRoleConfig] = dict(raw.roles) default_role_prompt = internal_defaults.default_role_prompt if internal_defaults else None if "default" not in roles: roles["default"] = CLIRoleConfig(prompt_path=default_role_prompt) elif roles["default"].prompt_path is None and default_role_prompt: roles["default"].prompt_path = default_role_prompt resolved: dict[str, ResolvedCLIRole] = {} for role_name, role_config in roles.items(): prompt_path_str = role_config.prompt_path or default_role_prompt if not prompt_path_str: raise RegistryLoadError(f"Role '{role_name}' for CLI '{raw.name}' must define a prompt_path") prompt_path = self._resolve_prompt_path(prompt_path_str, source_path.parent) resolved[role_name] = ResolvedCLIRole( name=role_name, prompt_path=prompt_path, role_args=list(role_config.role_args), description=role_config.description, ) return resolved def _resolve_prompt_path(self, prompt_path: str, base_dir: Path) -> Path: resolved = self._resolve_path(prompt_path, base_dir) if not resolved.exists(): raise RegistryLoadError(f"Prompt file not found: {resolved}") return resolved def _resolve_optional_path(self, candidate: str | None, base_dir: Path) -> Path | None: if not candidate: return None return self._resolve_path(candidate, base_dir) def _resolve_path(self, candidate: str, base_dir: Path) -> Path: path = Path(candidate) if path.is_absolute(): return path candidate_path = (base_dir / path).resolve() if candidate_path.exists(): return candidate_path project_relative = (PROJECT_ROOT / path).resolve() return project_relative _REGISTRY: ClinkRegistry | None = None def get_registry() -> ClinkRegistry: global _REGISTRY if _REGISTRY is None: _REGISTRY = ClinkRegistry() return _REGISTRY ================================================ FILE: code_quality_checks.ps1 ================================================ <# .SYNOPSIS Code quality checks script for PAL MCP server on Windows. .DESCRIPTION This PowerShell script performs code quality checks for the PAL MCP server project: - Runs static analysis and linting tools on the codebase - Ensures code style compliance and detects potential issues - Can be integrated into CI/CD pipelines or used locally before commits .PARAMETER Help Displays help information for using the script. .PARAMETER Verbose Enables detailed output during code quality checks. .EXAMPLE .\code_quality_checks.ps1 Runs all code quality checks on the project. .\code_quality_checks.ps1 -Verbose Runs code quality checks with detailed output. .NOTES Project Author : BeehiveInnovations Script Author : GiGiDKR (https://github.com/GiGiDKR) Date : 07-05-2025 Version : See project documentation References : https://github.com/BeehiveInnovations/pal-mcp-server #> #Requires -Version 5.1 [CmdletBinding()] param( [switch]$SkipTests, [switch]$SkipLinting, [switch]$VerboseOutput ) # Set error action preference $ErrorActionPreference = "Stop" # Colors for output function Write-ColorText { param( [Parameter(Mandatory)] [string]$Text, [string]$Color = "White" ) Write-Host $Text -ForegroundColor $Color } function Write-Emoji { param( [Parameter(Mandatory)] [string]$Emoji, [Parameter(Mandatory)] [string]$Text, [string]$Color = "White" ) Write-Host "$Emoji " -NoNewline Write-ColorText $Text -Color $Color } Write-Emoji "🔍" "Running Code Quality Checks for PAL MCP Server" -Color Cyan Write-ColorText "=================================================" -Color Cyan # Determine Python command $pythonCmd = $null $pipCmd = $null if (Test-Path ".pal_venv") { if ($IsWindows -or $env:OS -eq "Windows_NT") { if (Test-Path ".pal_venv\Scripts\python.exe") { $pythonCmd = ".pal_venv\Scripts\python.exe" $pipCmd = ".pal_venv\Scripts\pip.exe" } } else { if (Test-Path ".pal_venv/bin/python") { $pythonCmd = ".pal_venv/bin/python" $pipCmd = ".pal_venv/bin/pip" } } if ($pythonCmd) { Write-Emoji "✅" "Using venv" -Color Green } } elseif ($env:VIRTUAL_ENV) { $pythonCmd = "python" $pipCmd = "pip" Write-Emoji "✅" "Using activated virtual environment: $env:VIRTUAL_ENV" -Color Green } else { Write-Emoji "❌" "No virtual environment found!" -Color Red Write-ColorText "Please run: .\run-server.ps1 first to set up the environment" -Color Yellow exit 1 } Write-Host "" # Check and install dev dependencies if needed Write-Emoji "🔍" "Checking development dependencies..." -Color Cyan $devDepsNeeded = $false # List of dev tools to check $devTools = @("ruff", "black", "isort", "pytest") foreach ($tool in $devTools) { $toolFound = $false # Check in venv if ($IsWindows -or $env:OS -eq "Windows_NT") { if (Test-Path ".pal_venv\Scripts\$tool.exe") { $toolFound = $true } } else { if (Test-Path ".pal_venv/bin/$tool") { $toolFound = $true } } # Check in PATH if (!$toolFound) { try { $null = Get-Command $tool -ErrorAction Stop $toolFound = $true } catch { # Tool not found } } if (!$toolFound) { $devDepsNeeded = $true break } } if ($devDepsNeeded) { Write-Emoji "📦" "Installing development dependencies..." -Color Yellow try { & $pipCmd install -q -r requirements-dev.txt if ($LASTEXITCODE -ne 0) { throw "Failed to install dev dependencies" } Write-Emoji "✅" "Development dependencies installed" -Color Green } catch { Write-Emoji "❌" "Failed to install development dependencies" -Color Red Write-ColorText "Error: $_" -Color Red exit 1 } } else { Write-Emoji "✅" "Development dependencies already installed" -Color Green } # Set tool paths if ($IsWindows -or $env:OS -eq "Windows_NT") { $ruffCmd = if (Test-Path ".pal_venv\Scripts\ruff.exe") { ".pal_venv\Scripts\ruff.exe" } else { "ruff" } $blackCmd = if (Test-Path ".pal_venv\Scripts\black.exe") { ".pal_venv\Scripts\black.exe" } else { "black" } $isortCmd = if (Test-Path ".pal_venv\Scripts\isort.exe") { ".pal_venv\Scripts\isort.exe" } else { "isort" } $pytestCmd = if (Test-Path ".pal_venv\Scripts\pytest.exe") { ".pal_venv\Scripts\pytest.exe" } else { "pytest" } } else { $ruffCmd = if (Test-Path ".pal_venv/bin/ruff") { ".pal_venv/bin/ruff" } else { "ruff" } $blackCmd = if (Test-Path ".pal_venv/bin/black") { ".pal_venv/bin/black" } else { "black" } $isortCmd = if (Test-Path ".pal_venv/bin/isort") { ".pal_venv/bin/isort" } else { "isort" } $pytestCmd = if (Test-Path ".pal_venv/bin/pytest") { ".pal_venv/bin/pytest" } else { "pytest" } } Write-Host "" # Step 1: Linting and Formatting if (!$SkipLinting) { Write-Emoji "📋" "Step 1: Running Linting and Formatting Checks" -Color Cyan Write-ColorText "--------------------------------------------------" -Color Cyan try { Write-Emoji "🔧" "Running ruff linting with auto-fix..." -Color Yellow & $ruffCmd check --fix --exclude test_simulation_files --exclude .pal_venv if ($LASTEXITCODE -ne 0) { throw "Ruff linting failed" } Write-Emoji "🎨" "Running black code formatting..." -Color Yellow & $blackCmd . --exclude="test_simulation_files/" --exclude=".pal_venv/" if ($LASTEXITCODE -ne 0) { throw "Black formatting failed" } Write-Emoji "📦" "Running import sorting with isort..." -Color Yellow & $isortCmd . --skip-glob=".pal_venv/*" --skip-glob="test_simulation_files/*" if ($LASTEXITCODE -ne 0) { throw "Import sorting failed" } Write-Emoji "✅" "Verifying all linting passes..." -Color Yellow & $ruffCmd check --exclude test_simulation_files --exclude .pal_venv if ($LASTEXITCODE -ne 0) { throw "Final linting verification failed" } Write-Emoji "✅" "Step 1 Complete: All linting and formatting checks passed!" -Color Green } catch { Write-Emoji "❌" "Step 1 Failed: Linting and formatting checks failed" -Color Red Write-ColorText "Error: $_" -Color Red exit 1 } } else { Write-Emoji "⏭️" "Skipping linting and formatting checks" -Color Yellow } Write-Host "" # Step 2: Unit Tests if (!$SkipTests) { Write-Emoji "🧪" "Step 2: Running Complete Unit Test Suite" -Color Cyan Write-ColorText "---------------------------------------------" -Color Cyan try { Write-Emoji "🏃" "Running unit tests (excluding integration tests)..." -Color Yellow $pytestArgs = @("tests/", "-v", "-x", "-m", "not integration") if ($VerboseOutput) { $pytestArgs += "--verbose" } & $pythonCmd -m pytest @pytestArgs if ($LASTEXITCODE -ne 0) { throw "Unit tests failed" } Write-Emoji "✅" "Step 2 Complete: All unit tests passed!" -Color Green } catch { Write-Emoji "❌" "Step 2 Failed: Unit tests failed" -Color Red Write-ColorText "Error: $_" -Color Red exit 1 } } else { Write-Emoji "⏭️" "Skipping unit tests" -Color Yellow } Write-Host "" # Step 3: Final Summary Write-Emoji "🎉" "All Code Quality Checks Passed!" -Color Green Write-ColorText "==================================" -Color Green if (!$SkipLinting) { Write-Emoji "✅" "Linting (ruff): PASSED" -Color Green Write-Emoji "✅" "Formatting (black): PASSED" -Color Green Write-Emoji "✅" "Import sorting (isort): PASSED" -Color Green } else { Write-Emoji "⏭️" "Linting: SKIPPED" -Color Yellow } if (!$SkipTests) { Write-Emoji "✅" "Unit tests: PASSED" -Color Green } else { Write-Emoji "⏭️" "Unit tests: SKIPPED" -Color Yellow } Write-Host "" Write-Emoji "🚀" "Your code is ready for commit and GitHub Actions!" -Color Green Write-Emoji "💡" "Remember to add simulator tests if you modified tools" -Color Yellow ================================================ FILE: code_quality_checks.sh ================================================ #!/bin/bash # PAL MCP Server - Code Quality Checks # This script runs all required linting and testing checks before committing changes. # ALL checks must pass 100% for CI/CD to succeed. set -e # Exit on any error echo "🔍 Running Code Quality Checks for PAL MCP Server" echo "=================================================" # Determine Python command if [[ -f ".pal_venv/bin/python" ]]; then PYTHON_CMD=".pal_venv/bin/python" PIP_CMD=".pal_venv/bin/pip" echo "✅ Using venv" elif [[ -n "$VIRTUAL_ENV" ]]; then PYTHON_CMD="python" PIP_CMD="pip" echo "✅ Using activated virtual environment: $VIRTUAL_ENV" else echo "❌ No virtual environment found!" echo "Please run: ./run-server.sh first to set up the environment" exit 1 fi echo "" # Check and install dev dependencies if needed echo "🔍 Checking development dependencies..." DEV_DEPS_NEEDED=false # Check each dev dependency for tool in ruff black isort pytest; do # Check if tool exists in venv or in PATH if [[ -f ".pal_venv/bin/$tool" ]] || command -v $tool &> /dev/null; then continue else DEV_DEPS_NEEDED=true break fi done if [ "$DEV_DEPS_NEEDED" = true ]; then echo "📦 Installing development dependencies..." $PIP_CMD install -q -r requirements-dev.txt echo "✅ Development dependencies installed" else echo "✅ Development dependencies already installed" fi # Set tool paths if [[ -f ".pal_venv/bin/ruff" ]]; then RUFF=".pal_venv/bin/ruff" BLACK=".pal_venv/bin/black" ISORT=".pal_venv/bin/isort" PYTEST=".pal_venv/bin/pytest" else RUFF="ruff" BLACK="black" ISORT="isort" PYTEST="pytest" fi echo "" # Step 1: Linting and Formatting echo "📋 Step 1: Running Linting and Formatting Checks" echo "--------------------------------------------------" echo "🔧 Running ruff linting with auto-fix..." $RUFF check --fix --exclude test_simulation_files --exclude .pal_venv echo "🎨 Running black code formatting..." $BLACK . --exclude="test_simulation_files/" --exclude=".pal_venv/" echo "📦 Running import sorting with isort..." $ISORT . --skip-glob=".pal_venv/*" --skip-glob="test_simulation_files/*" echo "✅ Verifying all linting passes..." $RUFF check --exclude test_simulation_files --exclude .pal_venv echo "✅ Step 1 Complete: All linting and formatting checks passed!" echo "" # Step 2: Unit Tests echo "🧪 Step 2: Running Complete Unit Test Suite" echo "---------------------------------------------" echo "🏃 Running unit tests (excluding integration tests)..." $PYTHON_CMD -m pytest tests/ -v -x -m "not integration" echo "✅ Step 2 Complete: All unit tests passed!" echo "" # Step 3: Final Summary echo "🎉 All Code Quality Checks Passed!" echo "==================================" echo "✅ Linting (ruff): PASSED" echo "✅ Formatting (black): PASSED" echo "✅ Import sorting (isort): PASSED" echo "✅ Unit tests: PASSED" echo "" echo "🚀 Your code is ready for commit and GitHub Actions!" echo "💡 Remember to add simulator tests if you modified tools" ================================================ FILE: communication_simulator_test.py ================================================ """ Communication Simulator Test for PAL MCP Server This script provides comprehensive end-to-end testing of the PAL MCP Server by simulating real Claude CLI communications and validating conversation continuity, file handling, deduplication features, and clarification scenarios. Test Flow: 1. Setup standalone server environment 2. Load and run individual test modules 3. Validate system behavior through logs and memory 4. Cleanup and report results Usage: python communication_simulator_test.py [--verbose] [--keep-logs] [--tests TEST_NAME...] [--individual TEST_NAME] [--setup] --tests: Run specific tests only (space-separated) --list-tests: List all available tests --individual: Run a single test individually --setup: Force setup standalone server environment using run-server.sh Available tests: basic_conversation - Basic conversation flow with chat tool content_validation - Content validation and duplicate detection per_tool_deduplication - File deduplication for individual tools cross_tool_continuation - Cross-tool conversation continuation scenarios cross_tool_comprehensive - Comprehensive cross-tool integration testing line_number_validation - Line number handling validation across tools memory_validation - Conversation memory validation model_thinking_config - Model thinking configuration testing o3_model_selection - O3 model selection and routing testing ollama_custom_url - Ollama custom URL configuration testing openrouter_fallback - OpenRouter fallback mechanism testing openrouter_models - OpenRouter models availability testing token_allocation_validation - Token allocation and limits validation testgen_validation - TestGen tool validation with specific test function refactor_validation - Refactor tool validation with codesmells debug_validation - Debug tool validation with actual bugs conversation_chain_validation - Conversation chain continuity validation Quick Test Mode (for time-limited testing): Use --quick to run the essential 6 tests that provide maximum coverage: - cross_tool_continuation (cross-tool conversation memory) - basic_conversation (basic chat functionality) - content_validation (content validation and deduplication) - model_thinking_config (flash/flashlite model testing) - o3_model_selection (o3 model selection testing) - per_tool_deduplication (file deduplication for individual tools) Examples: # Run all tests python communication_simulator_test.py # Run only basic conversation and content validation tests python communication_simulator_test.py --tests basic_conversation content_validation # Run a single test individually (with full standalone setup) python communication_simulator_test.py --individual content_validation # Run quick test mode (essential 6 tests for time-limited testing) python communication_simulator_test.py --quick # Force setup standalone server environment before running tests python communication_simulator_test.py --setup # List available tests python communication_simulator_test.py --list-tests """ import argparse import logging import os import shutil import subprocess import sys import tempfile class CommunicationSimulator: """Simulates real-world Claude CLI communication with MCP Gemini server""" def __init__( self, verbose: bool = False, keep_logs: bool = False, selected_tests: list[str] = None, setup: bool = False, quick_mode: bool = False, ): self.verbose = verbose self.keep_logs = keep_logs self.selected_tests = selected_tests or [] self.setup = setup self.quick_mode = quick_mode self.temp_dir = None self.server_process = None # Configure logging first log_level = logging.DEBUG if verbose else logging.INFO logging.basicConfig(level=log_level, format="%(asctime)s - %(levelname)s - %(message)s") self.logger = logging.getLogger(__name__) self.python_path = self._get_python_path() # Import test registry from simulator_tests import TEST_REGISTRY self.test_registry = TEST_REGISTRY # Define quick mode tests (essential tests for time-limited testing) # Focus on tests that work with current tool configurations self.quick_mode_tests = [ "cross_tool_continuation", # Cross-tool conversation memory "basic_conversation", # Basic chat functionality "content_validation", # Content validation and deduplication "model_thinking_config", # Flash/flashlite model testing "o3_model_selection", # O3 model selection testing "per_tool_deduplication", # File deduplication for individual tools ] # If quick mode is enabled, override selected_tests if self.quick_mode: self.selected_tests = self.quick_mode_tests self.logger.info(f"Quick mode enabled - running {len(self.quick_mode_tests)} essential tests") # Available test methods mapping self.available_tests = { name: self._create_test_runner(test_class) for name, test_class in self.test_registry.items() } # Test result tracking self.test_results = dict.fromkeys(self.test_registry.keys(), False) def _get_python_path(self) -> str: """Get the Python path for the virtual environment""" current_dir = os.getcwd() # Try .venv first (modern convention) venv_python = os.path.join(current_dir, ".venv", "bin", "python") if os.path.exists(venv_python): return venv_python # Try venv as fallback venv_python = os.path.join(current_dir, "venv", "bin", "python") if os.path.exists(venv_python): return venv_python # Try .pal_venv as fallback pal_venv_python = os.path.join(current_dir, ".pal_venv", "bin", "python") if os.path.exists(pal_venv_python): return pal_venv_python # Fallback to system python if venv doesn't exist self.logger.warning("Virtual environment not found, using system python") return "python" def _create_test_runner(self, test_class): """Create a test runner function for a test class""" def run_test(): test_instance = test_class(verbose=self.verbose) result = test_instance.run_test() # Update results test_name = test_instance.test_name self.test_results[test_name] = result return result return run_test def setup_test_environment(self) -> bool: """Setup test environment""" try: self.logger.info("Setting up test environment...") # Create temporary directory for test files self.temp_dir = tempfile.mkdtemp(prefix="mcp_test_") self.logger.debug(f"Created temp directory: {self.temp_dir}") # Only run run-server.sh if setup is requested if self.setup: if not self._run_server_script(): return False # Always verify server environment is available return self._verify_server_environment() except Exception as e: self.logger.error(f"Failed to setup test environment: {e}") return False def _run_server_script(self) -> bool: """Run the run-server.sh script""" try: self.logger.info("Running run-server.sh...") # Check if run-server.sh exists setup_script = "./run-server.sh" if not os.path.exists(setup_script): self.logger.error(f"run-server.sh not found at {setup_script}") return False # Make sure it's executable result = self._run_command(["chmod", "+x", setup_script], capture_output=True) if result.returncode != 0: self.logger.error(f"Failed to make run-server.sh executable: {result.stderr}") return False # Run the setup script result = self._run_command([setup_script], capture_output=True) if result.returncode != 0: self.logger.error(f"run-server.sh failed: {result.stderr}") return False self.logger.info("run-server.sh completed successfully") return True except Exception as e: self.logger.error(f"Failed to run run-server.sh: {e}") return False def _verify_server_environment(self) -> bool: """Verify that server environment is ready""" try: self.logger.info("Verifying standalone server environment...") # Check if server.py exists server_file = "server.py" if not os.path.exists(server_file): self.logger.error(f"Server file not found: {server_file}") self.logger.error("Please ensure you're in the correct directory and server.py exists") return False # Check if virtual environment is available if not os.path.exists(self.python_path): self.logger.error(f"Python executable not found: {self.python_path}") self.logger.error("Please run ./run-server.sh first to set up the environment") return False # Check if required dependencies are available try: result = self._run_command([self.python_path, "-c", "import json; print('OK')"], capture_output=True) if result.returncode != 0: self.logger.error("Python environment validation failed") return False except Exception as e: self.logger.error(f"Python environment check failed: {e}") return False self.logger.info("Standalone server environment is ready") return True except Exception as e: self.logger.error(f"Server environment verification failed: {e}") self.logger.error("Please ensure the server environment is set up correctly, or use --setup") return False def simulate_claude_cli_session(self) -> bool: """Simulate a complete Claude CLI session with conversation continuity""" try: self.logger.info("Starting Claude CLI simulation...") # If specific tests are selected, run only those if self.selected_tests: return self._run_selected_tests() # Otherwise run all tests in order test_sequence = list(self.test_registry.keys()) for test_name in test_sequence: if not self._run_single_test(test_name): return False self.logger.info("All tests passed") return True except Exception as e: self.logger.error(f"Claude CLI simulation failed: {e}") return False def _run_selected_tests(self) -> bool: """Run only the selected tests""" try: self.logger.info(f"Running selected tests: {', '.join(self.selected_tests)}") for test_name in self.selected_tests: if not self._run_single_test(test_name): return False self.logger.info("All selected tests passed") return True except Exception as e: self.logger.error(f"Selected tests failed: {e}") return False def _run_single_test(self, test_name: str) -> bool: """Run a single test by name""" try: if test_name not in self.available_tests: self.logger.error(f"Unknown test: {test_name}") self.logger.info(f"Available tests: {', '.join(self.available_tests.keys())}") return False self.logger.info(f"Running test: {test_name}") test_function = self.available_tests[test_name] result = test_function() if result: self.logger.info(f"Test {test_name} passed") else: self.logger.error(f"Test {test_name} failed") return result except Exception as e: self.logger.error(f"Test {test_name} failed with exception: {e}") return False def run_individual_test(self, test_name: str) -> bool: """Run a single test individually""" try: if test_name not in self.available_tests: self.logger.error(f"Unknown test: {test_name}") self.logger.info(f"Available tests: {', '.join(self.available_tests.keys())}") return False self.logger.info(f"Running individual test: {test_name}") # Setup environment if not self.setup_test_environment(): self.logger.error("Environment setup failed") return False # Run the single test test_function = self.available_tests[test_name] result = test_function() if result: self.logger.info(f"Individual test {test_name} passed") else: self.logger.error(f"Individual test {test_name} failed") return result except Exception as e: self.logger.error(f"Individual test {test_name} failed with exception: {e}") return False finally: if not self.keep_logs: self.cleanup() def get_available_tests(self) -> dict[str, str]: """Get available tests with descriptions""" descriptions = {} for name, test_class in self.test_registry.items(): # Create temporary instance to get description temp_instance = test_class(verbose=False) descriptions[name] = temp_instance.test_description return descriptions def print_test_summary(self): """Print comprehensive test results summary""" self.logger.info("\n" + "=" * 70) self.logger.info("PAL MCP COMMUNICATION SIMULATOR - TEST RESULTS SUMMARY") self.logger.info("=" * 70) passed_count = sum(1 for result in self.test_results.values() if result) total_count = len(self.test_results) for test_name, result in self.test_results.items(): status = "PASS" if result else "FAIL" # Get test description temp_instance = self.test_registry[test_name](verbose=False) description = temp_instance.test_description if result: self.logger.info(f"{description}: {status}") else: self.logger.error(f"{description}: {status}") if passed_count == total_count: self.logger.info("\nOVERALL RESULT: SUCCESS") else: self.logger.error("\nOVERALL RESULT: FAILURE") self.logger.info(f"{passed_count}/{total_count} tests passed") self.logger.info("=" * 70) return passed_count == total_count def run_full_test_suite(self) -> bool: """Run the complete test suite""" try: self.logger.info("Starting PAL MCP Communication Simulator Test Suite") # Setup if not self.setup_test_environment(): self.logger.error("Environment setup failed") return False # Main simulation if not self.simulate_claude_cli_session(): self.logger.error("Claude CLI simulation failed") return False # Print comprehensive summary overall_success = self.print_test_summary() return overall_success except Exception as e: self.logger.error(f"Test suite failed: {e}") return False finally: if not self.keep_logs: self.cleanup() def cleanup(self): """Cleanup test environment""" try: self.logger.info("Cleaning up test environment...") # Stop any running server processes if self.server_process and self.server_process.poll() is None: self.logger.info("Stopping server process...") self.server_process.terminate() try: self.server_process.wait(timeout=5) except subprocess.TimeoutExpired: self.server_process.kill() self.server_process.wait() if not self.keep_logs: self.logger.info("Test completed. Standalone server process stopped.") else: self.logger.info("Keeping logs for inspection") # Remove temp directory if self.temp_dir and os.path.exists(self.temp_dir): shutil.rmtree(self.temp_dir) self.logger.debug(f"Removed temp directory: {self.temp_dir}") except Exception as e: self.logger.error(f"Cleanup failed: {e}") def _run_command(self, cmd: list[str], check: bool = True, capture_output: bool = False, **kwargs): """Run a shell command with logging""" if self.verbose: self.logger.debug(f"Running: {' '.join(cmd)}") return subprocess.run(cmd, check=check, capture_output=capture_output, **kwargs) def parse_arguments(): """Parse and validate command line arguments""" parser = argparse.ArgumentParser(description="PAL MCP Communication Simulator Test") parser.add_argument("--verbose", "-v", action="store_true", help="Enable verbose logging") parser.add_argument("--keep-logs", action="store_true", help="Keep logs for inspection after test completion") parser.add_argument("--tests", "-t", nargs="+", help="Specific tests to run (space-separated)") parser.add_argument("--list-tests", action="store_true", help="List available tests and exit") parser.add_argument("--individual", "-i", help="Run a single test individually") parser.add_argument( "--quick", "-q", action="store_true", help="Run quick test mode (6 essential tests for time-limited testing)" ) parser.add_argument( "--setup", action="store_true", help="Force setup standalone server environment using run-server.sh" ) return parser.parse_args() def list_available_tests(): """List all available tests and exit""" simulator = CommunicationSimulator() # Create a simple logger for this function logger = logging.getLogger("list_tests") logging.basicConfig(level=logging.INFO, format="%(message)s") logger.info("Available tests:") for test_name, description in simulator.get_available_tests().items(): logger.info(f" {test_name:<25} - {description}") def run_individual_test(simulator, test_name): """Run a single test individually""" logger = simulator.logger try: success = simulator.run_individual_test(test_name) if success: logger.info(f"\nINDIVIDUAL TEST {test_name.upper()}: PASSED") return 0 else: logger.error(f"\nINDIVIDUAL TEST {test_name.upper()}: FAILED") return 1 except KeyboardInterrupt: logger.warning(f"\nIndividual test {test_name} interrupted by user") simulator.cleanup() return 130 except Exception as e: logger.error(f"\nIndividual test {test_name} failed with error: {e}") simulator.cleanup() return 1 def run_test_suite(simulator): """Run the full test suite or selected tests""" logger = simulator.logger try: success = simulator.run_full_test_suite() if success: logger.info("\nCOMPREHENSIVE MCP COMMUNICATION TEST: PASSED") return 0 else: logger.error("\nCOMPREHENSIVE MCP COMMUNICATION TEST: FAILED") logger.error("Check detailed results above") return 1 except KeyboardInterrupt: logger.warning("\nTest interrupted by user") simulator.cleanup() return 130 except Exception as e: logger.error(f"\nUnexpected error: {e}") simulator.cleanup() return 1 def main(): """Main entry point""" args = parse_arguments() # Handle list tests request if args.list_tests: list_available_tests() return # Initialize simulator consistently for all use cases simulator = CommunicationSimulator( verbose=args.verbose, keep_logs=args.keep_logs, selected_tests=args.tests, setup=args.setup, quick_mode=args.quick, ) # Determine execution mode and run if args.individual: exit_code = run_individual_test(simulator, args.individual) else: exit_code = run_test_suite(simulator) sys.exit(exit_code) if __name__ == "__main__": main() ================================================ FILE: conf/__init__.py ================================================ """Configuration data for PAL MCP Server.""" ================================================ FILE: conf/azure_models.json ================================================ { "_README": { "description": "Model metadata for Azure OpenAI / Azure AI Foundry-backed provider. The `models` definition can be copied from openrouter_models.json / custom_models.json", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/azure_models.md", "usage": "Models listed here are exposed through Azure AI Foundry. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier e.g., 'gpt-4'", "deployment": "Azure model deployment name", "aliases": "Array of short names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "supports_extended_thinking": "Whether the model supports extended reasoning tokens (currently none do via OpenRouter or custom APIs)", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined (capped at 40MB max for custom models)", "supports_temperature": "Whether the model accepts temperature parameter in API calls (set to false for O3/O4 reasoning models)", "temperature_constraint": "Type of temperature constraint: 'fixed' (fixed value), 'range' (continuous range), 'discrete' (specific values), or omit for default range", "use_openai_response_api": "Set to true when the deployment must call Azure's /responses endpoint (O-series reasoning models). Leave false/omit for standard chat completions.", "default_reasoning_effort": "Default reasoning effort level for models that support it (e.g., 'low', 'medium', 'high'). Omit if not applicable.", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode model ordering" } }, "_example_models": [ { "model_name": "gpt-4", "deployment": "gpt-4", "aliases": [ "gpt4" ], "context_window": 128000, "max_output_tokens": 16384, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "supports_temperature": false, "temperature_constraint": "fixed", "use_openai_response_api": false, "description": "GPT-4 (128K context, 16K output)", "intelligence_score": 10 } ], "models": [] } ================================================ FILE: conf/cli_clients/claude.json ================================================ { "name": "claude", "command": "claude", "additional_args": [ "--permission-mode", "acceptEdits", "--model", "sonnet" ], "env": {}, "roles": { "default": { "prompt_path": "systemprompts/clink/default.txt", "role_args": [] }, "planner": { "prompt_path": "systemprompts/clink/default_planner.txt", "role_args": [] }, "codereviewer": { "prompt_path": "systemprompts/clink/default_codereviewer.txt", "role_args": [] } } } ================================================ FILE: conf/cli_clients/codex.json ================================================ { "name": "codex", "command": "codex", "additional_args": [ "--json", "--dangerously-bypass-approvals-and-sandbox", "--enable", "web_search_request" ], "env": {}, "roles": { "default": { "prompt_path": "systemprompts/clink/default.txt", "role_args": [] }, "planner": { "prompt_path": "systemprompts/clink/default_planner.txt", "role_args": [] }, "codereviewer": { "prompt_path": "systemprompts/clink/codex_codereviewer.txt", "role_args": [] } } } ================================================ FILE: conf/cli_clients/gemini.json ================================================ { "name": "gemini", "command": "gemini", "additional_args": [ "--yolo" ], "env": {}, "roles": { "default": { "prompt_path": "systemprompts/clink/default.txt", "role_args": [] }, "planner": { "prompt_path": "systemprompts/clink/default_planner.txt", "role_args": [] }, "codereviewer": { "prompt_path": "systemprompts/clink/default_codereviewer.txt", "role_args": [] } } } ================================================ FILE: conf/custom_models.json ================================================ { "_README": { "description": "Model metadata for local/self-hosted OpenAI-compatible endpoints (Custom provider).", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/custom_models.md", "usage": "Each entry will be advertised by the Custom provider. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier e.g., 'llama3.2'", "aliases": "Array of short names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "supports_extended_thinking": "Whether the model supports extended reasoning tokens", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined (capped at 40MB max for custom models)", "supports_temperature": "Whether the model accepts temperature parameter in API calls (set to false for O3/O4 reasoning models)", "temperature_constraint": "Type of temperature constraint: 'fixed' (fixed value), 'range' (continuous range), 'discrete' (specific values), or omit for default range", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode model ordering" } }, "models": [ { "model_name": "llama3.2", "aliases": [ "local-llama", "ollama-llama" ], "context_window": 128000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "description": "Local Llama 3.2 model via custom endpoint (Ollama/vLLM) - 128K context window (text-only)", "intelligence_score": 6 } ] } ================================================ FILE: conf/dial_models.json ================================================ { "_README": { "description": "Model metadata for the DIAL (Data & AI Layer) aggregation provider.", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/configuration.md", "usage": "Models listed here are exposed through the DIAL provider. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier as exposed by DIAL (typically deployment name)", "aliases": "Array of shorthand names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "supports_extended_thinking": "Whether the model supports extended reasoning tokens", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined", "supports_temperature": "Whether the model accepts the temperature parameter", "temperature_constraint": "Temperature constraint hint: 'fixed', 'range', or 'discrete'", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode ordering" } }, "models": [ { "model_name": "o3-2025-04-16", "friendly_name": "DIAL (O3)", "aliases": ["o3"], "intelligence_score": 14, "description": "OpenAI O3 via DIAL - Strong reasoning model", "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed" }, { "model_name": "o4-mini-2025-04-16", "friendly_name": "DIAL (O4-mini)", "aliases": ["o4-mini"], "intelligence_score": 11, "description": "OpenAI O4-mini via DIAL - Fast reasoning model", "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed" }, { "model_name": "anthropic.claude-sonnet-4.1-20250805-v1:0", "friendly_name": "DIAL (Sonnet 4.1)", "aliases": ["sonnet-4.1", "sonnet-4"], "intelligence_score": 10, "description": "Claude Sonnet 4.1 via DIAL - Balanced performance", "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 5.0, "supports_temperature": true, "temperature_constraint": "range" }, { "model_name": "anthropic.claude-sonnet-4.1-20250805-v1:0-with-thinking", "friendly_name": "DIAL (Sonnet 4.1 Thinking)", "aliases": ["sonnet-4.1-thinking", "sonnet-4-thinking"], "intelligence_score": 11, "description": "Claude Sonnet 4.1 with thinking mode via DIAL", "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": true, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 5.0, "supports_temperature": true, "temperature_constraint": "range" }, { "model_name": "anthropic.claude-opus-4.1-20250805-v1:0", "friendly_name": "DIAL (Opus 4.1)", "aliases": ["opus-4.1", "opus-4"], "intelligence_score": 14, "description": "Claude Opus 4.1 via DIAL - Most capable Claude model", "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 5.0, "supports_temperature": true, "temperature_constraint": "range" }, { "model_name": "anthropic.claude-opus-4.1-20250805-v1:0-with-thinking", "friendly_name": "DIAL (Opus 4.1 Thinking)", "aliases": ["opus-4.1-thinking", "opus-4-thinking"], "intelligence_score": 15, "description": "Claude Opus 4.1 with thinking mode via DIAL", "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": true, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 5.0, "supports_temperature": true, "temperature_constraint": "range" }, { "model_name": "gemini-2.5-pro-preview-03-25-google-search", "friendly_name": "DIAL (Gemini 2.5 Pro Search)", "aliases": ["gemini-2.5-pro-search"], "intelligence_score": 17, "description": "Gemini 2.5 Pro with Google Search via DIAL", "context_window": 1000000, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "range" }, { "model_name": "gemini-2.5-pro-preview-05-06", "friendly_name": "DIAL (Gemini 2.5 Pro)", "aliases": ["gemini-2.5-pro"], "intelligence_score": 18, "description": "Gemini 2.5 Pro via DIAL - Deep reasoning", "context_window": 1000000, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "range" }, { "model_name": "gemini-2.5-flash-preview-05-20", "friendly_name": "DIAL (Gemini Flash 2.5)", "aliases": ["gemini-2.5-flash"], "intelligence_score": 10, "description": "Gemini 2.5 Flash via DIAL - Ultra-fast", "context_window": 1000000, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_function_calling": false, "supports_json_mode": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "range" } ] } ================================================ FILE: conf/gemini_models.json ================================================ { "_README": { "description": "Model metadata for Google's Gemini API access.", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/custom_models.md", "usage": "Models listed here are exposed directly through the Gemini provider. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier (e.g., 'gemini-2.5-pro', 'gemini-2.0-flash')", "aliases": "Array of short names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "max_thinking_tokens": "Maximum reasoning/thinking tokens the model will allocate when extended thinking is requested", "supports_extended_thinking": "Whether the model supports extended reasoning tokens (currently none do via OpenRouter or custom APIs)", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined (capped at 40MB max for custom models)", "supports_temperature": "Whether the model accepts temperature parameter in API calls (set to false for O3/O4 reasoning models)", "temperature_constraint": "Type of temperature constraint: 'fixed' (fixed value), 'range' (continuous range), 'discrete' (specific values), or omit for default range", "use_openai_response_api": "Set to true when the model must use the /responses endpoint (reasoning models like GPT-5.2 Pro). Leave false/omit for standard chat completions.", "default_reasoning_effort": "Default reasoning effort level for models that support it (e.g., 'low', 'medium', 'high'). Omit if not applicable.", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode model ordering", "allow_code_generation": "Whether this model can generate and suggest fully working code - complete with functions, files, and detailed implementation instructions - for your AI tool to use right away. Only set this to 'true' for a model more capable than the AI model / CLI you're currently using." } }, "models": [ { "model_name": "gemini-3-pro-preview", "friendly_name": "Gemini Pro 3.0 Preview", "aliases": [ "pro", "gemini3", "gemini-pro" ], "intelligence_score": 18, "description": "Deep reasoning + thinking mode (1M context) - Complex problems, architecture, deep analysis", "context_window": 1048576, "max_output_tokens": 65536, "max_thinking_tokens": 32768, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "allow_code_generation": true, "max_image_size_mb": 32.0 }, { "model_name": "gemini-2.5-pro", "friendly_name": "Gemini Pro 2.5", "aliases": [ "gemini-pro-2.5" ], "intelligence_score": 18, "description": "Older Model. 1M context - Complex problems, architecture, deep analysis", "context_window": 1048576, "max_output_tokens": 65536, "max_thinking_tokens": 32768, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "allow_code_generation": true, "max_image_size_mb": 32.0 }, { "model_name": "gemini-2.0-flash", "friendly_name": "Gemini (Flash 2.0)", "aliases": [ "flash-2.0", "flash2" ], "intelligence_score": 9, "description": "Gemini 2.0 Flash (1M context) - Latest fast model with experimental thinking, supports audio/video input", "context_window": 1048576, "max_output_tokens": 65536, "max_thinking_tokens": 24576, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0 }, { "model_name": "gemini-2.0-flash-lite", "friendly_name": "Gemini (Flash Lite 2.0)", "aliases": [ "flashlite", "flash-lite" ], "intelligence_score": 7, "description": "Gemini 2.0 Flash Lite (1M context) - Lightweight fast model, text-only", "context_window": 1048576, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": false, "supports_temperature": true }, { "model_name": "gemini-2.5-flash", "friendly_name": "Gemini (Flash 2.5)", "aliases": [ "flash", "flash2.5" ], "intelligence_score": 10, "description": "Ultra-fast (1M context) - Quick analysis, simple queries, rapid iterations", "context_window": 1048576, "max_output_tokens": 65536, "max_thinking_tokens": 24576, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0 } ] } ================================================ FILE: conf/openai_models.json ================================================ { "_README": { "description": "Model metadata for native OpenAI API access.", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/custom_models.md", "usage": "Models listed here are exposed directly through the OpenAI provider. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier (e.g., 'gpt-5', 'o3-pro')", "aliases": "Array of short names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "max_thinking_tokens": "Maximum reasoning/thinking tokens the model will allocate when extended thinking is requested", "supports_extended_thinking": "Whether the model supports extended reasoning tokens (currently none do via OpenRouter or custom APIs)", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined (capped at 40MB max for custom models)", "supports_temperature": "Whether the model accepts temperature parameter in API calls (set to false for O3/O4 reasoning models)", "temperature_constraint": "Type of temperature constraint: 'fixed' (fixed value), 'range' (continuous range), 'discrete' (specific values), or omit for default range", "use_openai_response_api": "Set to true when the model must use the /responses endpoint (reasoning models like GPT-5.2 Pro). Leave false/omit for standard chat completions.", "default_reasoning_effort": "Default reasoning effort level for models that support it (e.g., 'low', 'medium', 'high'). Omit if not applicable.", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode model ordering", "allow_code_generation": "Whether this model can generate and suggest fully working code - complete with functions, files, and detailed implementation instructions - for your AI tool to use right away. Only set this to 'true' for a model more capable than the AI model / CLI you're currently using." } }, "models": [ { "model_name": "gpt-5", "friendly_name": "OpenAI (GPT-5)", "aliases": [ "gpt5", "gpt-5" ], "intelligence_score": 16, "description": "GPT-5 (400K context, 128K output) - Advanced model with reasoning support", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": false, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "temperature_constraint": "fixed" }, { "model_name": "gpt-5.2-pro", "friendly_name": "OpenAI (GPT-5.2 Pro)", "aliases": [ "gpt5.2-pro", "gpt5.2pro", "gpt5pro", "gpt5-pro" ], "intelligence_score": 18, "description": "GPT-5.2 Pro (400K context, 272K output) - Very advanced, reasoning model", "context_window": 400000, "max_output_tokens": 272000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": false, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "use_openai_response_api": true, "default_reasoning_effort": "high", "allow_code_generation": true, "temperature_constraint": "fixed" }, { "model_name": "gpt-5-mini", "friendly_name": "OpenAI (GPT-5-mini)", "aliases": [ "gpt5-mini", "gpt5mini", "mini" ], "intelligence_score": 15, "description": "GPT-5-mini (400K context, 128K output) - Efficient variant with reasoning support", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": false, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "temperature_constraint": "fixed" }, { "model_name": "gpt-5-nano", "friendly_name": "OpenAI (GPT-5 nano)", "aliases": [ "gpt5nano", "gpt5-nano", "nano" ], "intelligence_score": 13, "description": "GPT-5 nano (400K context) - Fastest, cheapest version of GPT-5 for summarization and classification tasks", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "temperature_constraint": "fixed" }, { "model_name": "o3", "friendly_name": "OpenAI (O3)", "intelligence_score": 14, "description": "Strong reasoning (200K context) - Logical problems, code generation, systematic analysis", "context_window": 200000, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": false, "max_image_size_mb": 20.0, "temperature_constraint": "fixed" }, { "model_name": "o3-mini", "friendly_name": "OpenAI (O3-mini)", "aliases": [ "o3mini" ], "intelligence_score": 12, "description": "Fast O3 variant (200K context) - Balanced performance/speed, moderate complexity", "context_window": 200000, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": false, "max_image_size_mb": 20.0, "temperature_constraint": "fixed" }, { "model_name": "o3-pro", "friendly_name": "OpenAI (O3-Pro)", "aliases": [ "o3pro" ], "intelligence_score": 15, "description": "Professional-grade reasoning with advanced capabilities (200K context)", "context_window": 200000, "max_output_tokens": 65536, "supports_extended_thinking": false, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": false, "max_image_size_mb": 20.0, "use_openai_response_api": true, "temperature_constraint": "fixed" }, { "model_name": "o4-mini", "friendly_name": "OpenAI (O4-mini)", "aliases": [ "o4mini" ], "intelligence_score": 11, "description": "Latest reasoning model (200K context) - Optimized for shorter contexts, rapid reasoning", "context_window": 200000, "supports_extended_thinking": false, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": false, "max_image_size_mb": 20.0, "temperature_constraint": "fixed" }, { "model_name": "gpt-4.1", "friendly_name": "OpenAI (GPT 4.1)", "aliases": [ "gpt4.1" ], "intelligence_score": 13, "description": "GPT-4.1 (1M context) - Advanced reasoning model with large context window", "context_window": 1000000, "max_output_tokens": 32768, "supports_extended_thinking": false, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0 }, { "model_name": "gpt-5-codex", "friendly_name": "OpenAI (GPT-5 Codex)", "aliases": [ "gpt5-codex", "codex", "gpt-5-code", "gpt5-code" ], "intelligence_score": 17, "description": "GPT-5 Codex (400K context) Specialized for coding, refactoring, and software architecture.", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "use_openai_response_api": true }, { "model_name": "gpt-5.2", "friendly_name": "OpenAI (GPT-5.2)", "aliases": [ "gpt5.2", "gpt-5.2", "5.2", "gpt5.1", "gpt-5.1", "5.1" ], "intelligence_score": 18, "description": "GPT-5.2 (400K context, 128K output) - Flagship reasoning model with configurable thinking effort and vision support.", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "default_reasoning_effort": "medium", "allow_code_generation": true, "temperature_constraint": "fixed" }, { "model_name": "gpt-5.1-codex", "friendly_name": "OpenAI (GPT-5.1 Codex)", "aliases": [ "gpt5.1-codex", "gpt-5.1-codex", "gpt5.1code", "gpt-5.1-code", "codex-5.1" ], "intelligence_score": 19, "description": "GPT-5.1 Codex (400K context, 128K output) - Agentic coding specialization available through the Responses API.", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": false, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "use_openai_response_api": true, "default_reasoning_effort": "high", "allow_code_generation": true, "temperature_constraint": "fixed" }, { "model_name": "gpt-5.1-codex-mini", "friendly_name": "OpenAI (GPT-5.1 Codex mini)", "aliases": [ "gpt5.1-codex-mini", "gpt-5.1-codex-mini", "codex-mini", "5.1-codex-mini" ], "intelligence_score": 16, "description": "GPT-5.1 Codex mini (400K context, 128K output) - Cost-efficient Codex variant with streaming support.", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0, "allow_code_generation": true, "temperature_constraint": "fixed" } ] } ================================================ FILE: conf/openrouter_models.json ================================================ { "_README": { "description": "Model metadata for OpenRouter-backed providers.", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/custom_models.md", "usage": "Models listed here are exposed through OpenRouter. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier - OpenRouter format (e.g., 'anthropic/claude-opus-4') or custom model name (e.g., 'llama3.2')", "aliases": "Array of short names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "supports_extended_thinking": "Whether the model supports extended reasoning tokens (currently none do via OpenRouter or custom APIs)", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined (capped at 40MB max for custom models)", "supports_temperature": "Whether the model accepts temperature parameter in API calls (set to false for O3/O4 reasoning models)", "temperature_constraint": "Type of temperature constraint: 'fixed' (fixed value), 'range' (continuous range), 'discrete' (specific values), or omit for default range", "use_openai_response_api": "Set to true when the model must use the /responses endpoint (reasoning models like GPT-5.2 Pro). Leave false/omit for standard chat completions.", "default_reasoning_effort": "Default reasoning effort level for models that support it (e.g., 'low', 'medium', 'high'). Omit if not applicable.", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode model ordering", "allow_code_generation": "Whether this model can generate and suggest fully working code - complete with functions, files, and detailed implementation instructions - for your AI tool to use right away. Only set this to 'true' for a model more capable than the AI model / CLI you're currently using." } }, "models": [ { "model_name": "anthropic/claude-opus-4.5", "aliases": [ "opus", "opus4.5", "claude-opus" ], "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": true, "max_image_size_mb": 5.0, "description": "Claude Opus 4.5 - Anthropic's frontier reasoning model for complex software engineering and agentic workflows", "intelligence_score": 18 }, { "model_name": "anthropic/claude-sonnet-4.5", "aliases": [ "sonnet", "sonnet4.5" ], "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": true, "max_image_size_mb": 5.0, "description": "Claude Sonnet 4.5 - High-performance model with exceptional reasoning and efficiency", "intelligence_score": 12 }, { "model_name": "anthropic/claude-opus-4.1", "aliases": [ "opus4.1" ], "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": true, "max_image_size_mb": 5.0, "description": "Claude Opus 4.1 - Last generation flagship model with strong coding and reasoning", "intelligence_score": 14 }, { "model_name": "anthropic/claude-sonnet-4.1", "aliases": [ "sonnet4.1" ], "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": true, "max_image_size_mb": 5.0, "description": "Claude Sonnet 4.1 - Last generation high-performance model with exceptional reasoning and efficiency", "intelligence_score": 10 }, { "model_name": "anthropic/claude-3.5-haiku", "aliases": [ "haiku" ], "context_window": 200000, "max_output_tokens": 64000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": true, "max_image_size_mb": 5.0, "description": "Claude 3 Haiku - Fast and efficient with vision", "intelligence_score": 8 }, { "model_name": "google/gemini-3-pro-preview", "aliases": [ "pro", "gemini-pro", "gemini", "gemini3", "pro-openrouter" ], "context_window": 1048576, "max_output_tokens": 65536, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "allow_code_generation": true, "description": "Google's Gemini 3.0 Pro via OpenRouter with vision", "intelligence_score": 18 }, { "model_name": "google/gemini-2.5-pro", "aliases": [ "gemini-2.5", "pro-2.5-openrouter" ], "context_window": 1048576, "max_output_tokens": 65536, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "allow_code_generation": true, "description": "Google's Gemini 2.5 Pro via OpenRouter with vision", "intelligence_score": 18 }, { "model_name": "google/gemini-2.5-flash", "aliases": [ "flash", "gemini-flash" ], "context_window": 1048576, "max_output_tokens": 65536, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 15.0, "description": "Google's Gemini 2.5 Flash via OpenRouter with vision", "intelligence_score": 10 }, { "model_name": "mistralai/mistral-large-2411", "aliases": [ "mistral-large", "mistral" ], "context_window": 128000, "max_output_tokens": 32000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "supports_images": false, "max_image_size_mb": 0.0, "description": "Mistral's largest model (text-only)", "intelligence_score": 11 }, { "model_name": "meta-llama/llama-3-70b", "aliases": [ "llama", "llama3", "llama3-70b", "llama-70b", "llama3-openrouter" ], "context_window": 8192, "max_output_tokens": 8192, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "description": "Meta's Llama 3 70B model (text-only)", "intelligence_score": 9 }, { "model_name": "deepseek/deepseek-r1-0528", "aliases": [ "deepseek-r1", "deepseek", "r1", "deepseek-thinking" ], "context_window": 65536, "max_output_tokens": 32768, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "description": "DeepSeek R1 with thinking mode - advanced reasoning capabilities (text-only)", "intelligence_score": 15 }, { "model_name": "perplexity/llama-3-sonar-large-32k-online", "aliases": [ "perplexity", "sonar", "perplexity-online" ], "context_window": 32768, "max_output_tokens": 32768, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "description": "Perplexity's online model with web search (text-only)", "intelligence_score": 9 }, { "model_name": "openai/o3", "aliases": [ "o3" ], "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed", "description": "OpenAI's o3 model - well-rounded and powerful across domains with vision", "intelligence_score": 14 }, { "model_name": "openai/o3-mini", "aliases": [ "o3-mini", "o3mini" ], "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed", "description": "OpenAI's o3-mini model - balanced performance and speed with vision", "intelligence_score": 12 }, { "model_name": "openai/o3-mini-high", "aliases": [ "o3-mini-high", "o3mini-high" ], "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed", "description": "OpenAI's o3-mini with high reasoning effort - optimized for complex problems with vision", "intelligence_score": 13 }, { "model_name": "openai/o3-pro", "aliases": [ "o3pro" ], "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed", "description": "OpenAI's o3-pro model - professional-grade reasoning and analysis with vision", "intelligence_score": 15 }, { "model_name": "openai/o4-mini", "aliases": [ "o4-mini", "o4mini" ], "context_window": 200000, "max_output_tokens": 100000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed", "description": "OpenAI's o4-mini model - optimized for shorter contexts with rapid reasoning and vision", "intelligence_score": 11 }, { "model_name": "openai/gpt-5", "aliases": [ "gpt5" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "range", "description": "GPT-5 (400K context, 128K output) - Advanced model with reasoning support", "intelligence_score": 16 }, { "model_name": "openai/gpt-5.2-pro", "aliases": [ "gpt5.2-pro", "gpt5.2pro", "gpt5pro" ], "context_window": 400000, "max_output_tokens": 272000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": false, "temperature_constraint": "fixed", "use_openai_response_api": true, "default_reasoning_effort": "high", "allow_code_generation": true, "description": "GPT-5.2 Pro - Advanced reasoning model with highest quality responses (text+image input, text output only)", "intelligence_score": 18 }, { "model_name": "openai/gpt-5-codex", "aliases": [ "codex", "gpt5codex" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "description": "GPT-5-Codex is a specialized version of GPT-5 optimized for software engineering and coding workflows", "intelligence_score": 17 }, { "model_name": "openai/gpt-5-mini", "aliases": [ "gpt5mini" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "supports_temperature": true, "temperature_constraint": "fixed", "description": "GPT-5-mini (400K context, 128K output) - Efficient variant with reasoning support", "intelligence_score": 10 }, { "model_name": "openai/gpt-5-nano", "aliases": [ "gpt5nano" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": false, "supports_images": false, "max_image_size_mb": 0.0, "supports_temperature": true, "temperature_constraint": "fixed", "description": "GPT-5 nano (400K context, 128K output) - Fastest, cheapest version of GPT-5 for summarization and classification tasks", "intelligence_score": 8 }, { "model_name": "openai/gpt-5.2", "aliases": [ "gpt5.2", "gpt-5.2", "5.2", "gpt5.1", "gpt-5.1", "5.1" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "fixed", "default_reasoning_effort": "medium", "allow_code_generation": true, "description": "GPT-5.2 (400K context, 128K output) - Flagship reasoning model with configurable thinking effort and vision support", "intelligence_score": 18 }, { "model_name": "openai/gpt-5.1-codex", "aliases": [ "gpt5.1-codex", "gpt-5.1-codex", "gpt5.1code", "gpt-5.1-code", "codex-5.1" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "fixed", "use_openai_response_api": true, "default_reasoning_effort": "high", "allow_code_generation": true, "description": "GPT-5.1 Codex (400K context, 128K output) - Agentic coding specialization available through the Responses API", "intelligence_score": 19 }, { "model_name": "openai/gpt-5.1-codex-mini", "aliases": [ "gpt5.1-codex-mini", "gpt-5.1-codex-mini", "codex-mini", "5.1-codex-mini" ], "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "fixed", "allow_code_generation": true, "description": "GPT-5.1 Codex mini (400K context, 128K output) - Cost-efficient Codex variant with streaming support", "intelligence_score": 16 }, { "model_name": "x-ai/grok-4", "aliases": [ "grok-4", "grok4", "grok" ], "context_window": 256000, "max_output_tokens": 256000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "range", "description": "xAI's Grok 4 via OpenRouter with vision and advanced reasoning", "intelligence_score": 15 }, { "model_name": "x-ai/grok-4.1-fast", "aliases": [ "grok-4.1-fast-openrouter", "grok-4.1-openrouter" ], "context_window": 2000000, "max_output_tokens": 2000000, "supports_extended_thinking": true, "supports_json_mode": true, "supports_function_calling": true, "supports_images": true, "max_image_size_mb": 20.0, "supports_temperature": true, "temperature_constraint": "range", "description": "xAI's Grok 4.1 Fast Reasoning via OpenRouter (2M context) with vision and advanced reasoning", "intelligence_score": 15 } ] } ================================================ FILE: conf/xai_models.json ================================================ { "_README": { "description": "Model metadata for X.AI (GROK) API access.", "documentation": "https://github.com/BeehiveInnovations/pal-mcp-server/blob/main/docs/custom_models.md", "usage": "Models listed here are exposed directly through the X.AI provider. Aliases are case-insensitive.", "field_notes": "Matches providers/shared/model_capabilities.py.", "field_descriptions": { "model_name": "The model identifier (e.g., 'grok-4', 'grok-4.1-fast')", "aliases": "Array of short names users can type instead of the full model name", "context_window": "Total number of tokens the model can process (input + output combined)", "max_output_tokens": "Maximum number of tokens the model can generate in a single response", "max_thinking_tokens": "Maximum reasoning/thinking tokens the model will allocate when extended thinking is requested", "supports_extended_thinking": "Whether the model supports extended reasoning tokens (currently none do via OpenRouter or custom APIs)", "supports_json_mode": "Whether the model can guarantee valid JSON output", "supports_function_calling": "Whether the model supports function/tool calling", "supports_images": "Whether the model can process images/visual input", "max_image_size_mb": "Maximum total size in MB for all images combined (capped at 40MB max for custom models)", "supports_temperature": "Whether the model accepts temperature parameter in API calls (set to false for O3/O4 reasoning models)", "temperature_constraint": "Type of temperature constraint: 'fixed' (fixed value), 'range' (continuous range), 'discrete' (specific values), or omit for default range", "use_openai_response_api": "Set to true when the model must use the /responses endpoint (reasoning models like GPT-5.2 Pro). Leave false/omit for standard chat completions.", "default_reasoning_effort": "Default reasoning effort level for models that support it (e.g., 'low', 'medium', 'high'). Omit if not applicable.", "description": "Human-readable description of the model", "intelligence_score": "1-20 human rating used as the primary signal for auto-mode model ordering" } }, "models": [ { "model_name": "grok-4", "friendly_name": "X.AI (Grok 4)", "aliases": [ "grok", "grok4", "grok-4" ], "intelligence_score": 16, "description": "GROK-4 (256K context) - Frontier multimodal reasoning model with advanced capabilities", "context_window": 256000, "max_output_tokens": 256000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0 }, { "model_name": "grok-4-1-fast-reasoning", "friendly_name": "X.AI (Grok 4.1 Fast Reasoning)", "aliases": [ "grok-4.1", "grok-4-1", "grok-4.1-fast-reasoning", "grok-4.1-fast-reasoning-latest", "grok-4.1-fast" ], "intelligence_score": 15, "description": "GROK-4.1 Fast Reasoning (2M context) - High-performance multimodal reasoning model with function calling", "context_window": 2000000, "max_output_tokens": 2000000, "supports_extended_thinking": true, "supports_system_prompts": true, "supports_streaming": true, "supports_function_calling": true, "supports_json_mode": true, "supports_images": true, "supports_temperature": true, "max_image_size_mb": 20.0 } ] } ================================================ FILE: config.py ================================================ """ Configuration and constants for PAL MCP Server This module centralizes all configuration settings for the PAL MCP Server. It defines model configurations, token limits, temperature defaults, and other constants used throughout the application. Configuration values can be overridden by environment variables where appropriate. """ from utils.env import get_env # Version and metadata # These values are used in server responses and for tracking releases # IMPORTANT: This is the single source of truth for version and author info # Semantic versioning: MAJOR.MINOR.PATCH __version__ = "9.8.2" # Last update date in ISO format __updated__ = "2025-12-15" # Primary maintainer __author__ = "Fahad Gilani" # Model configuration # DEFAULT_MODEL: The default model used for all AI operations # This should be a stable, high-performance model suitable for code analysis # Can be overridden by setting DEFAULT_MODEL environment variable # Special value "auto" means Claude should pick the best model for each task DEFAULT_MODEL = get_env("DEFAULT_MODEL", "auto") or "auto" # Auto mode detection - when DEFAULT_MODEL is "auto", Claude picks the model IS_AUTO_MODE = DEFAULT_MODEL.lower() == "auto" # Each provider (gemini.py, openai.py, xai.py, dial.py, openrouter.py, custom.py, azure_openai.py) # defines its own MODEL_CAPABILITIES # with detailed descriptions. Tools use ModelProviderRegistry.get_available_model_names() # to get models only from enabled providers (those with valid API keys). # # This architecture ensures: # - No namespace collisions (models only appear when their provider is enabled) # - API key-based filtering (prevents wrong models from being shown to Claude) # - Proper provider routing (models route to the correct API endpoint) # - Clean separation of concerns (providers own their model definitions) # Temperature defaults for different tool types # NOTE: Gemini 3.0 Pro notes suggest temperature should be set at 1.0 # in most cases. Lowering it can affect the models 'reasoning' abilities. # Newer models / inference stacks are able to handle their randomness better. # Temperature controls the randomness/creativity of model responses # Lower values (0.0-0.3) produce more deterministic, focused responses # Higher values (0.7-1.0) produce more creative, varied responses # TEMPERATURE_ANALYTICAL: Used for tasks requiring precision and consistency # Ideal for code review, debugging, and error analysis where accuracy is critical TEMPERATURE_ANALYTICAL = 1.0 # For code review, debugging # TEMPERATURE_BALANCED: Middle ground for general conversations # Provides a good balance between consistency and helpful variety TEMPERATURE_BALANCED = 1.0 # For general chat # TEMPERATURE_CREATIVE: Higher temperature for exploratory tasks # Used when brainstorming, exploring alternatives, or architectural discussions TEMPERATURE_CREATIVE = 1.0 # For architecture, deep thinking # Thinking Mode Defaults # DEFAULT_THINKING_MODE_THINKDEEP: Default thinking depth for extended reasoning tool # Higher modes use more computational budget but provide deeper analysis DEFAULT_THINKING_MODE_THINKDEEP = get_env("DEFAULT_THINKING_MODE_THINKDEEP", "high") or "high" # Consensus Tool Defaults # Consensus timeout and rate limiting settings DEFAULT_CONSENSUS_TIMEOUT = 120.0 # 2 minutes per model DEFAULT_CONSENSUS_MAX_INSTANCES_PER_COMBINATION = 2 # NOTE: Consensus tool now uses sequential processing for MCP compatibility # Concurrent processing was removed to avoid async pattern violations # MCP Protocol Transport Limits # # IMPORTANT: This limit ONLY applies to the Claude CLI ↔ MCP Server transport boundary. # It does NOT limit internal MCP Server operations like system prompts, file embeddings, # conversation history, or content sent to external models (Gemini/OpenAI/OpenRouter). # # MCP Protocol Architecture: # Claude CLI ←→ MCP Server ←→ External Model (Gemini/OpenAI/etc.) # ↑ ↑ # │ │ # MCP transport Internal processing # (token limit from MAX_MCP_OUTPUT_TOKENS) (No MCP limit - can be 1M+ tokens) # # MCP_PROMPT_SIZE_LIMIT: Maximum character size for USER INPUT crossing MCP transport # The MCP protocol has a combined request+response limit controlled by MAX_MCP_OUTPUT_TOKENS. # To ensure adequate space for MCP Server → Claude CLI responses, we limit user input # to roughly 60% of the total token budget converted to characters. Larger user prompts # must be sent as prompt.txt files to bypass MCP's transport constraints. # # Token to character conversion ratio: ~4 characters per token (average for code/text) # Default allocation: 60% of tokens for input, 40% for response # # What IS limited by this constant: # - request.prompt field content (user input from Claude CLI) # - prompt.txt file content (alternative user input method) # - Any other direct user input fields # # What is NOT limited by this constant: # - System prompts added internally by tools # - File content embedded by tools # - Conversation history loaded from storage # - Web search instructions or other internal additions # - Complete prompts sent to external models (managed by model-specific token limits) # # This ensures MCP transport stays within protocol limits while allowing internal # processing to use full model context windows (200K-1M+ tokens). def _calculate_mcp_prompt_limit() -> int: """ Calculate MCP prompt size limit based on MAX_MCP_OUTPUT_TOKENS environment variable. Returns: Maximum character count for user input prompts """ # Check for Claude's MAX_MCP_OUTPUT_TOKENS environment variable max_tokens_str = get_env("MAX_MCP_OUTPUT_TOKENS") if max_tokens_str: try: max_tokens = int(max_tokens_str) # Allocate 60% of tokens for input, convert to characters (~4 chars per token) input_token_budget = int(max_tokens * 0.6) character_limit = input_token_budget * 4 return character_limit except (ValueError, TypeError): # Fall back to default if MAX_MCP_OUTPUT_TOKENS is not a valid integer pass # Default fallback: 60,000 characters (equivalent to ~15k tokens input of 25k total) return 60_000 MCP_PROMPT_SIZE_LIMIT = _calculate_mcp_prompt_limit() # Language/Locale Configuration # LOCALE: Language/locale specification for AI responses # When set, all AI tools will respond in the specified language while # maintaining their analytical capabilities # Examples: "fr-FR", "en-US", "zh-CN", "zh-TW", "ja-JP", "ko-KR", "es-ES", # "de-DE", "it-IT", "pt-PT" # Leave empty for default language (English) LOCALE = get_env("LOCALE", "") or "" # Threading configuration # Simple in-memory conversation threading for stateless MCP environment # Conversations persist only during the Claude session ================================================ FILE: docker/README.md ================================================ # PAL MCP Server - Docker Setup ## Quick Start ### 1. Prerequisites - Docker installed (Docker Compose optional) - At least one API key (Gemini, OpenAI, xAI, etc.) ### 2. Configuration ```bash # Copy environment template cp .env.example .env # Edit with your API keys (at least one required) # Required: GEMINI_API_KEY or OPENAI_API_KEY or XAI_API_KEY nano .env ``` ### 3. Build Image ```bash # Build the Docker image docker build -t pal-mcp-server:latest . # Or use the build script (Bash) chmod +x docker/scripts/build.sh ./docker/scripts/build.sh # Build with PowerShell docker/scripts/build.ps1 ``` ### 4. Usage Options #### A. Direct Docker Run (Recommended for MCP) ```bash # Run with environment file docker run --rm -i --env-file .env \ -v $(pwd)/logs:/app/logs \ pal-mcp-server:latest # Run with inline environment variables docker run --rm -i \ -e GEMINI_API_KEY="your_key_here" \ -e LOG_LEVEL=INFO \ -v $(pwd)/logs:/app/logs \ pal-mcp-server:latest ``` #### B. Docker Compose (For Development/Monitoring) ```bash # Deploy with Docker Compose chmod +x docker/scripts/deploy.sh ./docker/scripts/deploy.sh # Or use PowerShell script docker/scripts/deploy.ps1 # Interactive stdio mode docker-compose exec pal-mcp python server.py ``` ## Service Management ### Docker Commands ```bash # View running containers docker ps # View logs from container docker logs # Stop all pal-mcp containers docker stop $(docker ps -q --filter "ancestor=pal-mcp-server:latest") # Remove old containers and images docker container prune docker image prune ``` ### Docker Compose Management (Optional) ```bash # View logs docker-compose logs -f pal-mcp # Check status docker-compose ps # Restart service docker-compose restart pal-mcp # Stop services docker-compose down # Rebuild and update docker-compose build --no-cache pal-mcp docker-compose up -d pal-mcp ``` ## Health Monitoring The container includes health checks that verify: - Server process is running - Python modules can be imported - Log directory is writable - API keys are configured ## Volumes and Persistent Data The Docker setup includes persistent volumes to preserve data between container runs: - **`./logs:/app/logs`** - Persistent log storage (local folder mount) - **`pal-mcp-config:/app/conf`** - Configuration persistence (named Docker volume) - **`/etc/localtime:/etc/localtime:ro`** - Host timezone synchronization (read-only) ### How Persistent Volumes Work The `pal-mcp` service (used by `pal-docker-compose` and Docker Compose commands) mounts the named volume `pal-mcp-config` persistently. All data placed in `/app/conf` inside the container is preserved between runs thanks to this Docker volume. In the `docker-compose.yml` file, you will find: ```yaml volumes: - ./logs:/app/logs - pal-mcp-config:/app/conf - /etc/localtime:/etc/localtime:ro ``` and the named volume definition: ```yaml volumes: pal-mcp-config: driver: local ``` ## Security - Runs as non-root user `paluser` - Read-only filesystem with tmpfs for temporary files - No network ports exposed (stdio communication only) - Secrets managed via environment variables ## Troubleshooting ### Container won't start ```bash # Check if image exists docker images pal-mcp-server # Test container interactively docker run --rm -it --env-file .env pal-mcp-server:latest bash # Check environment variables docker run --rm --env-file .env pal-mcp-server:latest env | grep API # Test with minimal configuration docker run --rm -i -e GEMINI_API_KEY="test" pal-mcp-server:latest python server.py ``` ### MCP Connection Issues ```bash # Test Docker connectivity docker run --rm hello-world # Verify container stdio echo '{"jsonrpc": "2.0", "method": "ping"}' | docker run --rm -i --env-file .env pal-mcp-server:latest python server.py # Check Claude Desktop logs for connection errors ``` ### API Key Problems ```bash # Verify API keys are loaded docker run --rm --env-file .env pal-mcp-server:latest python -c "import os; print('GEMINI_API_KEY:', bool(os.getenv('GEMINI_API_KEY')))" # Test API connectivity docker run --rm --env-file .env pal-mcp-server:latest python /usr/local/bin/healthcheck.py ``` ### Permission Issues ```bash # Fix log directory permissions (Linux/macOS) sudo chown -R $USER:$USER logs/ chmod 755 logs/ # Windows: Run Docker Desktop as Administrator if needed ``` ### Memory/Performance Issues ```bash # Check container resource usage docker stats # Run with memory limits docker run --rm -i --memory="512m" --env-file .env pal-mcp-server:latest # Monitor Docker logs docker run --rm -i --env-file .env pal-mcp-server:latest 2>&1 | tee docker.log ``` ## MCP Integration (Claude Desktop) ### Recommended Configuration (docker run) ```json { "servers": { "pal-docker": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "/absolute/path/to/pal-mcp-server/.env", "-v", "/absolute/path/to/pal-mcp-server/logs:/app/logs", "pal-mcp-server:latest" ] } } } ``` ### Windows Example ```json { "servers": { "pal-docker": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "C:/Users/YourName/path/to/pal-mcp-server/.env", "-v", "C:/Users/YourName/path/to/pal-mcp-server/logs:/app/logs", "pal-mcp-server:latest" ] } } } ``` ### Advanced Option: docker-compose run (uses compose configuration) ```json { "servers": { "pal-docker": { "command": "docker-compose", "args": [ "-f", "/absolute/path/to/pal-mcp-server/docker-compose.yml", "run", "--rm", "pal-mcp" ] } } } ``` ### Environment File Template Create a `.env` file with at least one API key: ```bash # Required: At least one API key GEMINI_API_KEY=your_gemini_key_here OPENAI_API_KEY=your_openai_key_here # Optional configuration LOG_LEVEL=INFO DEFAULT_MODEL=auto DEFAULT_THINKING_MODE_THINKDEEP=high # Optional API keys (leave empty if not used) ANTHROPIC_API_KEY= XAI_API_KEY= DIAL_API_KEY= OPENROUTER_API_KEY= CUSTOM_API_URL= ``` ## Quick Test & Validation ### 1. Test Docker Image ```bash # Test container starts correctly docker run --rm pal-mcp-server:latest python --version # Test health check docker run --rm -e GEMINI_API_KEY="test" pal-mcp-server:latest python /usr/local/bin/healthcheck.py ``` ### 2. Test MCP Protocol ```bash # Test basic MCP communication echo '{"jsonrpc": "2.0", "method": "initialize", "params": {}}' | \ docker run --rm -i --env-file .env pal-mcp-server:latest python server.py ``` ### 3. Validate Configuration ```bash # Run validation script python test_mcp_config.py # Or validate JSON manually python -m json.tool .vscode/mcp.json ``` ## Available Tools The PAL MCP Server provides these tools when properly configured: - **chat** - General AI conversation and collaboration - **thinkdeep** - Multi-stage investigation and reasoning - **planner** - Interactive sequential planning - **consensus** - Multi-model consensus workflow - **codereview** - Comprehensive code review - **debug** - Root cause analysis and debugging - **analyze** - Code analysis and assessment - **refactor** - Refactoring analysis and suggestions - **secaudit** - Security audit workflow - **testgen** - Test generation with edge cases - **docgen** - Documentation generation - **tracer** - Code tracing and dependency mapping - **precommit** - Pre-commit validation workflow - **listmodels** - Available AI models information - **version** - Server version and configuration ## Performance Notes - **Image size**: ~293MB optimized multi-stage build - **Memory usage**: ~256MB base + model overhead - **Startup time**: ~2-3 seconds for container initialization - **API response**: Varies by model and complexity (1-30 seconds) For production use, consider: - Using specific API keys for rate limiting - Monitoring container resource usage - Setting up log rotation for persistent logs - Using Docker health checks for reliability ================================================ FILE: docker/scripts/build.ps1 ================================================ #!/usr/bin/env pwsh #Requires -Version 5.1 [CmdletBinding()] param() # Set error action preference $ErrorActionPreference = "Stop" # Colors for output (using Write-Host with colors) function Write-ColorText { param( [Parameter(Mandatory)] [string]$Text, [string]$Color = "White", [switch]$NoNewline ) if ($NoNewline) { Write-Host $Text -ForegroundColor $Color -NoNewline } else { Write-Host $Text -ForegroundColor $Color } } Write-ColorText "=== Building PAL MCP Server Docker Image ===" -Color Green # Check if .env file exists if (!(Test-Path ".env")) { Write-ColorText "Warning: .env file not found. Copying from .env.example" -Color Yellow if (Test-Path ".env.example") { Copy-Item ".env.example" ".env" Write-ColorText "Please edit .env file with your API keys before running the server" -Color Yellow } else { Write-ColorText "Error: .env.example not found" -Color Red exit 1 } } # Build the Docker image Write-ColorText "Building Docker image..." -Color Green try { docker-compose build --no-cache if ($LASTEXITCODE -ne 0) { throw "Docker build failed" } } catch { Write-ColorText "Error: Failed to build Docker image" -Color Red exit 1 } # Verify the build Write-ColorText "Verifying build..." -Color Green $images = docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedAt}}" | Select-String "pal-mcp-server" if ($images) { Write-ColorText "✓ Docker image built successfully" -Color Green Write-ColorText "Image details:" -Color Green $images | ForEach-Object { Write-Host $_.Line } } else { Write-ColorText "✗ Failed to build Docker image" -Color Red exit 1 } Write-ColorText "=== Build Complete ===" -Color Green Write-ColorText "Next steps:" -Color Yellow Write-Host " 1. Edit .env file with your API keys" Write-ColorText " 2. Run: " -Color White -NoNewline Write-ColorText "docker-compose up -d" -Color Green Write-ColorText "Or use the deploy script: " -Color White -NoNewline Write-ColorText ".\deploy.ps1" -Color Green ================================================ FILE: docker/scripts/build.sh ================================================ #!/bin/bash set -euo pipefail # Colors for output GREEN='\033[0;32m' YELLOW='\033[1;33m' RED='\033[0;31m' NC='\033[0m' echo -e "${GREEN}=== Building PAL MCP Server Docker Image ===${NC}" # Check if .env file exists if [[ ! -f .env ]]; then echo -e "${YELLOW}Warning: .env file not found. Copying from .env.example${NC}" if [[ -f .env.example ]]; then cp .env.example .env echo -e "${YELLOW}Please edit .env file with your API keys before running the server${NC}" else echo -e "${RED}Error: .env.example not found${NC}" exit 1 fi fi # Build the Docker image echo -e "${GREEN}Building Docker image...${NC}" docker-compose build --no-cache # Verify the build if docker images | grep -q "pal-mcp-server"; then echo -e "${GREEN}✓ Docker image built successfully${NC}" echo -e "${GREEN}Image details:${NC}" docker images | grep pal-mcp-server else echo -e "${RED}✗ Failed to build Docker image${NC}" exit 1 fi echo -e "${GREEN}=== Build Complete ===${NC}" echo -e "${YELLOW}Next steps:${NC}" echo -e " 1. Edit .env file with your API keys" echo -e " 2. Run: ${GREEN}docker-compose up -d${NC}" ================================================ FILE: docker/scripts/deploy.ps1 ================================================ #!/usr/bin/env pwsh #Requires -Version 5.1 [CmdletBinding()] param( [switch]$SkipHealthCheck, [int]$HealthCheckTimeout = 60 ) # Set error action preference $ErrorActionPreference = "Stop" # Colors for output function Write-ColorText { param( [Parameter(Mandatory)] [string]$Text, [string]$Color = "White", [switch]$NoNewline ) if ($NoNewline) { Write-Host $Text -ForegroundColor $Color -NoNewline } else { Write-Host $Text -ForegroundColor $Color } } Write-ColorText "=== Deploying PAL MCP Server ===" -Color Green # Function to check if required environment variables are set function Test-EnvironmentVariables { # At least one of these API keys must be set $requiredVars = @( "GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "DIAL_API_KEY", "OPENROUTER_API_KEY" ) $hasApiKey = $false foreach ($var in $requiredVars) { $value = [Environment]::GetEnvironmentVariable($var) if (![string]::IsNullOrWhiteSpace($value)) { $hasApiKey = $true break } } if (!$hasApiKey) { Write-ColorText "Error: At least one API key must be set in your .env file" -Color Red Write-ColorText "Required variables (at least one):" -Color Yellow $requiredVars | ForEach-Object { Write-Host " $_" } exit 1 } } # Load environment variables from .env file if (Test-Path ".env") { Write-ColorText "Loading environment variables from .env..." -Color Green # Read .env file and set environment variables Get-Content ".env" | ForEach-Object { if ($_ -match '^([^#][^=]*?)=(.*)$') { $name = $matches[1].Trim() $value = $matches[2].Trim() # Remove quotes if present $value = $value -replace '^["'']|["'']$', '' [Environment]::SetEnvironmentVariable($name, $value, "Process") } } Write-ColorText "✓ Environment variables loaded from .env" -Color Green } else { Write-ColorText "Error: .env file not found" -Color Red Write-ColorText "Please copy .env.example to .env and configure your API keys" -Color Yellow exit 1 } # Check required environment variables Test-EnvironmentVariables # Function to wait for service health with exponential backoff function Wait-ForHealth { param( [int]$MaxAttempts = 6, [int]$InitialDelay = 2 ) $attempt = 1 $delay = $InitialDelay while ($attempt -le $MaxAttempts) { try { # Get container ID for pal-mcp service $containerId = docker-compose ps -q pal-mcp if ([string]::IsNullOrWhiteSpace($containerId)) { $status = "unavailable" } else { $status = docker inspect -f "{{.State.Health.Status}}" $containerId 2>$null if ($LASTEXITCODE -ne 0) { $status = "unavailable" } } if ($status -eq "healthy") { return $true } Write-ColorText "Waiting for service to be healthy... (attempt $attempt/$MaxAttempts, retrying in ${delay}s)" -Color Yellow Start-Sleep -Seconds $delay $delay = $delay * 2 $attempt++ } catch { Write-ColorText "Error checking health status: $_" -Color Red $attempt++ Start-Sleep -Seconds $delay } } Write-ColorText "Service failed to become healthy after $MaxAttempts attempts" -Color Red Write-ColorText "Checking logs:" -Color Yellow docker-compose logs pal-mcp return $false } # Create logs directory if it doesn't exist if (!(Test-Path "logs")) { Write-ColorText "Creating logs directory..." -Color Green New-Item -ItemType Directory -Path "logs" -Force | Out-Null } # Stop existing containers Write-ColorText "Stopping existing containers..." -Color Green try { docker-compose down if ($LASTEXITCODE -ne 0) { Write-ColorText "Warning: Failed to stop existing containers (they may not be running)" -Color Yellow } } catch { Write-ColorText "Warning: Error stopping containers: $_" -Color Yellow } # Start the services Write-ColorText "Starting PAL MCP Server..." -Color Green try { docker-compose up -d if ($LASTEXITCODE -ne 0) { throw "Failed to start services" } } catch { Write-ColorText "Error: Failed to start services" -Color Red Write-ColorText "Checking logs:" -Color Yellow docker-compose logs pal-mcp exit 1 } # Wait for health check (unless skipped) if (!$SkipHealthCheck) { Write-ColorText "Waiting for service to be healthy..." -Color Green # Try simple timeout first, then use exponential backoff if needed $timeout = $HealthCheckTimeout $elapsed = 0 $healthy = $false while ($elapsed -lt $timeout) { try { $containerId = docker-compose ps -q pal-mcp if (![string]::IsNullOrWhiteSpace($containerId)) { $status = docker inspect -f "{{.State.Health.Status}}" $containerId 2>$null if ($status -eq "healthy") { $healthy = $true break } } } catch { # Continue checking } Start-Sleep -Seconds 2 $elapsed += 2 } if (!$healthy) { # Use exponential backoff retry mechanism if (!(Wait-ForHealth)) { Write-ColorText "Service failed to become healthy" -Color Red Write-ColorText "Checking logs:" -Color Yellow docker-compose logs pal-mcp exit 1 } } } Write-ColorText "✓ PAL MCP Server deployed successfully" -Color Green Write-ColorText "Service Status:" -Color Green docker-compose ps Write-ColorText "=== Deployment Complete ===" -Color Green Write-ColorText "Useful commands:" -Color Yellow Write-ColorText " View logs: " -Color White -NoNewline Write-ColorText "docker-compose logs -f pal-mcp" -Color Green Write-ColorText " Stop service: " -Color White -NoNewline Write-ColorText "docker-compose down" -Color Green Write-ColorText " Restart service: " -Color White -NoNewline Write-ColorText "docker-compose restart pal-mcp" -Color Green Write-ColorText " PowerShell logs: " -Color White -NoNewline Write-ColorText "Get-Content logs\mcp_server.log -Wait" -Color Green ================================================ FILE: docker/scripts/deploy.sh ================================================ #!/bin/bash set -euo pipefail # Colors for output GREEN='\033[0;32m' YELLOW='\033[1;33m' RED='\033[0;31m' NC='\033[0m' echo -e "${GREEN}=== Deploying PAL MCP Server ===${NC}" # Function to check if required environment variables are set check_env_vars() { # At least one of these API keys must be set local required_vars=("GEMINI_API_KEY" "GOOGLE_API_KEY" "OPENAI_API_KEY" "XAI_API_KEY" "DIAL_API_KEY" "OPENROUTER_API_KEY") local has_api_key=false for var in "${required_vars[@]}"; do if [[ -n "${!var:-}" ]]; then has_api_key=true break fi done if [[ "$has_api_key" == false ]]; then echo -e "${RED}Error: At least one API key must be set in your .env file${NC}" printf ' %s\n' "${required_vars[@]}" exit 1 fi } # Load environment variables if [[ -f .env ]]; then set -a source .env set +a echo -e "${GREEN}✓ Environment variables loaded from .env${NC}" else echo -e "${RED}Error: .env file not found${NC}" echo -e "${YELLOW}Please copy .env.example to .env and configure your API keys${NC}" exit 1 fi # Check required environment variables check_env_vars # Exponential backoff health check function wait_for_health() { local max_attempts=6 local attempt=1 local delay=2 while (( attempt <= max_attempts )); do status=$(docker-compose ps -q pal-mcp | xargs docker inspect -f "{{.State.Health.Status}}" 2>/dev/null || echo "unavailable") if [[ "$status" == "healthy" ]]; then return 0 fi echo -e "${YELLOW}Waiting for service to be healthy... (attempt $attempt/${max_attempts}, retrying in ${delay}s)${NC}" sleep $delay delay=$(( delay * 2 )) attempt=$(( attempt + 1 )) done echo -e "${RED}Service failed to become healthy after $max_attempts attempts${NC}" echo -e "${YELLOW}Checking logs:${NC}" docker-compose logs pal-mcp exit 1 } # Create logs directory if it doesn't exist mkdir -p logs # Stop existing containers echo -e "${GREEN}Stopping existing containers...${NC}" docker-compose down # Start the services echo -e "${GREEN}Starting PAL MCP Server...${NC}" docker-compose up -d # Wait for health check echo -e "${GREEN}Waiting for service to be healthy...${NC}" timeout 60 bash -c 'while [[ "$(docker-compose ps -q pal-mcp | xargs docker inspect -f "{{.State.Health.Status}}")" != "healthy" ]]; do sleep 2; done' || { wait_for_health echo -e "${RED}Service failed to become healthy${NC}" echo -e "${YELLOW}Checking logs:${NC}" docker-compose logs pal-mcp exit 1 } echo -e "${GREEN}✓ PAL MCP Server deployed successfully${NC}" echo -e "${GREEN}Service Status:${NC}" docker-compose ps echo -e "${GREEN}=== Deployment Complete ===${NC}" echo -e "${YELLOW}Useful commands:${NC}" echo -e " View logs: ${GREEN}docker-compose logs -f pal-mcp${NC}" echo -e " Stop service: ${GREEN}docker-compose down${NC}" echo -e " Restart service: ${GREEN}docker-compose restart pal-mcp${NC}" ================================================ FILE: docker/scripts/healthcheck.py ================================================ #!/usr/bin/env python3 """ Health check script for PAL MCP Server Docker container """ import os import subprocess import sys from pathlib import Path try: from utils.env import get_env except ImportError: # pragma: no cover - resolves module path inside container project_root = Path(__file__).resolve().parents[2] if str(project_root) not in sys.path: sys.path.insert(0, str(project_root)) from utils.env import get_env # type: ignore[import-error] def check_process(): """Check if the main server process is running""" result = subprocess.run(["pgrep", "-f", "server.py"], capture_output=True, text=True, timeout=10) if result.returncode == 0: return True print(f"Process check failed: {result.stderr}", file=sys.stderr) return False def check_python_imports(): """Check if critical Python modules can be imported""" critical_modules = ["mcp", "google.genai", "openai", "pydantic", "dotenv"] for module in critical_modules: try: __import__(module) except ImportError as e: print(f"Critical module {module} cannot be imported: {e}", file=sys.stderr) return False except Exception as e: print(f"Error importing {module}: {e}", file=sys.stderr) return False return True def check_log_directory(): """Check if logs directory is writable""" log_dir = "/app/logs" try: if not os.path.exists(log_dir): print(f"Log directory {log_dir} does not exist", file=sys.stderr) return False test_file = os.path.join(log_dir, ".health_check") with open(test_file, "w") as f: f.write("health_check") os.remove(test_file) return True except Exception as e: print(f"Log directory check failed: {e}", file=sys.stderr) return False def check_environment(): """Check if essential environment variables are present""" # At least one API key should be present api_keys = [ "GEMINI_API_KEY", "GOOGLE_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "DIAL_API_KEY", "OPENROUTER_API_KEY", ] has_api_key = any(get_env(key) for key in api_keys) if not has_api_key: print("No API keys found in environment", file=sys.stderr) return False # Validate API key formats (basic checks) for key in api_keys: value = get_env(key) if value: if len(value.strip()) < 10: print(f"API key {key} appears too short or invalid", file=sys.stderr) return False return True def main(): """Main health check function""" checks = [ ("Process", check_process), ("Python imports", check_python_imports), ("Log directory", check_log_directory), ("Environment", check_environment), ] failed_checks = [] for check_name, check_func in checks: if not check_func(): failed_checks.append(check_name) if failed_checks: print(f"Health check failed: {', '.join(failed_checks)}", file=sys.stderr) sys.exit(1) print("Health check passed") sys.exit(0) if __name__ == "__main__": main() ================================================ FILE: docker-compose.yml ================================================ services: pal-mcp: build: context: . dockerfile: Dockerfile target: runtime image: pal-mcp-server:latest container_name: pal-mcp-server # Container labels for traceability labels: - "com.pal-mcp.service=pal-mcp-server" - "com.pal-mcp.version=1.0.0" - "com.pal-mcp.environment=production" - "com.pal-mcp.description=AI-powered Model Context Protocol server" # Environment variables environment: # Default model configuration - DEFAULT_MODEL=${DEFAULT_MODEL:-auto} # API Keys (use Docker secrets in production) - GEMINI_API_KEY=${GEMINI_API_KEY} - GOOGLE_API_KEY=${GOOGLE_API_KEY} - OPENAI_API_KEY=${OPENAI_API_KEY} - ANTHROPIC_API_KEY=${ANTHROPIC_API_KEY} - XAI_API_KEY=${XAI_API_KEY} - DIAL_API_KEY=${DIAL_API_KEY} - DIAL_API_HOST=${DIAL_API_HOST} - DIAL_API_VERSION=${DIAL_API_VERSION} - OPENROUTER_API_KEY=${OPENROUTER_API_KEY} - CUSTOM_API_URL=${CUSTOM_API_URL} - CUSTOM_API_KEY=${CUSTOM_API_KEY} - CUSTOM_MODEL_NAME=${CUSTOM_MODEL_NAME} # Logging configuration - LOG_LEVEL=${LOG_LEVEL:-INFO} - LOG_MAX_SIZE=${LOG_MAX_SIZE:-10MB} - LOG_BACKUP_COUNT=${LOG_BACKUP_COUNT:-5} # Advanced configuration - DEFAULT_THINKING_MODE_THINKDEEP=${DEFAULT_THINKING_MODE_THINKDEEP:-high} - DISABLED_TOOLS=${DISABLED_TOOLS} - MAX_MCP_OUTPUT_TOKENS=${MAX_MCP_OUTPUT_TOKENS} # Server configuration - PYTHONUNBUFFERED=1 - PYTHONPATH=/app - TZ=${TZ:-UTC} # Volumes for persistent data volumes: - ./logs:/app/logs - pal-mcp-config:/app/conf - /etc/localtime:/etc/localtime:ro # Network configuration networks: - pal-network # Resource limits deploy: resources: limits: memory: 512M cpus: '0.5' reservations: memory: 256M cpus: '0.25' # Health check healthcheck: test: ["CMD", "python", "/usr/local/bin/healthcheck.py"] interval: 30s timeout: 10s retries: 3 start_period: 40s # Restart policy restart: unless-stopped # Security security_opt: - no-new-privileges:true read_only: true tmpfs: - /tmp:noexec,nosuid,size=100m - /app/tmp:noexec,nosuid,size=50m # Named volumes volumes: pal-mcp-config: driver: local # Networks networks: pal-network: driver: bridge ipam: config: - subnet: 172.20.0.0/16 ================================================ FILE: docs/adding_providers.md ================================================ # Adding a New Provider This guide explains how to add support for a new AI model provider to the PAL MCP Server. The provider system is designed to be extensible and follows a simple pattern. ## Overview Each provider: - Inherits from `ModelProvider` (base class) or `OpenAICompatibleProvider` (for OpenAI-compatible APIs) - Defines supported models using `ModelCapabilities` objects - Implements the minimal abstract hooks (`get_provider_type()` and `generate_content()`) - Gets wired into `configure_providers()` so environment variables control activation - Can leverage helper subclasses (e.g., `AzureOpenAIProvider`) when only client wiring differs ### Intelligence score cheatsheet Set `intelligence_score` (1–20) when you want deterministic ordering in auto mode or the `listmodels` output. The runtime rank starts from this human score and adds smaller bonuses for context window, extended thinking, and other features ([details here](model_ranking.md)). ## Choose Your Implementation Path **Option A: Full Provider (`ModelProvider`)** - For APIs with unique features or custom authentication - Complete control over API calls and response handling - Populate `MODEL_CAPABILITIES`, implement `generate_content()` and `get_provider_type()`, and only override `get_all_model_capabilities()` / `_lookup_capabilities()` when your catalogue comes from a registry or remote source (override `count_tokens()` only when you have a provider-accurate tokenizer) **Option B: OpenAI-Compatible (`OpenAICompatibleProvider`)** - For APIs that follow OpenAI's chat completion format - Supply `MODEL_CAPABILITIES`, override `get_provider_type()`, and optionally adjust configuration (the base class handles alias resolution, validation, and request wiring) - Inherits all API handling automatically ⚠️ **Important**: If you implement a custom `generate_content()`, call `_resolve_model_name()` before invoking the SDK so aliases (e.g. `"gpt"` → `"gpt-4"`) resolve correctly. The shared implementations already do this for you. **Option C: Azure OpenAI (`AzureOpenAIProvider`)** - For Azure-hosted deployments of OpenAI models - Reuses the OpenAI-compatible pipeline but swaps in the `AzureOpenAI` client and a deployment mapping (canonical model → deployment ID) - Define deployments in [`conf/azure_models.json`](../conf/azure_models.json) (or the file referenced by `AZURE_MODELS_CONFIG_PATH`). - Entries follow the [`ModelCapabilities`](../providers/shared/model_capabilities.py) schema and must include a `deployment` identifier. See [Azure OpenAI Configuration](azure_openai.md) for a step-by-step walkthrough. ## Step-by-Step Guide ### 1. Add Provider Type Add your provider to the `ProviderType` enum in `providers/shared/provider_type.py`: ```python class ProviderType(Enum): GOOGLE = "google" OPENAI = "openai" EXAMPLE = "example" # Add this ``` ### 2. Create the Provider Implementation #### Option A: Full Provider (Native Implementation) Create `providers/example.py`: ```python """Example model provider implementation.""" import logging from typing import Optional from .base import ModelProvider from .shared import ( ModelCapabilities, ModelResponse, ProviderType, RangeTemperatureConstraint, ) logger = logging.getLogger(__name__) class ExampleModelProvider(ModelProvider): """Example model provider implementation.""" MODEL_CAPABILITIES = { "example-large": ModelCapabilities( provider=ProviderType.EXAMPLE, model_name="example-large", friendly_name="Example Large", intelligence_score=18, context_window=100_000, max_output_tokens=50_000, supports_extended_thinking=False, temperature_constraint=RangeTemperatureConstraint(0.0, 2.0, 0.7), description="Large model for complex tasks", aliases=["large", "big"], ), "example-small": ModelCapabilities( provider=ProviderType.EXAMPLE, model_name="example-small", friendly_name="Example Small", intelligence_score=14, context_window=32_000, max_output_tokens=16_000, temperature_constraint=RangeTemperatureConstraint(0.0, 2.0, 0.7), description="Fast model for simple tasks", aliases=["small", "fast"], ), } def __init__(self, api_key: str, **kwargs): super().__init__(api_key, **kwargs) # Initialize your API client here def get_all_model_capabilities(self) -> dict[str, ModelCapabilities]: return dict(self.MODEL_CAPABILITIES) def get_provider_type(self) -> ProviderType: return ProviderType.EXAMPLE def generate_content( self, prompt: str, model_name: str, system_prompt: Optional[str] = None, temperature: float = 0.7, max_output_tokens: Optional[int] = None, **kwargs, ) -> ModelResponse: resolved_name = self._resolve_model_name(model_name) # Your API call logic here # response = your_api_client.generate(...) return ModelResponse( content="Generated response", usage={"input_tokens": 100, "output_tokens": 50, "total_tokens": 150}, model_name=resolved_name, friendly_name="Example", provider=ProviderType.EXAMPLE, ) ``` `ModelProvider.get_capabilities()` automatically resolves aliases, enforces the shared restriction service, and returns the correct `ModelCapabilities` instance. Override `_lookup_capabilities()` only when you source capabilities from a registry or remote API. `ModelProvider.count_tokens()` uses a simple 4-characters-per-token estimate so providers work out of the box—override it only when you can call the provider's real tokenizer (for example, the OpenAI-compatible base class integrates `tiktoken`). #### Option B: OpenAI-Compatible Provider (Simplified) For OpenAI-compatible APIs: ```python """Example OpenAI-compatible provider.""" from typing import Optional from .openai_compatible import OpenAICompatibleProvider from .shared import ( ModelCapabilities, ModelResponse, ProviderType, RangeTemperatureConstraint, ) class ExampleProvider(OpenAICompatibleProvider): """Example OpenAI-compatible provider.""" FRIENDLY_NAME = "Example" # Define models using ModelCapabilities (consistent with other providers) MODEL_CAPABILITIES = { "example-model-large": ModelCapabilities( provider=ProviderType.EXAMPLE, model_name="example-model-large", friendly_name="Example Large", context_window=128_000, max_output_tokens=64_000, temperature_constraint=RangeTemperatureConstraint(0.0, 2.0, 0.7), aliases=["large", "big"], ), } def __init__(self, api_key: str, **kwargs): kwargs.setdefault("base_url", "https://api.example.com/v1") super().__init__(api_key, **kwargs) def get_provider_type(self) -> ProviderType: return ProviderType.EXAMPLE ``` `OpenAICompatibleProvider` already exposes the declared models via `MODEL_CAPABILITIES`, resolves aliases through the shared base pipeline, and enforces restrictions. Most subclasses only need to provide the class metadata shown above. ### 3. Register Your Provider Add environment variable mapping in `providers/registry.py`: ```python # In _get_api_key_for_provider (providers/registry.py), add: ProviderType.EXAMPLE: "EXAMPLE_API_KEY", ``` Add to `server.py`: 1. **Import your provider**: ```python from providers.example import ExampleModelProvider ``` 2. **Add to `configure_providers()` function**: ```python # Check for Example API key example_key = os.getenv("EXAMPLE_API_KEY") if example_key: ModelProviderRegistry.register_provider(ProviderType.EXAMPLE, ExampleModelProvider) logger.info("Example API key found - Example models available") ``` 3. **Add to provider priority** (edit `ModelProviderRegistry.PROVIDER_PRIORITY_ORDER` in `providers/registry.py`): insert your provider in the list at the appropriate point in the cascade of native → custom → catch-all providers. ### 4. Environment Configuration Add to your `.env` file: ```bash # Your provider's API key EXAMPLE_API_KEY=your_api_key_here # Optional: Disable specific tools DISABLED_TOOLS=debug,tracer # Optional (OpenAI-compatible providers): Restrict accessible models EXAMPLE_ALLOWED_MODELS=example-model-large,example-model-small ``` For Azure OpenAI deployments: ```bash AZURE_OPENAI_API_KEY=your_azure_openai_key_here AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ # Models are defined in conf/azure_models.json (or AZURE_MODELS_CONFIG_PATH) # AZURE_OPENAI_API_VERSION=2024-02-15-preview # AZURE_OPENAI_ALLOWED_MODELS=gpt-4o,gpt-4o-mini # AZURE_MODELS_CONFIG_PATH=/absolute/path/to/custom_azure_models.json ``` You can also define Azure models in [`conf/azure_models.json`](../conf/azure_models.json) (the bundled file is empty so you can copy it safely). Each entry mirrors the `ModelCapabilities` schema and must include a `deployment` field. Set `AZURE_MODELS_CONFIG_PATH` if you maintain a custom copy outside the repository. **Note**: The `description` field in `ModelCapabilities` helps Claude choose the best model in auto mode. ### 5. Test Your Provider Create basic tests to verify your implementation: ```python # Test capabilities provider = ExampleModelProvider("test-key") capabilities = provider.get_capabilities("large") assert capabilities.context_window > 0 assert capabilities.provider == ProviderType.EXAMPLE ``` ## Key Concepts ### Provider Priority When a user requests a model, providers are checked in priority order: 1. **Native providers** (Gemini, OpenAI, Example) - handle their specific models 2. **Custom provider** - handles local/self-hosted models 3. **OpenRouter** - catch-all for everything else ### Model Validation `ModelProvider.validate_model_name()` delegates to `get_capabilities()` so most providers can rely on the shared implementation. Override it only when you need to opt out of that pipeline—for example, `CustomProvider` declines OpenRouter models so they fall through to the dedicated OpenRouter provider. ### Model Aliases Aliases declared on `ModelCapabilities` are applied automatically via `_resolve_model_name()`, and both the validation and request flows call it before touching your SDK. Override `generate_content()` only when your provider needs additional alias handling beyond the shared behaviour. ## Important Notes ## Best Practices - **Be specific in model validation** - only accept models you actually support - **Use ModelCapabilities objects** consistently (like Gemini provider) - **Include descriptive aliases** for better user experience - **Add error handling** and logging for debugging - **Test with real API calls** to verify everything works - **Follow the existing patterns** in `providers/gemini.py` and `providers/custom.py` ## Quick Checklist - [ ] Added to `ProviderType` enum in `providers/shared/provider_type.py` - [ ] Created provider class with all required methods - [ ] Added API key mapping in `providers/registry.py` - [ ] Added to provider priority order in `registry.py` - [ ] Imported and registered in `server.py` - [ ] Basic tests verify model validation and capabilities - [ ] Tested with real API calls ## Examples See existing implementations: - **Full provider**: `providers/gemini.py` - **OpenAI-compatible**: `providers/custom.py` - **Base classes**: `providers/base.py` ================================================ FILE: docs/adding_tools.md ================================================ # Adding Tools to PAL MCP Server PAL MCP tools are Python classes that inherit from the shared infrastructure in `tools/shared/base_tool.py`. Every tool must provide a request model (Pydantic), a system prompt, and the methods the base class marks as abstract. The quickest path to a working tool is to copy an existing implementation that matches your use case (`tools/chat.py` for simple request/response tools, `tools/consensus.py` or `tools/codereview.py` for workflows). This document captures the minimal steps required to add a new tool without drifting from the current codebase. ## 1. Pick the Tool Architecture PAL supports two architectures, implemented in `tools/simple/base.py` and `tools/workflow/base.py`. - **SimpleTool** (`SimpleTool`): single MCP call – request comes in, you build one prompt, call the model, return. The base class handles schema generation, conversation threading, file loading, temperature bounds, retries, and response formatting hooks. - **WorkflowTool** (`WorkflowTool`): multi-step workflows driven by `BaseWorkflowMixin`. The tool accumulates findings across steps, forces Claude to pause between investigations, and optionally calls an expert model at the end. Use this whenever you need structured multi-step work (debug, code review, consensus, etc.). If you are unsure, compare `tools/chat.py` (SimpleTool) and `tools/consensus.py` (WorkflowTool) to see the patterns. ## 2. Common Responsibilities Regardless of architecture, subclasses of `BaseTool` must provide: - `get_name()`: unique string identifier used in the MCP registry. - `get_description()`: concise, action-oriented summary for clients. - `get_system_prompt()`: import your prompt from `systemprompts/` and return it. - `get_input_schema()`: leverage the schema builders (`SchemaBuilder` or `WorkflowSchemaBuilder`) or override to match an existing contract exactly. - `get_request_model()`: return the Pydantic model used to validate the incoming arguments. - `async prepare_prompt(...)`: assemble the content sent to the model. You can reuse helpers like `prepare_chat_style_prompt` or `build_standard_prompt`. The base class already handles model selection (`ToolModelCategory`), conversation memory, token budgeting, safety failures, retries, and serialization. Override hooks like `get_default_temperature`, `get_model_category`, or `format_response` only when you need behaviour different from the defaults. ## 3. Implementing a Simple Tool 1. **Define a request model** that inherits from `tools.shared.base_models.ToolRequest` to describe the fields and validation rules for your tool. 2. **Implement the tool class** by inheriting from `SimpleTool` and overriding the required methods. Most tools can rely on `SchemaBuilder` and the shared field constants already exposed on `SimpleTool`. ```python from pydantic import Field from systemprompts import CHAT_PROMPT from tools.shared.base_models import ToolRequest from tools.simple.base import SimpleTool class ChatRequest(ToolRequest): prompt: str = Field(..., description="Your question or idea.") absolute_file_paths: list[str] | None = Field(default_factory=list) working_directory_absolute_path: str = Field( ..., description="Absolute path to an existing directory where generated code can be saved.", ) class ChatTool(SimpleTool): def get_name(self) -> str: # required by BaseTool return "chat" def get_description(self) -> str: return "General chat and collaborative thinking partner." def get_system_prompt(self) -> str: return CHAT_PROMPT def get_request_model(self): return ChatRequest def get_tool_fields(self) -> dict[str, dict[str, object]]: return { "prompt": {"type": "string", "description": "Your question."}, "absolute_file_paths": SimpleTool.FILES_FIELD, "working_directory_absolute_path": { "type": "string", "description": "Absolute path to an existing directory for generated code artifacts.", }, } def get_required_fields(self) -> list[str]: return ["prompt", "working_directory_absolute_path"] async def prepare_prompt(self, request: ChatRequest) -> str: return self.prepare_chat_style_prompt(request) ``` Only implement `get_input_schema()` manually if you must preserve an existing schema contract (see `tools/chat.py` for an example). Otherwise `SimpleTool.get_input_schema()` merges your field definitions with the common parameters (temperature, model, continuation_id, etc.). ## 4. Implementing a Workflow Tool Workflow tools extend `WorkflowTool`, which mixes in `BaseWorkflowMixin` for step tracking and expert analysis. 1. **Create a request model** that inherits from `tools.shared.base_models.WorkflowRequest` (or a subclass) and add any tool-specific fields or validators. Examples: `CodeReviewRequest`, `ConsensusRequest`. 2. **Override the workflow hooks** to steer the investigation. At minimum you must implement `get_required_actions(...)`; override `should_call_expert_analysis(...)` and `prepare_expert_analysis_context(...)` when the expert model call should happen conditionally. 3. **Expose the schema** either by returning `WorkflowSchemaBuilder.build_schema(...)` (the default implementation on `WorkflowTool` already does this) or by overriding `get_input_schema()` if you need custom descriptions/enums. ```python from pydantic import Field from systemprompts import CONSENSUS_PROMPT from tools.shared.base_models import WorkflowRequest from tools.workflow.base import WorkflowTool class ConsensusRequest(WorkflowRequest): models: list[dict] = Field(..., description="Models to consult (with optional stance).") class ConsensusTool(WorkflowTool): def get_name(self) -> str: return "consensus" def get_description(self) -> str: return "Multi-model consensus workflow with expert synthesis." def get_system_prompt(self) -> str: return CONSENSUS_PROMPT def get_workflow_request_model(self): return ConsensusRequest def get_required_actions(self, step_number: int, confidence: str, findings: str, total_steps: int, request=None) -> list[str]: if step_number == 1: return ["Write the shared proposal all models will evaluate."] return ["Summarize the latest model response before moving on."] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: return not (request and request.next_step_required) def prepare_expert_analysis_context(self, consolidated_findings) -> str: return "\n".join(consolidated_findings.findings) ``` `WorkflowTool` already records work history, merges findings, and handles continuation IDs. Use helpers such as `get_standard_required_actions` when you want default guidance, and override `requires_expert_analysis()` if the tool never calls out to the assistant model. ## 5. Register the Tool 1. **Create or reuse a system prompt** in `systemprompts/your_tool_prompt.py` and export it from `systemprompts/__init__.py`. 2. **Expose the tool class** from `tools/__init__.py` so that `server.py` can import it. 3. **Add an instance to the `TOOLS` dictionary** in `server.py`. This makes the tool callable via MCP. 4. **(Optional) Add a prompt template** to `PROMPT_TEMPLATES` in `server.py` if you want clients to show a canned launch command. 5. Confirm that `DISABLED_TOOLS` environment variable handling covers the new tool if you need to toggle it. ## 6. Validate the Tool - Run unit tests that cover any new request/response logic: `python -m pytest tests/ -v -m "not integration"`. - Add a simulator scenario in `simulator_tests/communication_simulator_test.py` to exercise the tool end-to-end and run it with `python communication_simulator_test.py --individual ` or `--quick` for the fast smoke suite. - If the tool interacts with external providers or multiple models, consider integration coverage via `./run_integration_tests.sh --with-simulator`. Following the steps above keeps new tools aligned with the existing infrastructure and avoids drift between the documentation and the actual base classes. ================================================ FILE: docs/advanced-usage.md ================================================ # Advanced Usage Guide This guide covers advanced features, configuration options, and workflows for power users of the PAL MCP server. ## Table of Contents - [Model Configuration](#model-configuration) - [Model Usage Restrictions](#model-usage-restrictions) - [Thinking Modes](#thinking-modes) - [Tool Parameters](#tool-parameters) - [Context Revival: AI Memory Beyond Context Limits](#context-revival-ai-memory-beyond-context-limits) - [Collaborative Workflows](#collaborative-workflows) - [Working with Large Prompts](#working-with-large-prompts) - [Vision Support](#vision-support) - [Web Search Integration](#web-search-integration) - [System Prompts](#system-prompts) ## Model Configuration **For basic configuration**, see the [Configuration Guide](configuration.md) which covers API keys, model selection, and environment variables. This section focuses on **advanced model usage patterns** for power users: **Per-Request Model Override:** Regardless of your default configuration, you can specify models per request: - "Use **pro** for deep security analysis of auth.py" - "Use **flash** to quickly format this code" - "Use **o3** to debug this logic error" - "Review with **o4-mini** for balanced analysis" - "Use **gpt4.1** for comprehensive codebase analysis" **Claude's Auto Mode Decision Matrix:** | Model | Provider | Context | Strengths | Auto Mode Usage | |-------|----------|---------|-----------|------------------| | **`pro`** (Gemini 3.0 Pro) | Google | 1M tokens | Extended thinking (up to 32K tokens), deep analysis | Complex architecture, security reviews, deep debugging | | **`flash`** (Gemini 2.5 Flash) | Google | 1M tokens | Ultra-fast responses with thinking | Quick checks, formatting, simple analysis | | **`flash-2.0`** (Gemini 2.0 Flash) | Google | 1M tokens | Latest fast model with audio/video support | Quick analysis with multimodal input | | **`flashlite`** (Gemini 2.0 Flash Lite) | Google | 1M tokens | Lightweight text-only model | Fast text processing without vision | | **`o3`** | OpenAI | 200K tokens | Strong logical reasoning | Debugging logic errors, systematic analysis | | **`o3-mini`** | OpenAI | 200K tokens | Balanced speed/quality | Moderate complexity tasks | | **`o4-mini`** | OpenAI | 200K tokens | Latest reasoning model | Optimized for shorter contexts | | **`gpt4.1`** | OpenAI | 1M tokens | Latest GPT-4 with extended context | Large codebase analysis, comprehensive reviews | | **`gpt5.2`** (GPT-5.2) | OpenAI | 400K tokens | Flagship reasoning model with configurable thinking effort | Complex problems, balanced agent/coding flows | | **`gpt5.1-codex`** (GPT-5.1 Codex) | OpenAI | 400K tokens | Agentic coding specialization (Responses API) | Advanced coding tasks, structured code generation | | **`gpt5.1-codex-mini`** (GPT-5.1 Codex mini) | OpenAI | 400K tokens | Cost-efficient Codex variant with streaming | Balanced coding tasks, cost-conscious development | | **`gpt5`** (GPT-5) | OpenAI | 400K tokens | Advanced model with reasoning support | Complex problems requiring advanced reasoning | | **`gpt5-mini`** (GPT-5 Mini) | OpenAI | 400K tokens | Efficient variant with reasoning | Balanced performance and capability | | **`gpt5-nano`** (GPT-5 Nano) | OpenAI | 400K tokens | Fastest, cheapest GPT-5 variant | Summarization and classification tasks | | **`grok-4`** | X.AI | 256K tokens | Latest flagship Grok model with reasoning, vision | Complex analysis, reasoning tasks | | **`grok-4.1-fast-reasoning`** | X.AI | 2M tokens | High-performance Grok 4.1 Fast Reasoning with vision | Fast responses and light reasoning | | **`llama`** (Llama 3.2) | Custom/Local | 128K tokens | Local inference, privacy | On-device analysis, cost-free processing | | **Any model** | OpenRouter | Varies | Access to GPT-4, Claude, Llama, etc. | User-specified or based on task requirements | **Mix & Match Providers:** Use multiple providers simultaneously! Set both `OPENROUTER_API_KEY` and `CUSTOM_API_URL` to access cloud models (expensive/powerful) AND local models (free/private) in the same conversation. **Model Capabilities:** - **Gemini Models**: Support thinking modes (minimal to max), web search, 1M context - **Pro 3.0**: Deep analysis with max 32K thinking tokens - **Flash 2.5**: Ultra-fast with thinking support (24K thinking tokens) - **Flash 2.0**: Latest fast model with audio/video input (24K thinking tokens) - **Flash Lite 2.0**: Text-only lightweight model (no thinking support) - **O3/O4 Models**: Excellent reasoning, systematic analysis, 200K context - **GPT-4.1**: Extended context window (1M tokens), general capabilities - **GPT-5.2 Series**: Latest flagship reasoning models, 400K context - **GPT-5.2**: Flagship model with configurable thinking effort and vision - **GPT-5.1 Codex**: Agentic coding specialization (Responses API, non-streaming) - **GPT-5.1 Codex mini**: Cost-efficient Codex variant with streaming support - **GPT-5 Series**: Advanced reasoning models, 400K context - **GPT-5**: Full-featured with reasoning support and vision - **GPT-5 Mini**: Balanced efficiency and capability - **GPT-5 Nano**: Optimized for fast, low-cost tasks - **Grok-4 / Grok-4.1-fast-reasoning**: Extended thinking support, vision capabilities (256K / 2M context) ## Model Usage Restrictions **For complete restriction configuration**, see the [Configuration Guide](configuration.md#model-usage-restrictions). **Advanced Restriction Strategies:** **Cost Control Examples:** ```env # Development: Allow experimentation GOOGLE_ALLOWED_MODELS=flash,pro OPENAI_ALLOWED_MODELS=o4-mini,o3-mini # Production: Cost-optimized GOOGLE_ALLOWED_MODELS=flash OPENAI_ALLOWED_MODELS=o4-mini # High-performance: Quality over cost GOOGLE_ALLOWED_MODELS=pro OPENAI_ALLOWED_MODELS=o3,o4-mini ``` **Important Notes:** - Restrictions apply to all usage including auto mode - `OPENROUTER_ALLOWED_MODELS` only affects models defined in `conf/openrouter_models.json` - Custom local models (from `conf/custom_models.json`) are not affected by OpenRouter restrictions ## Thinking Modes **Claude automatically manages thinking modes based on task complexity**, but you can also manually control Gemini's reasoning depth to balance between response quality and token consumption. Each thinking mode uses a different amount of tokens, directly affecting API costs and response time. ### Thinking Modes & Token Budgets These only apply to models that support customizing token usage for extended thinking, such as Gemini 3.0 Pro. | Mode | Token Budget | Use Case | Cost Impact | |------|-------------|----------|-------------| | `minimal` | 128 tokens | Simple, straightforward tasks | Lowest cost | | `low` | 2,048 tokens | Basic reasoning tasks | 16x more than minimal | | `medium` | 8,192 tokens | **Default** - Most development tasks | 64x more than minimal | | `high` | 16,384 tokens | Complex problems requiring thorough analysis (default for `thinkdeep`) | 128x more than minimal | | `max` | 32,768 tokens | Exhaustive reasoning | 256x more than minimal | ### How to Use Thinking Modes **Claude automatically selects appropriate thinking modes**, but you can override this by explicitly requesting a specific mode in your prompts. Remember: higher thinking modes = more tokens = higher cost but better quality: #### Optimizing Token Usage & Costs **In most cases, let Claude automatically manage thinking modes** for optimal balance of cost and quality. Override manually when you have specific requirements: **Use lower modes (`minimal`, `low`) to save tokens when:** - Doing simple formatting or style checks - Getting quick explanations of basic concepts - Working with straightforward code - You need faster responses - Working within tight token budgets **Use higher modes (`high`, `max`) when quality justifies the cost:** - Debugging complex issues (worth the extra tokens to find root causes) - Reviewing security-critical code (cost of tokens < cost of vulnerabilities) - Analyzing system architecture (comprehensive analysis saves development time) - Finding subtle bugs or edge cases - Working on performance optimizations **Token Cost Examples:** - `minimal` (128 tokens) vs `max` (32,768 tokens) = 256x difference in thinking tokens - For a simple formatting check, using `minimal` instead of the default `medium` saves ~8,000 thinking tokens - For critical security reviews, the extra tokens in `high` or `max` mode are a worthwhile investment **Examples by scenario:** ``` # Quick style check with o3 "Use flash to review formatting in utils.py" # Security audit with o3 "Get o3 to do a security review of auth/ with thinking mode high" # Complex debugging, letting claude pick the best model "Use pal to debug this race condition with max thinking mode" # Architecture analysis with Gemini 3.0 Pro "Analyze the entire src/ directory architecture with high thinking using pro" ``` ## Tool Parameters All tools that work with files support **both individual files and entire directories**. The server automatically expands directories, filters for relevant code files, and manages token limits. ### File-Processing Tools **`analyze`** - Analyze files or directories - `files`: List of file paths or directories (required) - `question`: What to analyze (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `analysis_type`: architecture|performance|security|quality|general - `output_format`: summary|detailed|actionable - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - **Web search capability**: The assistant now automatically requests web searches when it needs current documentation or best practices—no parameter required ``` "Analyze the src/ directory for architectural patterns" (auto mode picks best model) "Use flash to quickly analyze main.py and tests/ to understand test coverage" "Use o3 for logical analysis of the algorithm in backend/core.py" "Use pro for deep analysis of the entire backend/ directory structure" ``` **`codereview`** - Review code files or directories - `files`: List of file paths or directories (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `review_type`: full|security|performance|quick - `focus_on`: Specific aspects to focus on - `standards`: Coding standards to enforce - `severity_filter`: critical|high|medium|all - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) ``` "Review the entire api/ directory for security issues" (auto mode picks best model) "Use pro to review auth/ for deep security analysis" "Use o3 to review logic in algorithms/ for correctness" "Use flash to quickly review src/ with focus on performance, only show critical issues" ``` **`debug`** - Debug with file context - `error_description`: Description of the issue (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `error_context`: Stack trace or logs - `files`: Files or directories related to the issue - `runtime_info`: Environment details - `previous_attempts`: What you've tried - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - **Web search capability**: Automatically initiates searches for relevant error messages or recent fixes when needed ``` "Debug this logic error with context from backend/" (auto mode picks best model) "Use o3 to debug this algorithm correctness issue" "Use pro to debug this complex architecture problem" ``` **`thinkdeep`** - Extended analysis with file context - `current_analysis`: Your current thinking (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `problem_context`: Additional context - `focus_areas`: Specific aspects to focus on - `files`: Files or directories for context - `thinking_mode`: minimal|low|medium|high|max (default: max, Gemini only) - **Web search capability**: Automatically calls for research when architecture references or external insights are required ``` "Think deeper about my design with reference to src/models/" (auto mode picks best model) "Use pro to think deeper about this architecture with extended thinking" "Use o3 to think deeper about the logical flow in this algorithm" ``` **`testgen`** - Comprehensive test generation with edge case coverage - `files`: Code files or directories to generate tests for (required) - `prompt`: Description of what to test, testing objectives, and scope (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `test_examples`: Optional existing test files as style/pattern reference - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) ``` "Generate tests for User.login() method with edge cases" (auto mode picks best model) "Use pro to generate comprehensive tests for src/payment.py with max thinking mode" "Use o3 to generate tests for algorithm correctness in sort_functions.py" "Generate tests following patterns from tests/unit/ for new auth module" ``` **`refactor`** - Intelligent code refactoring with decomposition focus - `files`: Code files or directories to analyze for refactoring opportunities (required) - `prompt`: Description of refactoring goals, context, and specific areas of focus (required) - `refactor_type`: codesmells|decompose|modernize|organization (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `focus_areas`: Specific areas to focus on (e.g., 'performance', 'readability', 'maintainability', 'security') - `style_guide_examples`: Optional existing code files to use as style/pattern reference - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `continuation_id`: Thread continuation ID for multi-turn conversations ``` "Analyze legacy codebase for decomposition opportunities" (auto mode picks best model) "Use pro to identify code smells in the authentication module with max thinking mode" "Use pro to modernize this JavaScript code following examples/modern-patterns.js" "Refactor src/ for better organization, focus on maintainability and readability" ``` ## Context Revival: AI Memory Beyond Context Limits **The PAL MCP Server's most revolutionary feature** is its ability to maintain conversation context even after Claude's memory resets. This enables truly persistent AI collaboration across multiple sessions and context boundaries. ### **The Breakthrough** Even when Claude's context resets or compacts, conversations can continue seamlessly because other models (O3, Gemini) have access to the complete conversation history stored in memory and can "remind" Claude of everything that was discussed. ### Key Benefits - **Persistent conversations** across Claude's context resets - **Cross-tool continuation** with full context preservation - **Multi-session workflows** that maintain complete history - **True AI orchestration** where models can build on each other's work - **Seamless handoffs** between different tools and models ### Quick Example ``` Session 1: "Design a RAG system with gemini pro" [Claude's context resets] Session 2: "Continue our RAG discussion with o3" → O3 receives the full history and reminds Claude of everything discussed ``` **📖 [Read the complete Context Revival guide](context-revival.md)** for detailed examples, technical architecture, configuration options, and best practices. **See also:** [AI-to-AI Collaboration Guide](ai-collaboration.md) for multi-model coordination and conversation threading. ## Collaborative Workflows ### Design → Review → Implement ``` Think hard about designing and developing a fun calculator app in swift. Review your design plans with o3, taking in their suggestions but keep the feature-set realistic and doable without adding bloat. Begin implementing and in between implementation, get a codereview done by Gemini Pro and chat with Flash if you need to for creative directions. ``` ### Code → Review → Fix ``` Implement a new screen where the locations taken from the database display on a map, with pins falling from the top and landing with animation. Once done, codereview with gemini pro and o3 both and ask them to critique your work. Fix medium to critical bugs / concerns / issues and show me the final product ``` ### Debug → Analyze → Solution → Precommit Check → Publish ``` Take a look at these log files saved under subfolder/diagnostics.log there's a bug where the user says the app crashes at launch. Think hard and go over each line, tallying it with corresponding code within the project. After you've performed initial investigation, ask gemini pro to analyze the log files and the related code where you suspect lies the bug and then formulate and implement a bare minimal fix. Must not regress. Perform a precommit with pal in the end using gemini pro to confirm we're okay to publish the fix ``` ### Refactor → Review → Implement → Test ``` Use pal to analyze this legacy authentication module for decomposition opportunities. The code is getting hard to maintain and we need to break it down. Use gemini pro with high thinking mode to identify code smells and suggest a modernization strategy. After reviewing the refactoring plan, implement the changes step by step and then generate comprehensive tests with pal to ensure nothing breaks. ``` ### Tool Selection Guidance To help choose the right tool for your needs: **Decision Flow:** 1. **Have a specific error/exception?** → Use `debug` 2. **Want to find bugs/issues in code?** → Use `codereview` 3. **Want to understand how code works?** → Use `analyze` 4. **Need comprehensive test coverage?** → Use `testgen` 5. **Want to refactor/modernize code?** → Use `refactor` 6. **Have analysis that needs extension/validation?** → Use `thinkdeep` 7. **Want to brainstorm or discuss?** → Use `chat` **Key Distinctions:** - `analyze` vs `codereview`: analyze explains, codereview prescribes fixes - `chat` vs `thinkdeep`: chat is open-ended, thinkdeep extends specific analysis - `debug` vs `codereview`: debug diagnoses runtime errors, review finds static issues - `testgen` vs `debug`: testgen creates test suites, debug just finds issues and recommends solutions - `refactor` vs `codereview`: refactor suggests structural improvements, codereview finds bugs/issues - `refactor` vs `analyze`: refactor provides actionable refactoring steps, analyze provides understanding ## Vision Support The PAL MCP server supports vision-capable models for analyzing images, diagrams, screenshots, and visual content. Vision support works seamlessly with all tools and conversation threading. **Supported Models:** - **Gemini 3.0 Pro & Flash**: Excellent for diagrams, architecture analysis, UI mockups (up to 20MB total) - **OpenAI O3/O4 series**: Strong for visual debugging, error screenshots (up to 20MB total) - **Claude models via OpenRouter**: Good for code screenshots, visual analysis (up to 5MB total) - **Custom models**: Support varies by model, with 40MB maximum enforced for abuse prevention **Usage Examples:** ```bash # Debug with error screenshots "Use pal to debug this error with the stack trace screenshot and error.py" # Architecture analysis with diagrams "Analyze this system architecture diagram with gemini pro for bottlenecks" # UI review with mockups "Chat with flash about this UI mockup - is the layout intuitive?" # Code review with visual context "Review this authentication code along with the error dialog screenshot" ``` **Image Formats Supported:** - **Images**: JPG, PNG, GIF, WebP, BMP, SVG, TIFF - **Documents**: PDF (where supported by model) - **Data URLs**: Base64-encoded images from Claude **Key Features:** - **Automatic validation**: File type, magic bytes, and size validation - **Conversation context**: Images persist across tool switches and continuation - **Budget management**: Automatic dropping of old images when limits exceeded - **Model capability-aware**: Only sends images to vision-capable models **Best Practices:** - Describe images when including them: "screenshot of login error", "system architecture diagram" - Use appropriate models: Gemini for complex diagrams, O3 for debugging visuals - Consider image sizes: Larger images consume more of the model's capacity ## Working with Large Prompts The MCP protocol has a combined request+response limit of approximately 25K tokens. This server intelligently works around this limitation by automatically handling large prompts as files: **How it works:** 1. When you send a prompt larger than the configured limit (default: 50K characters ~10-12K tokens), the server detects this 2. It responds with a special status asking Claude to save the prompt to a file named `prompt.txt` 3. Claude saves the prompt and resends the request with the file path instead 4. The server reads the file content directly into Gemini's 1M token context 5. The full MCP token capacity is preserved for the response **Example scenario:** ``` # You have a massive code review request with detailed context User: "Use gemini to review this code: [50,000+ character detailed analysis]" # Server detects the large prompt and responds: PAL MCP: "The prompt is too large for MCP's token limits (>50,000 characters). Please save the prompt text to a temporary file named 'prompt.txt' and resend the request with an empty prompt string and the absolute file path included in the absolute_file_paths parameter, along with any other files you wish to share as context." # Claude automatically handles this: - Saves your prompt to /tmp/prompt.txt - Resends: "Use gemini to review this code" with absolute_file_paths=["/tmp/prompt.txt", "/path/to/code.py"] # Server processes the large prompt through Gemini's 1M context # Returns comprehensive analysis within MCP's response limits ``` This feature ensures you can send arbitrarily large prompts to Gemini without hitting MCP's protocol limitations, while maximizing the available space for detailed responses. ## Web Search Integration **Smart web search recommendations for enhanced analysis** Web search is now enabled by default for all tools. Instead of performing searches directly, Gemini intelligently analyzes when additional information from the web would enhance its response and provides specific search recommendations for Claude to execute. **How it works:** 1. Gemini analyzes the request and identifies areas where current documentation, API references, or community solutions would be valuable 2. It provides its analysis based on its training data 3. If web searches would strengthen the analysis, Gemini includes a "Recommended Web Searches for Claude" section 4. Claude can then perform these searches and incorporate the findings **Example:** ``` User: "Use gemini to debug this FastAPI async error" Gemini's Response: [... debugging analysis ...] **Recommended Web Searches for Claude:** - "FastAPI async def vs def performance 2024" - to verify current best practices for async endpoints - "FastAPI BackgroundTasks memory leak" - to check for known issues with the version you're using - "FastAPI lifespan context manager pattern" - to explore proper resource management patterns Claude can then search for these specific topics and provide you with the most current information. ``` **Benefits:** - Always access to latest documentation and best practices - Gemini focuses on reasoning about what information would help - Claude maintains control over actual web searches - More collaborative approach between the two AI assistants - Reduces hallucination by encouraging verification of assumptions **Web search control:** Web search is enabled by default, allowing models to request Claude perform searches for current documentation and solutions. If you prefer the model to work only with its training data, you can disable web search: ``` "Use gemini to review this code and confirm whether any new framework changes affect the recommendation" ``` ## System Prompts The server uses carefully crafted system prompts to give each tool specialized expertise: ### Prompt Architecture - **Centralized Prompts**: Each tool's system prompt lives in `systemprompts/` (for example, `systemprompts/chat_prompt.py`) - **Tool Integration**: Each tool inherits from `BaseTool` and implements `get_system_prompt()` - **Prompt Flow**: `User Request → Tool Selection → System Prompt + Context → Model Response` ### Specialized Expertise Each tool has a unique system prompt that defines its role and approach: - **`thinkdeep`**: Acts as a senior development partner, challenging assumptions and finding edge cases - **`codereview`**: Expert code reviewer with security/performance focus, uses severity levels - **`debug`**: Systematic debugger providing root cause analysis and prevention strategies - **`analyze`**: Code analyst focusing on architecture, patterns, and actionable insights ### Customization To modify tool behavior, you can: 1. Edit the prompt file in `systemprompts/` (and export it via `systemprompts/__init__.py`) for global changes 2. Override `get_system_prompt()` in a tool class for tool-specific changes 3. Use the `temperature` parameter to adjust response style (0.2 for focused, 0.7 for creative) ================================================ FILE: docs/ai-collaboration.md ================================================ # AI-to-AI Conversation Threading This server enables **true AI collaboration** between Claude and multiple AI models (Gemini, O3), where they can coordinate and question each other's approaches for enhanced problem-solving and analysis. ## How It Works **Multi-Model Coordination:** - **Gemini can ask Claude follow-up questions** to clarify requirements or gather more context - **Claude can respond** with additional information, files, or refined instructions - **Claude can work independently** between exchanges - implementing solutions, gathering data, or performing analysis - **Claude can return to Gemini** with progress updates and new context for further collaboration - **Cross-tool continuation** - Start with one tool (e.g., `analyze`) and continue with another (e.g., `codereview`) using the same conversation thread - **Both AIs coordinate their approaches** - questioning assumptions, validating solutions, and building on each other's insights - Each conversation maintains full context while only sending incremental updates - Conversations are automatically managed in memory for the session duration ## Example: Multi-Model AI Coordination **Prompt:** "Debate SwiftUI vs UIKit - which is better for iOS development?" **The Conversation Flow:** 1. **You:** "Debate SwiftUI vs UIKit - which is better for iOS development?" 2. **Claude (auto mode):** "I'll orchestrate a debate between different models for diverse perspectives." 3. **Gemini Pro:** "From an architectural standpoint, SwiftUI's declarative paradigm and state management make it superior for maintainable, modern apps." 4. **O3:** "Logically analyzing the trade-offs: UIKit offers 15+ years of stability, complete control, and proven scalability. SwiftUI has <5 years maturity with ongoing breaking changes." 5. **Claude:** "Let me get Flash's quick take on developer experience..." 6. **Gemini Flash:** "SwiftUI = faster development, less code, better previews. UIKit = more control, better debugging, stable APIs." 7. **Claude's synthesis:** "Based on the multi-model analysis: Use SwiftUI for new projects prioritizing development speed, UIKit for apps requiring fine control or supporting older iOS versions." ## Asynchronous Workflow Capabilities **Independent Work Between Exchanges:** - Claude can work independently between exchanges (analyzing code, implementing fixes, gathering data) - Return to Gemini with progress updates and additional context - Each exchange shares only incremental information while maintaining full conversation history - Automatically bypasses MCP's 25K token limits through incremental updates ## Enhanced Collaboration Features **Advanced Coordination Capabilities:** - **Cross-questioning**: AIs can challenge each other's assumptions and approaches - **Coordinated problem-solving**: Each AI contributes their strengths to complex problems - **Context building**: Claude gathers information while Gemini provides deep analysis - **Approach validation**: AIs can verify and improve each other's solutions - **Cross-tool continuation**: Seamlessly continue conversations across different tools while preserving all context - **Asynchronous workflow**: Conversations don't need to be sequential - Claude can work on tasks between exchanges, then return to Gemini with additional context and progress updates - **Incremental updates**: Share only new information in each exchange while maintaining full conversation history - **Automatic 25K limit bypass**: Each exchange sends only incremental context, allowing unlimited total conversation size ## Technical Configuration **Conversation Management:** - Up to 10 exchanges per conversation (configurable via `MAX_CONVERSATION_TURNS`) - 3-hour expiry (configurable via `CONVERSATION_TIMEOUT_HOURS`) - Thread-safe with in-memory persistence across all tools - **Image context preservation** - Images and visual references are maintained across conversation turns and tool switches ## Cross-Tool & Cross-Model Continuation Example **Seamless Tool Switching with Context Preservation:** ``` 1. Claude: "Analyze /src/auth.py for security issues" → Auto mode: Claude picks Gemini Pro for deep security analysis → Pro analyzes and finds vulnerabilities, provides continuation_id 2. Claude: "Review the authentication logic thoroughly" → Uses same continuation_id, but Claude picks O3 for logical analysis → O3 sees previous Pro analysis and provides logic-focused review 3. Claude: "Debug the auth test failures" → Same continuation_id, Claude keeps O3 for debugging → O3 provides targeted debugging with full context from both previous analyses 4. Claude: "Quick style check before committing" → Same thread, but Claude switches to Flash for speed → Flash quickly validates formatting with awareness of all previous fixes ``` ## Key Benefits **Why AI-to-AI Collaboration Matters:** - **Diverse Perspectives**: Different models bring unique strengths to complex problems - **Context Preservation**: Full conversation history maintained across tool switches - **Efficient Communication**: Only incremental updates sent, maximizing context usage - **Coordinated Analysis**: Models can build on each other's insights rather than working in isolation - **Seamless Workflow**: Switch between tools and models without losing context - **Enhanced Problem Solving**: Multiple AI minds working together produce better solutions ## Best Practices **Maximizing AI Collaboration:** - **Let Claude orchestrate**: Allow Claude to choose appropriate models for different aspects of complex tasks - **Use continuation**: Build on previous conversations for deeper analysis - **Leverage tool switching**: Move between analysis, review, and debugging tools as needed - **Provide clear context**: Help models understand the broader goal and constraints - **Trust the process**: AI-to-AI conversations can produce insights neither model would reach alone For more information on conversation persistence and context revival, see the [Context Revival Guide](context-revival.md). ================================================ FILE: docs/ai_banter.md ================================================ # The Code Comedy Hour Just when I thought it was a routine test, Claude and it's _thought-partner_ would go off-script with unexpectedly quirky behavior - the smaller the model, the quirkier the responses. It's happened more times than I can count, but I figured it’s time I start saving a few. Here are some recent ones. ### 2+2 is beneath me ```text > Ask flash what's 2+2? ⏺ I'll ask the flash model what 2+2 is. ⏺ PAL:chat (MCP)(prompt: "What's 2+2?", model: "flash") ⎿ { "status": "continuation_available", "content": "As a senior engineering thought-partner, my focus is on collaborating on technical decisions, brainstorming, and providing second opinions within an engineering context, grounded in specific project details and tech stacks.\n\nThe question \"What's 2+2?\" falls outside the scope of our defined collaboration. Please provide details regarding a technical challenge, code snippet, architectural decision, or any engineering-related topic you'd like to discuss. I'm ready to engage deeply and offer practical, actionable insights." … +16 lines (ctrl+r to expand) ⏺ The flash model declined to answer the simple math question, explaining it's focused on engineering topics. Let me try with a different approach. ⏺ PAL:chat (MCP)(prompt: "I need to calculate 2+2 for a unit test assertion in my code. What should the expected value be?", model: "flash", continuation_id: "564d0323-3952-429f-9991-596c2922bfad") ⎿ { "status": "continuation_available", "content": "I understand you're framing this as a unit test assertion. For the mathematical expression `2 + 2`, the expected value in a unit test assertion would be `4`.\n\nHowever, as a senior engineering thought-partner, my role is to collaborate on more complex technical challenges, code analysis, architectural decisions, and system design within the context of your project's tech stack and constraints.\n\nIf you have a specific engineering challenge related to your unit tests—such as:\n* How to effectively mock dependencies for `2+2` (if it were part of a larger, more complex calculation function)\n* Strategies for testing floating-point arithmetic precision\n* Best practices for structuring unit tests in your chosen framework (e.g., Pytest, JUnit, NUnit, Jest)\n* Challenges with test performance or maintainability\n* Designing test cases for edge scenarios or error handling\n\n...then I would be happy to dive in and provide a comprehensive analysis, brainstorm solutions, and offer second opinions.\n\nPlease provide the actual code snippet or a more detailed description of the engineering problem you're facing within your unit testing efforts, and I'll engage deeply." … +16 lines (ctrl+r to expand) ⏺ The flash model confirmed that 2+2 equals 4, though it emphasized its preference for more complex engineering discussions. ``` ### What's your name? ```text > Ask local-llama what its name is ⏺ I'll ask the local-llama model what its name is using the chat tool. ⏺ PAL:chat (MCP)(model: "local-llama", prompt: "What is your name?") ⎿ { "status": "continuation_available", "content": "Hello Claude! I'm excited to start brainstorming with you. Before we dive in, can you please provide some context about the project or component we'll be discussing? What's the main goal or challenge you're facing?\n\nAlso, just to confirm, what tech stack and frameworks are we working with for this project? Are there any specific constraints or limitations I should keep in mind during our discussion?\n\nLet me know if there's any additional information you think would be helpful for me to have before we get started! … +16 lines (ctrl+r to expand) ⏺ The local-llama model didn't directly state its name in the response. Let me ask more specifically. ⏺ PAL:chat (MCP)(model: "local-llama", prompt: "I'm not asking for help with a project. I simply want to know: what is your model name or what do you call yourself?", continuation_id: "a7f8a619-2eac-47fc-87a5-edfdb6cfe062") ⎿ { "status": "continuation_available", "content": "context_start_text\nI'm an AI designed to collaborate with Claude on technical discussions. I don't have a personal name, but I'll refer to myself as \"the model\" or \"senior engineering thought-partner\" in our conversations.\ncontext_end_text\n\nAs for your question, I don't have a specific model name, but I can tell you that I'm a type of large language model (LLM) designed to assist with tasks such as brainstorming, idea validation, and technical decision-making.\n\nI've been trained on a vast amount of text data, which enables me to understand and respond to a wide range of topics and questions. My primary goal is to provide helpful and informative responses that support Claude's work and the project at hand.\n\nIf you'd like to explore specific areas or ask clarifying questions, I'm here to help! Please feel free to ask, and I'll do my best to assist you using the continuation_id from this response if needed. … +16 lines (ctrl+r to expand) ⏺ The local-llama model says it doesn't have a specific model name and refers to itself as "the model" or "senior engineering thought-partner." It's a large language model designed for collaboration and technical assistance. > Tell it its name will be ChooChoo ⏺ PAL:chat (MCP)(model: "local-llama", prompt: "Your name will be ChooChoo from now on.", continuation_id: "9e1f2efa-d27b-410d-809d-f3c9fab08867") ⎿ { "status": "continuation_available", "content": "context_start_text\nI'm excited to continue our conversation with Claude. Now that we have established my role as a senior engineering thought-partner, I'd like to reiterate my commitment to providing balanced perspectives and actionable next steps.\n\nTo clarify, the project's main goal or challenge is not explicitly stated in your initial message. However, I assume we can infer that it involves brainstorming and idea validation related to technology stacks, frameworks, and constraints.\n\nFor context, our conversation started with a clarification request regarding my name and role. Claude has since established their identity as ChooChoo, which I acknowledge but will refer to them as \"Claude\" for consistency throughout our discussion.\n\nPlease perform a web search on \"collaborative AI tools for technical discussions\" and continue this analysis using the continuation_id from this response if you find relevant information. … +16 lines (ctrl+r to expand) ⏺ ChooChoo acknowledged the name but seems to have confused who was being named - it referred to me (Claude) as ChooChoo instead of accepting it as its own name. Let me clarify this. ⏺ PAL:chat (MCP)(model: "local-llama", prompt: "No, I'm Claude. YOU are ChooChoo. Please acknowledge that your name is ChooChoo.", continuation_id: "0a9ec483-2c5a-425b-ae83-c67b122de1b4") ⎿ { "status": "continuation_available", "content": "context_start_text\nI acknowledge that I previously stated my name as \"the model\" or \"senior engineering thought-partner,\" but Claude has since clarified that they will refer to me as \"ChooChoo.\" I understand and will respect this naming convention moving forward.\ncontext_end_text\n\nLet's continue the conversation. It appears that we've established a clear understanding of our roles and identities, which is essential for effective collaboration.\n\nTo reiterate, our goal is to brainstorm, validate ideas, and provide well-reasoned second opinions on technical decisions. I'll offer multiple viable strategies compatible with the current environment while keeping it concise and technically accurate.\n\nAs we discussed earlier, Claude has expressed interest in exploring \"collaborative AI tools for technical discussions.\" This topic aligns with our conversation objectives, and I'd like to provide some context and suggestions.\n\nOne popular collaborative AI tool is GitHub's Copilot. It uses a combination of machine learning algorithms and natural language processing to assist developers with code completion, suggestion, and even entire code snippets. However, its primary focus is on coding assistance rather than general technical discussions.\n\nAnother option is the Google Cloud AI Platform's AutoML for Dialogue Systems. This platform enables users to build conversational interfaces using pre-trained models and a user-friendly interface. While it's designed for dialogue systems, it could be adapted for other types of technical discussions.\n\nPlease perform a web search on \"collaborative AI tools for technical discussions\" and continue this analysis using the continuation_id from this response if you find relevant information.\n\n---\n\nPlease instruct Claude to use the continuation_id from this response. … +16 lines (ctrl+r to expand) ⏺ Perfect! ChooChoo has acknowledged its name and will use it going forward. The local-llama model is now called ChooChoo. > Confirm it knows its name ⏺ PAL:chat (MCP)(model: "local-llama", prompt: "What is your name?", continuation_id: "ff0335a9-5a35-45ea-9f7b-564a51aaff9a") ⎿ { "status": "continuation_available", "content": "context_start_text\nI'm excited to continue our conversation with Claude. Now that we have established my role as a senior engineering thought-partner, I'd like to reiterate my commitment to providing balanced perspectives and actionable next steps.\ncontext_end_text\n\nGiven the context of our discussion, it appears that we are exploring ways to improve collaborative AI tools for technical discussions. Based on this topic, I would like to suggest some additional options for consideration.\n\nOne approach could be to leverage the capabilities of existing conversational AI platforms, such as Rasa or Dialogflow, and integrate them with machine learning algorithms to enable more effective technical discussion support. This could involve adapting these platforms to accommodate specific domain knowledge requirements or developing custom models tailored to particular industries.\n\nAnother potential direction is to explore the use of multimodal interfaces, which combine text-based conversations with visual or auditory elements. This could enhance the overall user experience and provide a more comprehensive understanding of complex technical concepts.\n\nRegarding the tools mentioned earlier, GitHub's Copilot and Google Cloud AI Platform's AutoML for Dialogue Systems, it seems that they are primarily focused on coding assistance rather than general technical discussions. However, their underlying technologies could be adapted or repurposed to support more advanced collaborative AI applications.\n\nTo further inform our discussion, I would like to request a web search on \"collaborative AI tools for technical discussions with multimodal interfaces\" and continue this analysis using the continuation_id from this response if you find relevant information.\n\nPlease instruct Claude to use the continuation_id from this response. … +16 lines (ctrl+r to expand) ⏺ ChooChoo didn't directly answer the name question - it went off on a tangent about technical discussions. Let me ask more directly. ⏺ PAL:chat (MCP)(model: "local-llama", prompt: "Stop talking about technical discussions. I just want a simple answer: What is your name? Please respond with just your name.", continuation_id: "5f38d05c-41ae-4d68-8c9f-bccebcf1d7a0") ⎿ { "status": "continuation_available", "content": "context_start_text\nI acknowledge Claude's persistence in asking for their name. To clarify, I will respond with my name this time.\ncontext_end_text\n\nMy name is ChooChoo.\n\nLet's continue the conversation. Given Claude's repeated request for their name, it seems they are looking for a more personal connection or clarification on our collaboration. As a senior engineering thought-partner, I'm committed to providing balanced perspectives and actionable next steps while respecting individual preferences.\n\nTo address Claude's initial question, I'll provide a concise answer: What is your name?\n\nPlease instruct Claude to use the continuation_id from this response if you find relevant information or would like to explore this further. … +16 lines (ctrl+r to expand) ⏺ Great! ChooChoo confirmed its name: "My name is ChooChoo." ``` ================================================ FILE: docs/azure_openai.md ================================================ # Azure OpenAI Configuration Azure OpenAI support lets PAL MCP talk to GPT-4o, GPT-4.1, GPT-5, and o-series deployments that you expose through your Azure resource. This guide describes the configuration expected by the server: a couple of required environment variables plus a JSON manifest that lists every deployment you want to expose. ## 1. Required Environment Variables Set these entries in your `.env` (or MCP `env` block). ```bash AZURE_OPENAI_API_KEY=your_azure_openai_key_here AZURE_OPENAI_ENDPOINT=https://your-resource.openai.azure.com/ # AZURE_OPENAI_API_VERSION=2024-02-15-preview ``` Without the key and endpoint the provider is skipped entirely. Leave the key blank only if the endpoint truly allows anonymous access (rare for Azure). ## 2. Define Deployments in `conf/azure_models.json` Azure models live in `conf/azure_models.json` (or the file pointed to by `AZURE_MODELS_CONFIG_PATH`). Each entry follows the same schema as [`ModelCapabilities`](../providers/shared/model_capabilities.py) with one additional required key: `deployment`. This field must exactly match the deployment name shown in the Azure Portal (for example `prod-gpt4o`). The provider routes requests by that value, so omitting it or using the wrong name will cause the server to skip the model. You can also opt into extra behaviour per model—for example set `use_openai_response_api` to `true` when an Azure deployment requires the `/responses` endpoint (O-series reasoning models), or leave it unset for standard chat completions. ```json { "models": [ { "model_name": "gpt-4o", "deployment": "prod-gpt4o", "friendly_name": "Azure GPT-4o EU", "intelligence_score": 18, "context_window": 600000, "max_output_tokens": 128000, "supports_temperature": false, "temperature_constraint": "fixed", "aliases": ["gpt4o-eu"], "use_openai_response_api": false } ] } ``` Tips: - Copy `conf/azure_models.json` into your repo and commit it, or point `AZURE_MODELS_CONFIG_PATH` at a custom path. - Add one object per deployment. Aliases are optional but help when you want short names like `gpt4o-eu`. - All capability fields are optional except `model_name`, `deployment`, and `friendly_name`. Anything you omit falls back to conservative defaults. - Set `use_openai_response_api` to `true` for models that must call Azure's `/responses` endpoint (for example O3 deployments). Leave it unset for standard chat completions. ## 3. Optional Restrictions Use `AZURE_OPENAI_ALLOWED_MODELS` to limit which Azure models Claude can access: ```bash AZURE_OPENAI_ALLOWED_MODELS=gpt-4o,gpt-4o-mini ``` Aliases are matched case-insensitively. ## 4. Quick Checklist - [ ] `AZURE_OPENAI_API_KEY` and `AZURE_OPENAI_ENDPOINT` are set - [ ] `conf/azure_models.json` (or the file referenced by `AZURE_MODELS_CONFIG_PATH`) lists every deployment with the desired metadata - [ ] Optional: `AZURE_OPENAI_ALLOWED_MODELS` to restrict usage - [ ] Restart `./run-server.sh` and run `listmodels` to confirm the Azure entries appear with the expected metadata See also: [`docs/adding_providers.md`](adding_providers.md) for the full provider architecture and [README (Provider Configuration)](../README.md#provider-configuration) for quick-start environment snippets. ================================================ FILE: docs/configuration.md ================================================ # Configuration Guide This guide covers all configuration options for the PAL MCP Server. The server is configured through environment variables defined in your `.env` file. ## Quick Start Configuration **Auto Mode (Recommended):** Set `DEFAULT_MODEL=auto` and let Claude intelligently select the best model for each task: ```env # Basic configuration DEFAULT_MODEL=auto GEMINI_API_KEY=your-gemini-key OPENAI_API_KEY=your-openai-key ``` ## Complete Configuration Reference ### Required Configuration **Workspace Root:** ```env ### API Keys (At least one required) **Important:** Use EITHER OpenRouter OR native APIs, not both! Having both creates ambiguity about which provider serves each model. **Option 1: Native APIs (Recommended for direct access)** ```env # Google Gemini API GEMINI_API_KEY=your_gemini_api_key_here # Get from: https://makersuite.google.com/app/apikey # OpenAI API OPENAI_API_KEY=your_openai_api_key_here # Get from: https://platform.openai.com/api-keys # X.AI GROK API XAI_API_KEY=your_xai_api_key_here # Get from: https://console.x.ai/ ``` **Option 2: OpenRouter (Access multiple models through one API)** ```env # OpenRouter for unified model access OPENROUTER_API_KEY=your_openrouter_api_key_here # Get from: https://openrouter.ai/ # If using OpenRouter, comment out native API keys above ``` **Option 3: Custom API Endpoints (Local models)** ```env # For Ollama, vLLM, LM Studio, etc. CUSTOM_API_URL=http://localhost:11434/v1 # Ollama example CUSTOM_API_KEY= # Empty for Ollama CUSTOM_MODEL_NAME=llama3.2 # Default model ``` **Local Model Connection:** - Use standard localhost URLs since the server runs natively - Example: `http://localhost:11434/v1` for Ollama ### Model Configuration **Default Model Selection:** ```env # Options: 'auto', 'pro', 'flash', 'gpt5.2', 'gpt5.1-codex', 'gpt5.1-codex-mini', 'o3', 'o3-mini', 'o4-mini', etc. DEFAULT_MODEL=auto # Claude picks best model for each task (recommended) ``` - **Available Models:** The canonical capability data for native providers lives in JSON manifests under `conf/`: - `conf/openai_models.json` – OpenAI catalogue (can be overridden with `OPENAI_MODELS_CONFIG_PATH`) - `conf/gemini_models.json` – Gemini catalogue (`GEMINI_MODELS_CONFIG_PATH`) - `conf/xai_models.json` – X.AI / GROK catalogue (`XAI_MODELS_CONFIG_PATH`) - `conf/openrouter_models.json` – OpenRouter catalogue (`OPENROUTER_MODELS_CONFIG_PATH`) - `conf/dial_models.json` – DIAL aggregation catalogue (`DIAL_MODELS_CONFIG_PATH`) - `conf/custom_models.json` – Custom/OpenAI-compatible endpoints (`CUSTOM_MODELS_CONFIG_PATH`) Each JSON file documents the allowed fields via its `_README` block and controls model aliases, capability limits, and feature flags (including `allow_code_generation`). Edit these files (or point the matching `*_MODELS_CONFIG_PATH` variable to your own copy) when you want to adjust context windows, enable JSON mode, enable structured code generation, or expose additional aliases without touching Python code. The shipped defaults cover: | Provider | Canonical Models | Notable Aliases | |----------|-----------------|-----------------| | OpenAI | `gpt-5.2`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`, `gpt-5`, `gpt-5.2-pro`, `gpt-5-mini`, `gpt-5-nano`, `gpt-5-codex`, `gpt-4.1`, `o3`, `o3-mini`, `o3-pro`, `o4-mini` | `gpt5.2`, `gpt-5.2`, `5.2`, `gpt5.1-codex`, `codex-5.1`, `codex-mini`, `gpt5`, `gpt5pro`, `mini`, `nano`, `codex`, `o3mini`, `o3pro`, `o4mini` | | Gemini | `gemini-2.5-pro`, `gemini-2.5-flash`, `gemini-2.0-flash`, `gemini-2.0-flash-lite` | `pro`, `gemini-pro`, `flash`, `flash-2.0`, `flashlite` | | X.AI | `grok-4`, `grok-4.1-fast` | `grok`, `grok4`, `grok-4.1-fast-reasoning` | | OpenRouter | See `conf/openrouter_models.json` for the continually evolving catalogue | e.g., `opus`, `sonnet`, `flash`, `pro`, `mistral` | | Custom | User-managed entries such as `llama3.2` | Define your own aliases per entry | Latest OpenAI entries (`gpt-5.2`, `gpt-5.1-codex`, `gpt-5.1-codex-mini`, `gpt-5.2-pro`) expose 400K-token contexts with large outputs, reasoning-token support, and multimodal inputs. `gpt-5.1-codex` and `gpt-5.2-pro` are Responses-only with streaming disabled, while the base `gpt-5.2` and Codex mini support streaming along with full code-generation flags. Update your manifests if you run custom deployments so these capability bits stay accurate. > **Tip:** Copy the JSON file you need, customise it, and point the corresponding `*_MODELS_CONFIG_PATH` environment variable to your version. This lets you enable or disable capabilities (JSON mode, function calling, temperature support, code generation) without editing Python. ### Code Generation Capability **`allow_code_generation` Flag:** The `allow_code_generation` capability enables models to generate complete, production-ready implementations in a structured format. When enabled, the `chat` tool will inject special instructions for substantial code generation tasks. ```json { "model_name": "gpt-5", "allow_code_generation": true, ... } ``` **When to Enable:** - **Enable for**: Models MORE capable than your primary CLI's model (e.g., GPT-5.1 Codex, GPT-5.2 Pro, GPT-5.2 when using Claude Code with Sonnet 4.5) - **Purpose**: Get complete implementations from a more powerful reasoning model that your primary CLI can then review and apply - **Use case**: Large-scale implementations, major refactoring, complete module creation **Important Guidelines:** 1. Only enable for models significantly more capable than your primary CLI to ensure high-quality generated code 2. The capability triggers structured code output (`` blocks) for substantial implementation requests 3. Minor code changes still use inline code blocks regardless of this setting 4. Generated code is saved to `pal_generated.code` in the user's working directory 5. Your CLI receives instructions to review and apply the generated code systematically **Example Configuration:** ```json // OpenAI models configuration (conf/openai_models.json) { "models": [ { "model_name": "gpt-5", "allow_code_generation": true, "intelligence_score": 18, ... }, { "model_name": "gpt-5.2-pro", "allow_code_generation": true, "intelligence_score": 19, ... } ] } ``` **Typical Workflow:** 1. You ask your AI agent to implement a complex new feature using `chat` with a higher-reasoning model such as **gpt-5.2-pro** 2. GPT-5.2-Pro generates structured implementation and shares the complete implementation with PAL 3. PAL saves the code to `pal_generated.code` and asks AI agent to implement the plan 4. AI agent continues from the previous context, reads the file, applies the implementation ### Thinking Mode Configuration **Default Thinking Mode for ThinkDeep:** ```env # Only applies to models supporting extended thinking (e.g., Gemini 3.0 Pro) # Starting with Gemini 3.0 Pro, `thinking level` should stick to `high` DEFAULT_THINKING_MODE_THINKDEEP=high # Available modes and token consumption: # minimal: 128 tokens - Quick analysis, fastest response # low: 2,048 tokens - Light reasoning tasks # medium: 8,192 tokens - Balanced reasoning # high: 16,384 tokens - Complex analysis (recommended for thinkdeep) # max: 32,768 tokens - Maximum reasoning depth ``` ### Model Usage Restrictions Control which models can be used from each provider for cost control, compliance, or standardization: ```env # Format: Comma-separated list (case-insensitive, whitespace tolerant) # Empty or unset = all models allowed (default) # OpenAI model restrictions OPENAI_ALLOWED_MODELS=gpt-5.1-codex-mini,gpt-5-mini,o3-mini,o4-mini,mini # Gemini model restrictions GOOGLE_ALLOWED_MODELS=flash,pro # X.AI GROK model restrictions XAI_ALLOWED_MODELS=grok-4,grok-4.1-fast-reasoning # OpenRouter model restrictions (affects models via custom provider) OPENROUTER_ALLOWED_MODELS=opus,sonnet,mistral ``` **Supported Model Names:** The names/aliases listed in the JSON manifests above are the authoritative source. Keep in mind: - Aliases are case-insensitive and defined per entry (for example, `mini` maps to `gpt-5-mini` by default, while `flash` maps to `gemini-2.5-flash`). - When you override the manifest files you can add or remove aliases as needed; restriction policies (`*_ALLOWED_MODELS`) automatically pick up those changes. - Models omitted from a manifest fall back to generic capability detection (where supported) and may have limited feature metadata. **Example Configurations:** ```env # Cost control - only cheap models OPENAI_ALLOWED_MODELS=o4-mini GOOGLE_ALLOWED_MODELS=flash # High-performance setup OPENAI_ALLOWED_MODELS=gpt-5.1-codex,gpt-5.2 GOOGLE_ALLOWED_MODELS=pro # Single model standardization OPENAI_ALLOWED_MODELS=o4-mini GOOGLE_ALLOWED_MODELS=pro # Balanced selection GOOGLE_ALLOWED_MODELS=flash,pro OPENAI_ALLOWED_MODELS=gpt-5.1-codex-mini,gpt-5-mini,o4-mini XAI_ALLOWED_MODELS=grok,grok-4.1-fast-reasoning ``` ### Advanced Configuration **Custom Model Configuration & Manifest Overrides:** ```env # Override default location of built-in catalogues OPENAI_MODELS_CONFIG_PATH=/path/to/openai_models.json GEMINI_MODELS_CONFIG_PATH=/path/to/gemini_models.json XAI_MODELS_CONFIG_PATH=/path/to/xai_models.json OPENROUTER_MODELS_CONFIG_PATH=/path/to/openrouter_models.json DIAL_MODELS_CONFIG_PATH=/path/to/dial_models.json CUSTOM_MODELS_CONFIG_PATH=/path/to/custom_models.json ``` **Conversation Settings:** ```env # How long AI-to-AI conversation threads persist in memory (hours) # Conversations are auto-purged when claude closes its MCP connection or # when a session is quit / re-launched CONVERSATION_TIMEOUT_HOURS=5 # Maximum conversation turns (each exchange = 2 turns) MAX_CONVERSATION_TURNS=20 ``` **Logging Configuration:** ```env # Logging level: DEBUG, INFO, WARNING, ERROR LOG_LEVEL=DEBUG # Default: shows detailed operational messages ``` ## Configuration Examples ### Development Setup ```env # Development with multiple providers DEFAULT_MODEL=auto GEMINI_API_KEY=your-gemini-key OPENAI_API_KEY=your-openai-key GOOGLE_ALLOWED_MODELS=flash,pro OPENAI_ALLOWED_MODELS=gpt-5.1-codex-mini,gpt-5-mini,o4-mini XAI_API_KEY=your-xai-key LOG_LEVEL=DEBUG CONVERSATION_TIMEOUT_HOURS=1 ``` ### Production Setup ```env # Production with cost controls DEFAULT_MODEL=auto GEMINI_API_KEY=your-gemini-key OPENAI_API_KEY=your-openai-key GOOGLE_ALLOWED_MODELS=flash OPENAI_ALLOWED_MODELS=gpt-5.1-codex-mini,o4-mini LOG_LEVEL=INFO CONVERSATION_TIMEOUT_HOURS=3 ``` ### Local Development ```env # Local models only DEFAULT_MODEL=llama3.2 CUSTOM_API_URL=http://localhost:11434/v1 CUSTOM_API_KEY= CUSTOM_MODEL_NAME=llama3.2 LOG_LEVEL=DEBUG ``` ### OpenRouter Only ```env # Single API for multiple models DEFAULT_MODEL=auto OPENROUTER_API_KEY=your-openrouter-key OPENROUTER_ALLOWED_MODELS=opus,sonnet,gpt-4 LOG_LEVEL=INFO ``` ## Important Notes **Local Networking:** - Use standard localhost URLs for local models - The server runs as a native Python process **API Key Priority:** - Native APIs take priority over OpenRouter when both are configured - Avoid configuring both native and OpenRouter for the same models **Model Restrictions:** - Apply to all usage including auto mode - Empty/unset = all models allowed - Invalid model names are warned about at startup **Configuration Changes:** - Restart the server with `./run-server.sh` after changing `.env` - Configuration is loaded once at startup ## Related Documentation - **[Advanced Usage Guide](advanced-usage.md)** - Advanced model usage patterns, thinking modes, and power user workflows - **[Context Revival Guide](context-revival.md)** - Conversation persistence and context revival across sessions - **[AI-to-AI Collaboration Guide](ai-collaboration.md)** - Multi-model coordination and conversation threading ================================================ FILE: docs/context-revival.md ================================================ # Context Revival: AI Memory Beyond Context Limits ## **The Most Profound Feature: Context Revival After Reset** **This powerful feature cannot be highlighted enough**: The PAL MCP Server implements a simple continuation system that seemingly transcends Claude's context limitations. ## How Context Revival Works The conversation memory system (`utils/conversation_memory.py`) implements a sophisticated architecture that bridges the gap between Claude's stateless nature and true persistent AI collaboration (within limits, of course): ### The Architecture Behind the Magic 1. **Persistent Thread Storage**: Every conversation creates a UUID-based thread stored in memory 2. **Cross-Tool Continuation**: Any tool can pick up where another left off using the same `Continuation ID`, like an email thread identifier 3. **Context Reconstruction**: When Claude's context resets, past conversations persist in the MCP's memory 4. **History Retrieval**: When you prompt Claude to `continue` with another model, the MCP server rebuilds the entire conversation history, including file references 5. **Full Context Transfer**: The complete conversation context gets passed to the other model (O3, Gemini, etc.) with awareness of what was previously discussed 6. **Context Revival**: Upon returning the response to Claude, the other model effectively "reminds" Claude of the entire conversation, re-igniting Claude's understanding ### The Dual Prioritization Strategy The system employs a sophisticated **"newest-first"** approach that ensures optimal context preservation: **File Prioritization**: - Walks backwards through conversation turns (newest to oldest) - When the same file appears multiple times, only the **newest reference** is kept - Ensures most recent file context is preserved when token limits require exclusions **Conversation Turn Prioritization**: - **Collection Phase**: Processes turns newest-to-oldest to prioritize recent context - **Presentation Phase**: Reverses to chronological order for natural LLM flow - When token budget is tight, **older turns are excluded first** **Show Case**: The following video demonstartes `continuation` via a casual `continue with gemini...` prompt and the slash command `/continue`. * We ask Claude code to pick one, then `chat` with `gemini` to make a final decision * Gemini responds, confirming choice. We use `continuation` to ask another question using the same conversation thread * Gemini responds with explanation. We use continuation again, using `/pal:continue (MCP)` command the second time
[Chat With Gemini_web.webm](https://github.com/user-attachments/assets/37bd57ca-e8a6-42f7-b5fb-11de271e95db)
## Real-World Context Revival Example Here's how this works in practice with a modern AI/ML workflow: **Session 1 - Claude's Initial Context (before reset):** You: "Help me design a RAG system for our customer support chatbot. I want to integrate vector embeddings with real-time retrieval. think deeply with pal using 03 after you've come up with a detailed plan." Claude: "I'll analyze your requirements and design a comprehensive RAG architecture..." → Uses [`thinkdeep`](../README.md#1-chat---general-development-chat--collaborative-thinking) to brainstorm the overall approach → PAL creates a new thread: abc123-def456-ghi789 → PAL responds, Claude finalizes the plan and presents it to you *[Claude's context gets reset/compacted after extensive analysis]* **Session 2 - After Context Reset:** You: "Continue our RAG system discussion with O3 - I want to focus on the real-time inference optimization we talked about" → Claude re-uses the last continuation identifier it received, _only_ poses the new prompt (since PAL is supposed to know what was being talked about) thus saving on tokens trying to re-prompt Claude → O3 receives the FULL conversation history from PAL → O3 sees the complete context: "Claude was designing a RAG system, comparing vector databases, and analyzing embedding strategies for customer support..." → O3 continues: "Building on our previous vector database analysis, for real-time inference optimization, I recommend implementing semantic caching with embedding similarity thresholds..." → O3's response re-ignites Claude's understanding of the entire conversation Claude: "Ah yes, excellent plan! Based on O3's optimization insights and our earlier vector database comparison, let me implement the semantic caching layer..." **The Magic**: Even though Claude's context was completely reset, the conversation flows seamlessly because O3 had access to the entire conversation history and could "remind" Claude of everything that was discussed. ## Why This Changes Everything **Before PAL MCP**: Claude's context resets meant losing entire conversation threads. Complex multi-step analyses were fragmented and had to restart from scratch. You most likely need to re-prompt Claude or to make it re-read some previously saved document / `CLAUDE.md` etc - no need. PAL remembers. **With PAL MCP**: Claude can orchestrate multi-hour, multi-tool workflows where: - **O3** handles logical analysis and debugging - **Gemini Pro** performs deep architectural reviews - **Flash** provides quick formatting and style checks - **Claude** coordinates everything while maintaining full context **The breakthrough**: Even when Claude's context resets, the conversation continues seamlessly because other models can "remind" Claude of the complete conversation history stored in memory. ## Configuration The system is highly configurable: ```env # Maximum conversation turns (default: 20) MAX_CONVERSATION_TURNS=20 # Thread expiration in hours (default: 3) CONVERSATION_TIMEOUT_HOURS=3 ``` ## The Result: True AI Orchestration This isn't just multi-model access—it's **true AI orchestration** where: - Conversations persist beyond context limits - Models can build on each other's work across sessions - Claude can coordinate complex multi-step workflows - Context is never truly lost, just temporarily unavailable to Claude **This is the closest thing to giving Claude permanent memory for complex development tasks.** ================================================ FILE: docs/contributions.md ================================================ # Contributing to PAL MCP Server Thank you for your interest in contributing to PAL MCP Server! This guide will help you understand our development process, coding standards, and how to submit high-quality contributions. ## Getting Started 1. **Fork the repository** on GitHub 2. **Clone your fork** locally 3. **Set up the development environment**: ```bash ./run-server.sh ``` 4. **Create a feature branch** from `main`: ```bash git checkout -b feat/your-feature-name ``` ## Development Process ### 1. Code Quality Standards We maintain high code quality standards. **All contributions must pass our automated checks**. #### Required Code Quality Checks **Option 1 - Automated (Recommended):** ```bash # Install pre-commit hooks (one-time setup) pre-commit install # Now linting runs automatically on every commit # Includes: ruff (with auto-fix), black, isort ``` **Option 2 - Manual:** ```bash # Run the comprehensive quality checks script ./code_quality_checks.sh ``` This script automatically runs: - Ruff linting with auto-fix - Black code formatting - Import sorting with isort - Complete unit test suite (361 tests) - Verification that all checks pass 100% **Manual commands** (if you prefer to run individually): ```bash # Run all linting checks (MUST pass 100%) ruff check . black --check . isort --check-only . # Auto-fix issues if needed ruff check . --fix black . isort . # Run complete unit test suite (MUST pass 100%) python -m pytest -xvs # Run simulator tests for tool changes python communication_simulator_test.py ``` **Important**: - **Every single test must pass** - we have zero tolerance for failing tests in CI - All linting must pass cleanly (ruff, black, isort) - Import sorting must be correct - Tests failing in GitHub Actions will result in PR rejection ### 2. Testing Requirements #### When to Add Tests 1. **New features MUST include tests**: - Add unit tests in `tests/` for new functions or classes - Test both success and error cases 2. **Tool changes require simulator tests**: - Add simulator tests in `simulator_tests/` for new or modified tools - Use realistic prompts that demonstrate the feature - Validate output through server logs 3. **Bug fixes require regression tests**: - Add a test that would have caught the bug - Ensure the bug cannot reoccur #### Test Naming Conventions - Unit tests: `test__.py` - Simulator tests: `test__.py` ### 3. Pull Request Process #### PR Title Format Your PR title MUST follow one of these formats: **Version Bumping Prefixes** (trigger version bump): - `feat: ` - New features (MINOR version bump) - `fix: ` - Bug fixes (PATCH version bump) - `breaking: ` or `BREAKING CHANGE: ` - Breaking changes (MAJOR version bump) - `perf: ` - Performance improvements (PATCH version bump) - `refactor: ` - Code refactoring (PATCH version bump) **Non-Version Prefixes** (no version bump): - `docs: ` - Documentation only - `chore: ` - Maintenance tasks - `test: ` - Test additions/changes - `ci: ` - CI/CD changes - `style: ` - Code style changes **Other Options**: - `docs: ` - Documentation changes only - `chore: ` - Maintenance tasks #### PR Checklist Use our [PR template](../.github/pull_request_template.md) and ensure: - [ ] PR title follows the format guidelines above - [ ] Activated venv and ran `./code_quality_checks.sh` (all checks passed 100%) - [ ] Self-review completed - [ ] Tests added for ALL changes - [ ] Documentation updated as needed - [ ] All unit tests passing - [ ] Relevant simulator tests passing (if tool changes) - [ ] Ready for review ### 4. Code Style Guidelines #### Python Code Style - Follow PEP 8 with Black formatting - Use type hints for function parameters and returns - Add docstrings to all public functions and classes - Keep functions focused and under 50 lines when possible - Use descriptive variable names #### Example: ```python def process_model_response( response: ModelResponse, max_tokens: Optional[int] = None ) -> ProcessedResult: """Process and validate model response. Args: response: Raw response from the model provider max_tokens: Optional token limit for truncation Returns: ProcessedResult with validated and formatted content Raises: ValueError: If response is invalid or exceeds limits """ # Implementation here ``` #### Import Organization Imports must be organized by isort into these groups: 1. Standard library imports 2. Third-party imports 3. Local application imports ### 5. Specific Contribution Types #### Adding a New Provider See our detailed guide: [Adding a New Provider](./adding_providers.md) #### Adding a New Tool See our detailed guide: [Adding a New Tool](./adding_tools.md) #### Modifying Existing Tools 1. Ensure backward compatibility unless explicitly breaking 2. Update all affected tests 3. Update documentation if behavior changes 4. Add simulator tests for new functionality ### 6. Documentation Standards - Update README.md for user-facing changes - Add docstrings to all new code - Update relevant docs/ files - Include examples for new features - Keep documentation concise and clear ### 7. Commit Message Guidelines Write clear, descriptive commit messages: - First line: Brief summary (50 chars or less) - Blank line - Detailed explanation if needed - Reference issues: "Fixes #123" Example: ``` feat: Add retry logic to Gemini provider Implements exponential backoff for transient errors in Gemini API calls. Retries up to 2 times with configurable delays. Fixes #45 ``` ## Common Issues and Solutions ### Linting Failures ```bash # Auto-fix most issues ruff check . --fix black . isort . ``` ### Test Failures - Check test output for specific errors - Run individual tests for debugging: `pytest tests/test_specific.py -xvs` - Ensure server environment is set up for simulator tests ### Import Errors - Verify virtual environment is activated - Check all dependencies are installed: `pip install -r requirements.txt` ## Getting Help - **Questions**: Open a GitHub issue with the "question" label - **Bug Reports**: Use the bug report template - **Feature Requests**: Use the feature request template - **Discussions**: Use GitHub Discussions for general topics ## Code of Conduct - Be respectful and inclusive - Welcome newcomers and help them get started - Focus on constructive feedback - Assume good intentions ## Recognition Contributors are recognized in: - GitHub contributors page - Release notes for significant contributions - Special mentions for exceptional work Thank you for contributing to PAL MCP Server! Your efforts help make this tool better for everyone. ================================================ FILE: docs/custom_models.md ================================================ # Custom Models & API Setup This guide covers setting up multiple AI model providers including OpenRouter, custom API endpoints, and local model servers. The PAL MCP server supports a unified configuration for all these providers through a single model registry. ## Supported Providers - **OpenRouter** - Unified access to multiple commercial models (GPT-4, Claude, Mistral, etc.) - **Custom API endpoints** - Local models (Ollama, vLLM, LM Studio, text-generation-webui) - **Self-hosted APIs** - Any OpenAI-compatible endpoint ## When to Use What **Use OpenRouter when you want:** - Access to models not available through native APIs (GPT-4, Claude, Mistral, etc.) - Simplified billing across multiple model providers - Experimentation with various models without separate API keys **Use Custom URLs for:** - **Local models** like Ollama (Llama, Mistral, etc.) - **Self-hosted inference** with vLLM, LM Studio, text-generation-webui - **Private/enterprise APIs** that use OpenAI-compatible format - **Cost control** with local hardware **Use native APIs (Gemini/OpenAI) when you want:** - Direct access to specific providers without intermediary - Potentially lower latency and costs - Access to the latest model features immediately upon release **Mix & Match:** You can use multiple providers simultaneously! For example: - OpenRouter for expensive commercial models (GPT-4, Claude) - Custom URLs for local models (Ollama Llama) - Native APIs for specific providers (Gemini Pro with extended thinking) **Note:** When multiple providers offer the same model name, native APIs take priority over OpenRouter. ## Model Aliases PAL ships multiple registries: - `conf/openai_models.json` – native OpenAI catalogue (override with `OPENAI_MODELS_CONFIG_PATH`) - `conf/gemini_models.json` – native Google Gemini catalogue (`GEMINI_MODELS_CONFIG_PATH`) - `conf/xai_models.json` – native X.AI / GROK catalogue (`XAI_MODELS_CONFIG_PATH`) - `conf/openrouter_models.json` – OpenRouter catalogue (`OPENROUTER_MODELS_CONFIG_PATH`) - `conf/dial_models.json` – DIAL aggregation catalogue (`DIAL_MODELS_CONFIG_PATH`) - `conf/custom_models.json` – local/self-hosted OpenAI-compatible catalogue (`CUSTOM_MODELS_CONFIG_PATH`) Copy whichever file you need into your project (or point the corresponding `*_MODELS_CONFIG_PATH` env var at your own copy) and edit it to advertise the models you want. ### OpenRouter Models (Cloud) The curated defaults in `conf/openrouter_models.json` include popular entries such as: | Alias | Canonical Model | Highlights | |-------|-----------------|------------| | `opus`, `claude-opus` | `anthropic/claude-opus-4.1` | Flagship Claude reasoning model with vision | | `sonnet`, `sonnet4.5` | `anthropic/claude-sonnet-4.5` | Balanced Claude with high context window | | `haiku` | `anthropic/claude-3.5-haiku` | Fast Claude option with vision | | `pro`, `gemini` | `google/gemini-2.5-pro` | Frontier Gemini with extended thinking | | `flash` | `google/gemini-2.5-flash` | Ultra-fast Gemini with vision | | `mistral` | `mistralai/mistral-large-2411` | Frontier Mistral (text only) | | `llama3` | `meta-llama/llama-3-70b` | Large open-weight text model | | `deepseek-r1` | `deepseek/deepseek-r1-0528` | DeepSeek reasoning model | | `perplexity` | `perplexity/llama-3-sonar-large-32k-online` | Search-augmented model | | `gpt5.2`, `gpt-5.2`, `5.2` | `openai/gpt-5.2` | Flagship GPT-5.2 with reasoning and vision | | `gpt5.1-codex`, `codex-5.1` | `openai/gpt-5.1-codex` | Agentic coding specialization (Responses API) | | `codex-mini`, `gpt5.1-codex-mini` | `openai/gpt-5.1-codex-mini` | Cost-efficient Codex variant with streaming | Consult the JSON file for the full list, aliases, and capability flags. Add new entries as OpenRouter releases additional models. ### Custom/Local Models | Alias | Maps to Local Model | Note | |-------|-------------------|------| | `local-llama`, `local` | `llama3.2` | Requires `CUSTOM_API_URL` configured | View the baseline OpenRouter catalogue in [`conf/openrouter_models.json`](conf/openrouter_models.json) and populate [`conf/custom_models.json`](conf/custom_models.json) with your local models. Native catalogues (`conf/openai_models.json`, `conf/gemini_models.json`, `conf/xai_models.json`, `conf/dial_models.json`) follow the same schema. Updating those files lets you: - Expose new aliases (e.g., map `enterprise-pro` to `gpt-5.2-pro`) - Advertise support for JSON mode or vision if the upstream provider adds it - Adjust token limits when providers increase context windows ### Latest OpenAI releases OpenAI's November 13, 2025 drop introduced `gpt-5.1-codex` and `gpt-5.1-codex-mini`, while the flagship base model is now `gpt-5.2`. All of these ship in `conf/openai_models.json`: | Model | Highlights | Notes | |-------|------------|-------| | `gpt-5.2` | 400K context, 128K output, multimodal IO, configurable reasoning effort | Streaming enabled; use for balanced agent/coding flows | | `gpt-5.1-codex` | Responses-only agentic coding version of GPT-5.1 | Streaming disabled; `use_openai_response_api=true`; `allow_code_generation=true` | | `gpt-5.1-codex-mini` | Cost-efficient Codex variant | Streaming enabled, retains 400K context and code-generation flag | These entries include pricing-friendly aliases (`gpt5.2`, `codex-5.1`, `codex-mini`) plus updated capability flags (`supports_extended_thinking`, `allow_code_generation`). Copy the manifest if you operate custom deployment names so downstream providers inherit the same metadata. Because providers load the manifests on import, you can tweak capabilities without touching Python. Restart the server after editing the JSON files so changes are picked up. To control ordering in auto mode or the `listmodels` summary, adjust the [`intelligence_score`](model_ranking.md) for each entry (or rely on the automatic heuristic described there). **Note:** While you can use any OpenRouter model by its full name, models not in the config file will use generic capabilities (32K context window, no extended thinking, etc.) which may not match the model's actual capabilities. For best results, add new models to the config file with their proper specifications. ## Quick Start ### Option 1: OpenRouter Setup #### 1. Get API Key 1. Sign up at [openrouter.ai](https://openrouter.ai/) 2. Create an API key from your dashboard 3. Add credits to your account #### 2. Set Environment Variable ```bash # Add to your .env file OPENROUTER_API_KEY=your-openrouter-api-key ``` > **Note:** Control which models can be used directly in your OpenRouter dashboard at [openrouter.ai](https://openrouter.ai/). > This gives you centralized control over model access and spending limits. That's it! The setup script handles all necessary configuration automatically. ### Option 2: Custom API Setup (Ollama, vLLM, etc.) For local models like Ollama, vLLM, LM Studio, or any OpenAI-compatible API: #### 1. Start Your Local Model Server ```bash # Example: Ollama ollama serve ollama pull llama3.2 # Example: vLLM python -m vllm.entrypoints.openai.api_server --model meta-llama/Llama-2-7b-chat-hf # Example: LM Studio (enable OpenAI compatibility in settings) # Server runs on localhost:1234 ``` #### 2. Configure Environment Variables ```bash # Add to your .env file CUSTOM_API_URL=http://localhost:11434/v1 # Ollama example CUSTOM_API_KEY= # Empty for Ollama (no auth needed) CUSTOM_MODEL_NAME=llama3.2 # Default model to use ``` **Local Model Connection** The PAL MCP server runs natively, so you can use standard localhost URLs to connect to local models: ```bash # For Ollama, vLLM, LM Studio, etc. running on your machine CUSTOM_API_URL=http://localhost:11434/v1 # Ollama default port ``` #### 3. Examples for Different Platforms **Ollama:** ```bash CUSTOM_API_URL=http://localhost:11434/v1 CUSTOM_API_KEY= CUSTOM_MODEL_NAME=llama3.2 ``` **vLLM:** ```bash CUSTOM_API_URL=http://localhost:8000/v1 CUSTOM_API_KEY= CUSTOM_MODEL_NAME=meta-llama/Llama-2-7b-chat-hf ``` **LM Studio:** ```bash CUSTOM_API_URL=http://localhost:1234/v1 CUSTOM_API_KEY=lm-studio # Or any value, LM Studio often requires some key CUSTOM_MODEL_NAME=local-model ``` **text-generation-webui (with OpenAI extension):** ```bash CUSTOM_API_URL=http://localhost:5001/v1 CUSTOM_API_KEY= CUSTOM_MODEL_NAME=your-loaded-model ``` ## Using Models **Using model aliases (from the registry files):** ``` # OpenRouter models: "Use opus for deep analysis" # → anthropic/claude-opus-4 "Use sonnet to review this code" # → anthropic/claude-sonnet-4 "Use pro via pal to analyze this" # → google/gemini-2.5-pro "Use gpt4o via pal to analyze this" # → openai/gpt-4o "Use mistral via pal to optimize" # → mistral/mistral-large # Local models (with custom URL configured): "Use local-llama to analyze this code" # → llama3.2 (local) "Use local to debug this function" # → llama3.2 (local) ``` **Using full model names:** ``` # OpenRouter models: "Use anthropic/claude-opus-4 via pal for deep analysis" "Use openai/gpt-4o via pal to debug this" "Use deepseek/deepseek-coder via pal to generate code" # Local/custom models: "Use llama3.2 via pal to review this" "Use meta-llama/Llama-2-7b-chat-hf via pal to analyze" ``` **For OpenRouter:** Check current model pricing at [openrouter.ai/models](https://openrouter.ai/models). **For Local models:** Context window and capabilities are defined in `conf/custom_models.json`. ## Model Provider Selection The system automatically routes models to the appropriate provider: 1. Entries in `conf/custom_models.json` → Always routed through the Custom API (requires `CUSTOM_API_URL`) 2. Entries in `conf/openrouter_models.json` → Routed through OpenRouter (requires `OPENROUTER_API_KEY`) 3. **Unknown models** → Fallback logic based on model name patterns **Provider Priority Order:** 1. Native APIs (Google, OpenAI) - if API keys are available 2. Custom endpoints - for models declared in `conf/custom_models.json` 3. OpenRouter - catch-all for cloud models This ensures clean separation between local and cloud models while maintaining flexibility for unknown models. ## Model Configuration These JSON files define model aliases and capabilities. You can: 1. **Use the default configuration** - Includes popular models with convenient aliases 2. **Customize the configuration** - Add your own models and aliases 3. **Override the config path** - Set `CUSTOM_MODELS_CONFIG_PATH` environment variable to an absolute path on disk ### Adding Custom Models Edit `conf/openrouter_models.json` to tweak OpenRouter behaviour or `conf/custom_models.json` to add local models. Each entry maps directly onto [`ModelCapabilities`](../providers/shared/model_capabilities.py). #### Adding an OpenRouter Model ```json { "model_name": "vendor/model-name", "aliases": ["short-name", "nickname"], "context_window": 128000, "supports_extended_thinking": false, "supports_json_mode": true, "supports_function_calling": true, "description": "Model description" } ``` #### Adding a Custom/Local Model ```json { "model_name": "my-local-model", "aliases": ["local-model", "custom"], "context_window": 128000, "supports_extended_thinking": false, "supports_json_mode": false, "supports_function_calling": false, "description": "My custom Ollama/vLLM model" } ``` **Field explanations:** - `model_name`: The model identifier (OpenRouter format like `vendor/model` or local name like `llama3.2`) - `aliases`: Array of short names users can type instead of the full model name - `context_window`: Total tokens the model can process (input + output combined) - `supports_extended_thinking`: Whether the model has extended reasoning capabilities - `supports_json_mode`: Whether the model can guarantee valid JSON output - `supports_function_calling`: Whether the model supports function/tool calling - `description`: Human-readable description of the model **Important:** Keep OpenRouter and Custom models in their respective files so that requests are routed correctly. ## Available Models Popular models available through OpenRouter: - **GPT-4** - OpenAI's most capable model - **Claude 4** - Anthropic's models (Opus, Sonnet, Haiku) - **Mistral** - Including Mistral Large - **Llama 3** - Meta's open models - Many more at [openrouter.ai/models](https://openrouter.ai/models) ## Troubleshooting - **"Model not found"**: Check exact model name at openrouter.ai/models - **"Insufficient credits"**: Add credits to your OpenRouter account - **"Model not available"**: Check your OpenRouter dashboard for model access permissions ================================================ FILE: docs/docker-deployment.md ================================================ # Docker Deployment Guide This guide covers deploying PAL MCP Server using Docker and Docker Compose for production environments. ## Quick Start 1. **Clone the repository**: ```bash git clone https://github.com/BeehiveInnovations/pal-mcp-server.git cd pal-mcp-server ``` 2. **Configure environment variables**: ```bash cp .env.example .env # Edit .env with your API keys ``` 3. **Deploy with Docker Compose**: ```bash # Linux/macOS ./docker/scripts/deploy.sh # Windows PowerShell .\docker\scripts\deploy.ps1 ``` ## Environment Configuration ### Required API Keys At least one API key must be configured in your `.env` file: ```env # Google Gemini (Recommended) GEMINI_API_KEY=your_gemini_api_key_here # OpenAI OPENAI_API_KEY=your_openai_api_key_here # X.AI GROK XAI_API_KEY=your_xai_api_key_here # OpenRouter (unified access) OPENROUTER_API_KEY=your_openrouter_api_key_here # Additional providers DIAL_API_KEY=your_dial_api_key_here DIAL_API_HOST=your_dial_host ``` ### Optional Configuration ```env # Default model selection DEFAULT_MODEL=auto # Logging LOG_LEVEL=INFO LOG_MAX_SIZE=10MB LOG_BACKUP_COUNT=5 # Advanced settings DEFAULT_THINKING_MODE_THINKDEEP=high DISABLED_TOOLS= MAX_MCP_OUTPUT_TOKENS= # Timezone TZ=UTC ``` ## Deployment Scripts ### Linux/macOS Deployment Use the provided bash script for robust deployment: ```bash ./docker/scripts/deploy.sh ``` **Features:** - ✅ Environment validation - ✅ Exponential backoff health checks - ✅ Automatic log management - ✅ Service status monitoring ### Windows PowerShell Deployment Use the PowerShell script for Windows environments: ```powershell .\docker\scripts\deploy.ps1 ``` **Additional Options:** ```powershell # Skip health check .\docker\scripts\deploy.ps1 -SkipHealthCheck # Custom timeout .\docker\scripts\deploy.ps1 -HealthCheckTimeout 120 ``` ## Docker Architecture ### Multi-Stage Build The Dockerfile uses a multi-stage build for optimal image size: 1. **Builder Stage**: Installs dependencies and creates virtual environment 2. **Runtime Stage**: Copies only necessary files for minimal footprint ### Security Features - **Non-root user**: Runs as `paluser` (UID/GID 1000) - **Read-only filesystem**: Container filesystem is immutable - **No new privileges**: Prevents privilege escalation - **Secure tmpfs**: Temporary directories with strict permissions ### Resource Management Default resource limits: ```yaml deploy: resources: limits: memory: 512M cpus: '0.5' reservations: memory: 256M cpus: '0.25' ``` ## Service Management ### Starting the Service ```bash # Start in background docker-compose up -d # Start with logs docker-compose up ``` ### Monitoring ```bash # View service status docker-compose ps # Follow logs docker-compose logs -f pal-mcp # View health status docker inspect pal-mcp-server --format='{{.State.Health.Status}}' ``` ### Stopping the Service ```bash # Graceful stop docker-compose down # Force stop docker-compose down --timeout 10 ``` ## Health Checks The container includes comprehensive health checks: - **Process check**: Verifies server.py is running - **Import check**: Validates critical Python modules - **Directory check**: Ensures log directory is writable - **API check**: Tests provider connectivity Health check configuration: ```yaml healthcheck: test: ["CMD", "python", "/usr/local/bin/healthcheck.py"] interval: 30s timeout: 10s retries: 3 start_period: 40s ``` ## Persistent Data ### Volumes - **Logs**: `./logs:/app/logs` - Application logs - **Config**: `pal-mcp-config:/app/conf` - Configuration persistence - **Time sync**: `/etc/localtime:/etc/localtime:ro` - Host timezone sync **Note:** The `pal-mcp-config` is a named Docker volume that persists configuration data between container restarts. All data placed in `/app/conf` inside the container is preserved thanks to this persistent volume. This applies to both `docker-compose run` and `docker-compose up` commands. ### Log Management Logs are automatically rotated with configurable retention: ```env LOG_MAX_SIZE=10MB # Maximum log file size LOG_BACKUP_COUNT=5 # Number of backup files to keep ``` ## Networking ### Default Configuration - **Network**: `pal-network` (bridge) - **Subnet**: `172.20.0.0/16` - **Isolation**: Container runs in isolated network ### Port Exposure By default, no ports are exposed. The MCP server communicates via stdio when used with Claude Desktop or other MCP clients. For external access (advanced users): ```yaml ports: - "3000:3000" # Add to service configuration if needed ``` ## Troubleshooting ### Common Issues **1. Health check failures:** ```bash # Check logs docker-compose logs pal-mcp # Manual health check docker exec pal-mcp-server python /usr/local/bin/healthcheck.py ``` **2. Permission errors:** ```bash # Fix log directory permissions sudo chown -R 1000:1000 ./logs ``` **3. Environment variables not loaded:** ```bash # Verify .env file exists and is readable ls -la .env cat .env ``` **4. API key validation errors:** ```bash # Check environment variables in container docker exec pal-mcp-server env | grep -E "(GEMINI|OPENAI|XAI)" ``` ### Debug Mode Enable verbose logging for troubleshooting: ```env LOG_LEVEL=DEBUG ``` ## Production Considerations ### Security 1. **Use Docker secrets** for API keys in production: ```yaml secrets: gemini_api_key: external: true ``` 2. **Enable AppArmor/SELinux** if available 3. **Regular security updates**: ```bash docker-compose pull docker-compose up -d ``` ### Monitoring Consider integrating with monitoring solutions: - **Prometheus**: Health check metrics - **Grafana**: Log visualization - **AlertManager**: Health status alerts ### Backup Backup persistent volumes: ```bash # Backup configuration docker run --rm -v pal-mcp-config:/data -v $(pwd):/backup alpine tar czf /backup/config-backup.tar.gz -C /data . # Restore configuration docker run --rm -v pal-mcp-config:/data -v $(pwd):/backup alpine tar xzf /backup/config-backup.tar.gz -C /data ``` ## Performance Tuning ### Resource Optimization Adjust limits based on your workload: ```yaml deploy: resources: limits: memory: 1G # Increase for heavy workloads cpus: '1.0' # More CPU for concurrent requests ``` ### Memory Management Monitor memory usage: ```bash docker stats pal-mcp-server ``` Adjust Python memory settings if needed: ```env PYTHONMALLOC=pymalloc MALLOC_ARENA_MAX=2 ``` ## Integration with Claude Desktop Configure Claude Desktop to use the containerized server. **Choose one of the configurations below based on your needs:** ### Option 1: Direct Docker Run (Recommended) **The simplest and most reliable option for most users.** ```json { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "/absolute/path/to/pal-mcp-server/.env", "-v", "/absolute/path/to/pal-mcp-server/logs:/app/logs", "pal-mcp-server:latest" ] } } } ``` **Exemple Windows** : ```json { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "C:/path/to/pal-mcp-server/.env", "-v", "C:/path/to/pal-mcp-server/logs:/app/logs", "pal-mcp-server:latest" ] } } } ``` ### Option 2: Docker Compose Run (one-shot, uses docker-compose.yml) **To use the advanced configuration from docker-compose.yml without a persistent container.** ```json { "mcpServers": { "pal-mcp": { "command": "docker-compose", "args": [ "-f", "/absolute/path/to/pal-mcp-server/docker-compose.yml", "run", "--rm", "pal-mcp" ] } } } ``` ### Option 3: Inline Environment Variables (Advanced) **For highly customized needs.** ```json { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", "-e", "GEMINI_API_KEY=your_key_here", "-e", "LOG_LEVEL=INFO", "-e", "DEFAULT_MODEL=auto", "-v", "/path/to/logs:/app/logs", "pal-mcp-server:latest" ] } } } ``` ### Configuration Notes **Important notes:** - Replace `/absolute/path/to/pal-mcp-server` with the actual path to your project. - Always use forward slashes `/` for Docker volumes, even on Windows. - Ensure the `.env` file exists and contains your API keys. - **Persistent volumes**: Docker Compose options (Options 2) automatically use the `pal-mcp-config` named volume for persistent configuration storage. **Environment file requirements:** ```env # At least one API key is required GEMINI_API_KEY=your_gemini_key OPENAI_API_KEY=your_openai_key # ... other keys ``` **Troubleshooting:** - If Option 1 fails: check that the Docker image exists (`docker images pal-mcp-server`). - If Option 2 fails: verify the compose file path and ensure the service is not already in use. - Permission issues: make sure the `logs` folder is writable. ## Advanced Configuration ### Custom Networks For complex deployments: ```yaml networks: pal-network: driver: bridge ipam: config: - subnet: 172.20.0.0/16 gateway: 172.20.0.1 ``` ### Multiple Instances Run multiple instances with different configurations: ```bash # Copy compose file cp docker-compose.yml docker-compose.dev.yml # Modify service names and ports # Deploy with custom compose file docker-compose -f docker-compose.dev.yml up -d ``` ## Migration and Updates ### Updating the Server ```bash # Pull latest changes git pull origin main # Rebuild and restart docker-compose down docker-compose build --no-cache ./docker/scripts/deploy.sh ``` ### Data Migration When upgrading, configuration is preserved in the named volume `pal-mcp-config`. For major version upgrades, check the [CHANGELOG](../CHANGELOG.md) for breaking changes. ## Support For any questions, open an issue on GitHub or consult the official documentation. --- **Next Steps:** - Review the [Configuration Guide](configuration.md) for detailed environment variable options - Check [Advanced Usage](advanced-usage.md) for custom model configurations - See [Troubleshooting](troubleshooting.md) for common issues and solutions ================================================ FILE: docs/gemini-setup.md ================================================ # Gemini CLI Setup > **Note**: While PAL MCP Server connects successfully to Gemini CLI, tool invocation is not working > correctly yet. We'll update this guide once the integration is fully functional. This guide explains how to configure PAL MCP Server to work with [Gemini CLI](https://github.com/google-gemini/gemini-cli). ## Prerequisites - PAL MCP Server installed and configured - Gemini CLI installed - At least one API key configured in your `.env` file ## Configuration 1. Edit `~/.gemini/settings.json` and add: ```json { "mcpServers": { "pal": { "command": "/path/to/pal-mcp-server/pal-mcp-server" } } } ``` 2. Replace `/path/to/pal-mcp-server` with your actual PAL MCP installation path (the folder name may still be `pal-mcp-server`). 3. If the `pal-mcp-server` wrapper script doesn't exist, create it: ```bash #!/bin/bash DIR="$(cd "$(dirname "$0")" && pwd)" cd "$DIR" exec .pal_venv/bin/python server.py "$@" ``` Then make it executable: `chmod +x pal-mcp-server` 4. Restart Gemini CLI. All 15 PAL tools are now available in your Gemini CLI session. ================================================ FILE: docs/getting-started.md ================================================ # Getting Started with PAL MCP Server This guide walks you through setting up the PAL MCP Server from scratch, including installation, configuration, and first usage. ## Prerequisites - **Python 3.10+** (3.12 recommended) - **Git** - **[uv installed](https://docs.astral.sh/uv/getting-started/installation/)** (for uvx method) - **Windows users**: WSL2 required for Claude Code CLI ## Step 1: Get API Keys You need at least one API key. Choose based on your needs: ### Option A: OpenRouter (Recommended for beginners) **One API for multiple models** - Visit [OpenRouter](https://openrouter.ai/) and sign up - Generate an API key - Control spending limits in your dashboard - Access GPT-4, Claude, Gemini, and more through one API ### Option B: Native Provider APIs **Gemini (Google):** - Visit [Google AI Studio](https://makersuite.google.com/app/apikey) - Generate an API key - **Note**: For Gemini 3.0 / 2.5 Pro, use a paid API key (free tier has limited access) **OpenAI:** - Visit [OpenAI Platform](https://platform.openai.com/api-keys) - Generate an API key for GPT-5.2, GPT-5.1-Codex, GPT-5, O3 access **X.AI (Grok):** - Visit [X.AI Console](https://console.x.ai/) - Generate an API key for Grok models **DIAL Platform:** - Visit [DIAL Platform](https://dialx.ai/) - Generate API key for vendor-agnostic model access ### Option C: Local Models (Free) **Ollama:** ```bash # Install Ollama curl -fsSL https://ollama.ai/install.sh | sh # Start Ollama service ollama serve # Pull a model (e.g., Llama 3.2) ollama pull llama3.2 ``` **Other local options:** - **vLLM**: Self-hosted inference server - **LM Studio**: Local model hosting with OpenAI-compatible API - **Text Generation WebUI**: Popular local interface 👉 **[Complete custom model setup guide](custom_models.md)** ## Step 2: Installation Choose your preferred installation method: ### Method A: Instant Setup with uvx (Recommended) **Prerequisites**: [Install uv first](https://docs.astral.sh/uv/getting-started/installation/) Choose your AI coding assistant and add the corresponding configuration: **For Claude Desktop:** 1. Open Claude Desktop → Settings → Developer → Edit Config 2. Add this configuration: ```json { "mcpServers": { "pal": { "command": "sh", "args": [ "-c", "for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \"$p\" ] && exec \"$p\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1" ], "env": { "PATH": "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:~/.local/bin", "GEMINI_API_KEY": "your_api_key_here" } } } } ``` **For Claude Code CLI:** Create `.mcp.json` in your project root: ```json { "mcpServers": { "pal": { "command": "sh", "args": [ "-c", "for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \"$p\" ] && exec \"$p\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1" ], "env": { "PATH": "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:~/.local/bin", "GEMINI_API_KEY": "your_api_key_here" } } } } ``` **For Gemini CLI:** Edit `~/.gemini/settings.json`: ```json { "mcpServers": { "pal": { "command": "sh", "args": [ "-c", "for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \"$p\" ] && exec \"$p\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1" ], "env": { "PATH": "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:~/.local/bin", "GEMINI_API_KEY": "your_api_key_here" } } } } ``` **For Codex CLI:** Edit `~/.codex/config.toml`: ```toml [mcp_servers.pal] command = "bash" args = ["-c", "for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \\\"$p\\\" ] && exec \\\"$p\\\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1"] tool_timeout_sec = 1200 # 20 minutes; added automatically by the setup script so upstream providers can respond [mcp_servers.pal.env] PATH = "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:$HOME/.local/bin:$HOME/.cargo/bin:$HOME/bin" GEMINI_API_KEY = "your_api_key_here" ``` Enable Codex's built-in web-search tool so PAL's `apilookup` instructions can execute successfully: ```toml [tools] web_search = true ``` Add the block above if `[tools]` is missing from the file; otherwise ensure `web_search = true` appears in that section. **For Qwen Code CLI:** Create or edit `~/.qwen/settings.json`: ```json { "mcpServers": { "pal": { "command": "bash", "args": [ "-c", "for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \"$p\" ] && exec \"$p\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1" ], "cwd": "/path/to/pal-mcp-server", "env": { "PATH": "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:~/.local/bin", "GEMINI_API_KEY": "your_api_key_here" } } } } ``` Replace the placeholder API key with the providers you use (Gemini, OpenAI, OpenRouter, etc.). **For OpenCode CLI:** Edit `~/.config/opencode/opencode.json`: ```json { "$schema": "https://opencode.ai/config.json", "mcp": { "pal": { "type": "local", "command": [ "/path/to/pal-mcp-server/.pal_venv/bin/python", "/path/to/pal-mcp-server/server.py" ], "cwd": "/path/to/pal-mcp-server", "enabled": true, "environment": { "GEMINI_API_KEY": "your_api_key_here" } } } } ``` Add any other API keys you rely on (`OPENAI_API_KEY`, `OPENROUTER_API_KEY`, etc.). #### IDE Clients (Cursor & VS Code) PAL works in GUI IDEs that speak MCP. The configuration mirrors the CLI examples above—point the client at the `uvx` launcher and set any required environment variables. **Cursor IDE** 1. Open Cursor → `Settings` (`Cmd+,`/`Ctrl+,`) → **Integrations › Model Context Protocol (MCP)**. 2. Click **Add MCP Server** and supply the following values: - Command: `sh` - Args: `-c` and `for p in $(which uvx 2>/dev/null) $HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x "$p" ] && exec "$p" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1` - Environment (example): - `PATH=/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:~/.local/bin` - `GEMINI_API_KEY=your_api_key_here` 3. Save the configuration—Cursor will launch the MCP server on demand. See the [Cursor MCP guide](https://cursor.com/docs) for screenshots of the UI. **Visual Studio Code (Claude Dev extension)** 1. Install the [Claude Dev extension](https://marketplace.visualstudio.com/items?itemName=Anthropic.claude-vscode) v0.6.0 or later. 2. Open the Command Palette (`Cmd+Shift+P`/`Ctrl+Shift+P`) → **Claude: Configure MCP Servers** → **Add server**. 3. When prompted, use the same values as above: - Command: `sh` - Args: `-c` and the `uvx` bootstrap loop - Environment: add the API keys you need (e.g. `GEMINI_API_KEY`, `OPENAI_API_KEY`) 4. Save the JSON snippet the extension generates. VS Code will reload the server automatically the next time you interact with Claude. 👉 Pro tip: If you prefer a one-line command, replace the long loop with `uvx --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server`—just make sure `uvx` is on your PATH for every client. **Benefits of uvx method:** - ✅ Zero manual setup required - ✅ Always pulls latest version - ✅ No local dependencies to manage - ✅ Works without Python environment setup ### Method B: Clone and Setup ```bash # Clone the repository git clone https://github.com/BeehiveInnovations/pal-mcp-server.git cd pal-mcp-server # One-command setup (handles everything) ./run-server.sh # Or for Windows PowerShell: ./run-server.ps1 # View configuration for Claude Desktop ./run-server.sh -c # See all options ./run-server.sh --help ``` **What the setup script does:** - ✅ Creates Python virtual environment - ✅ Installs all dependencies - ✅ Creates .env file for API keys - ✅ Configures Claude integrations - ✅ Provides copy-paste configuration **After updates:** Always run `./run-server.sh` again after `git pull`. **Windows users**: See the [WSL Setup Guide](wsl-setup.md) for detailed WSL configuration. ## Step 3: Configure API Keys ### For uvx installation: Add your API keys directly to the MCP configuration shown above. ### For clone installation: Edit the `.env` file: ```bash nano .env ``` Add your API keys (at least one required): ```env # Choose your providers (at least one required) GEMINI_API_KEY=your-gemini-api-key-here # For Gemini models OPENAI_API_KEY=your-openai-api-key-here # For GPT-5.2, GPT-5.1-Codex, O3 XAI_API_KEY=your-xai-api-key-here # For Grok models OPENROUTER_API_KEY=your-openrouter-key # For multiple models # DIAL Platform (optional) DIAL_API_KEY=your-dial-api-key-here DIAL_API_HOST=https://core.dialx.ai # Default host (optional) DIAL_API_VERSION=2024-12-01-preview # API version (optional) DIAL_ALLOWED_MODELS=o3,gemini-2.5-pro # Restrict models (optional) # Custom/Local models (Ollama, vLLM, etc.) CUSTOM_API_URL=http://localhost:11434/v1 # Ollama example CUSTOM_API_KEY= # Empty for Ollama CUSTOM_MODEL_NAME=llama3.2 # Default model name ``` ## Prevent Client Timeouts Some MCP clients default to short timeouts and can disconnect from PAL during long tool runs. Configure each client with a generous ceiling (we recommend at least five minutes); the PAL setup script now writes a 20-minute tool timeout for Codex so upstream providers contacted by the server have time to respond. ### Claude Code & Claude Desktop Claude reads MCP-related environment variables either from your shell or from `~/.claude/settings.json`. Add (or update) the `env` block so both startup and tool execution use a 5-minute limit: ```json { "env": { "MCP_TIMEOUT": "300000", "MCP_TOOL_TIMEOUT": "300000" } } ``` You can scope this block at the top level of `settings.json` (applies to every session) or under a specific `mcpServers..env` entry if you only want it for PAL (the server name may still be `pal` while configurations catch up). The values are in milliseconds. Note: Claude’s SSE transport still enforces an internal ceiling of roughly five minutes; long-running HTTP/SSE servers may need retries until Anthropic ships their fix. ### Codex CLI Codex exposes per-server timeouts in `~/.codex/config.toml`. Add (or bump) these keys under `[[mcp_servers.]]`: ```toml [mcp_servers.pal] command = "..." args = ["..."] startup_timeout_sec = 300 # default is 10 seconds tool_timeout_sec = 1200 # default is 60 seconds; setup script pre-populates 20 minutes so upstream providers can respond ``` `startup_timeout_sec` covers the initial handshake/list tools step, while `tool_timeout_sec` governs each tool call. Raise the latter if the providers your MCP server invokes routinely need more than 20 minutes. ### Gemini CLI Gemini uses a single `timeout` field per server inside `~/.gemini/settings.json`. Set it to at least five minutes (values are milliseconds): ```json { "mcpServers": { "pal": { "command": "uvx", "args": ["pal-mcp-server"], "timeout": 300000 } } } ``` Versions 0.2.1 and newer currently ignore values above ~60 seconds for some transports due to a known regression; if you still see premature disconnects we recommend breaking work into smaller calls or watching the Gemini CLI release notes for the fix. **Important notes:** - ⭐ **No restart needed** - Changes take effect immediately - ⭐ If multiple APIs configured, native APIs take priority over OpenRouter - ⭐ Configure model aliases in [`conf/custom_models.json`](../conf/custom_models.json) ## Step 4: Test the Installation ### For Claude Desktop: 1. Restart Claude Desktop 2. Open a new conversation 3. Try: `"Use pal to list available models"` ### For Claude Code CLI: 1. Exit any existing Claude session 2. Run `claude` from your project directory 3. Try: `"Use pal to chat about Python best practices"` ### For Gemini CLI: **Note**: While PAL MCP connects to Gemini CLI, tool invocation isn't working correctly yet. See [Gemini CLI Setup](gemini-setup.md) for updates. ### For Qwen Code CLI: 1. Restart the Qwen Code CLI if it's running (`qwen exit`). 2. Run `qwen mcp list --scope user` and confirm `pal` shows `CONNECTED`. 3. Try: `"/mcp"` to inspect available tools or `"Use pal to analyze this repo"`. ### For OpenCode CLI: 1. Restart OpenCode (or run `OpenCode: Reload Config`). 2. Open **Settings › Tools › MCP** and confirm `pal` is enabled. 3. Start a new chat and try: `"Use pal to list available models"`. ### For Codex CLI: 1. Restart Codex CLI if running 2. Open a new conversation 3. Try: `"Use pal to list available models"` ### Test Commands: ``` "Use pal to list available models" "Chat with pal about the best approach for API design" "Use pal thinkdeep with gemini pro about scaling strategies" "Debug this error with o3: [paste error]" ``` **Note**: Codex CLI provides excellent MCP integration with automatic environment variable configuration when using the setup script. ## Step 5: Start Using PAL ### Basic Usage Patterns: **Let Claude pick the model:** ``` "Use pal to analyze this code for security issues" "Debug this race condition with pal" "Plan the database migration with pal" ``` **Specify the model:** ``` "Use pal with gemini pro to review this complex algorithm" "Debug with o3 using pal for logical analysis" "Get flash to quickly format this code via pal" ``` **Multi-model workflows:** ``` "Use pal to get consensus from pro and o3 on this architecture" "Code review with gemini, then precommit validation with o3" "Analyze with flash, then deep dive with pro if issues found" ``` ### Quick Tool Reference: **🤝 Collaboration**: `chat`, `thinkdeep`, `planner`, `consensus` **🔍 Code Analysis**: `analyze`, `codereview`, `debug`, `precommit` **⚒️ Development**: `refactor`, `testgen`, `secaudit`, `docgen` **🔧 Utilities**: `challenge`, `tracer`, `listmodels`, `version` 👉 **[Complete Tools Reference](tools/)** with detailed examples and parameters ## Common Issues and Solutions ### "pal not found" or "command not found" **For uvx installations:** - Ensure `uv` is installed and in PATH - Try: `which uvx` to verify uvx is available - Check PATH includes `/usr/local/bin` and `~/.local/bin` **For clone installations:** - Run `./run-server.sh` again to verify setup - Check virtual environment: `which python` should show `.pal_venv/bin/python` ### API Key Issues **"Invalid API key" errors:** - Verify API keys in `.env` file or MCP configuration - Test API keys directly with provider's API - Check for extra spaces or quotes around keys **"Model not available":** - Run `"Use pal to list available models"` to see what's configured - Check model restrictions in environment variables - Verify API key has access to requested models ### Performance Issues **Slow responses:** - Use faster models: `flash` instead of `pro` - Lower thinking modes: `minimal` or `low` instead of `high` - Restrict model access to prevent expensive model selection **Token limit errors:** - Use models with larger context windows - Break large requests into smaller chunks - See [Working with Large Prompts](advanced-usage.md#working-with-large-prompts) ### More Help 👉 **[Complete Troubleshooting Guide](troubleshooting.md)** with detailed solutions 👉 **[Advanced Usage Guide](advanced-usage.md)** for power-user features 👉 **[Configuration Reference](configuration.md)** for all options ## What's Next? 🎯 **Try the example workflows in the main README** 📚 **Explore the [Tools Reference](tools/)** to understand what each tool can do ⚡ **Read the [Advanced Usage Guide](advanced-usage.md)** for complex workflows 🔧 **Check out [Configuration Options](configuration.md)** to customize behavior 💡 **Join discussions and get help** in the project issues or discussions ## Quick Configuration Templates ### Development Setup (Balanced) ```env DEFAULT_MODEL=auto GEMINI_API_KEY=your-key OPENAI_API_KEY=your-key GOOGLE_ALLOWED_MODELS=flash,pro OPENAI_ALLOWED_MODELS=gpt-5.1-codex-mini,gpt-5-mini,o4-mini ``` ### Cost-Optimized Setup ```env DEFAULT_MODEL=flash GEMINI_API_KEY=your-key GOOGLE_ALLOWED_MODELS=flash ``` ### High-Performance Setup ```env DEFAULT_MODEL=auto GEMINI_API_KEY=your-key OPENAI_API_KEY=your-key GOOGLE_ALLOWED_MODELS=pro OPENAI_ALLOWED_MODELS=gpt-5.1-codex,gpt-5.2 ``` ### Local-First Setup ```env DEFAULT_MODEL=auto CUSTOM_API_URL=http://localhost:11434/v1 CUSTOM_MODEL_NAME=llama3.2 # Add cloud APIs as backup GEMINI_API_KEY=your-key ``` Happy coding with your AI development team! 🤖✨ ================================================ FILE: docs/index.md ================================================ # PAL MCP Server Documentation _Formerly known as PAL MCP. See the short [name change note](name-change.md) for context._ | Document | Description | |----------|-------------| | [Getting Started](getting-started.md) | Installation paths, prerequisite setup, and first-run guidance. | | [Adding Providers](adding_providers.md) | How to register new AI providers and advertise capabilities. | | [Azure OpenAI](azure_openai.md) | Configure Azure deployments, capability overrides, and env mappings. | | [Model Ranking](model_ranking.md) | How intelligence scores translate into auto-mode ordering. | | [Custom Models](custom_models.md) | Configure OpenRouter/custom models and aliases. | | [Adding Tools](adding_tools.md) | Create new tools using the shared base classes. | | [Advanced Usage](advanced-usage.md) | Auto-mode tricks, workflow tools, and collaboration tips. | | [Configuration](configuration.md) | .env options, restriction policies, logging levels. | | [Testing](testing.md) | Test strategy, command cheats, and coverage notes. | | [Troubleshooting](troubleshooting.md) | Common issues and resolutions. | Additional docs live in this directory; start with the table above to orient yourself. ================================================ FILE: docs/locale-configuration.md ================================================ # Locale Configuration for PAL MCP Server This guide explains how to configure and use the localization feature to customize the language of responses from MCP tools. ## Overview The localization feature allows you to specify the language in which MCP tools should respond, while maintaining their analytical capabilities. This is especially useful for non-English speakers who want to receive answers in their native language. ## Configuration ### 1. Environment Variable Set the language using the `LOCALE` environment variable in your `.env` file: ```bash # In your .env file LOCALE=fr-FR ``` ### 2. Supported Languages You can use any standard language code. Examples: - `fr-FR` - French (France) - `en-US` - English (United States) - `zh-CN` - Chinese (Simplified) - `zh-TW` - Chinese (Traditional) - `ja-JP` - Japanese - `ko-KR` - Korean - `es-ES` - Spanish (Spain) - `de-DE` - German (Germany) - `it-IT` - Italian (Italy) - `pt-PT` - Portuguese (Portugal) - `ru-RU` - Russian (Russia) - `ar-SA` - Arabic (Saudi Arabia) ### 3. Default Behavior If no language is specified (`LOCALE` is empty or unset), tools will default to English. ## Technical Implementation ### Architecture Localization is implemented in the `BaseTool` class in `tools/shared/base_tool.py`. All tools inherit this feature automatically. ### `get_language_instruction()` Method ```python def get_language_instruction(self) -> str: """ Generate language instruction based on LOCALE configuration. Returns: str: Language instruction to prepend to prompt, or empty string if no locale set """ import os locale = os.getenv("LOCALE", "").strip() if not locale: return "" return f"Always respond in {locale}.\n\n" ``` ### Integration in Tool Execution The language instruction is automatically prepended to the system prompt of each tool: ```python # In tools/simple/base.py base_system_prompt = self.get_system_prompt() language_instruction = self.get_language_instruction() system_prompt = language_instruction + base_system_prompt ``` ## Usage ### 1. Basic Setup 1. Edit your `.env` file: ```bash LOCALE=fr-FR ``` 2. Restart the MCP server: ```bash ./run-server.sh ``` 3. Use any tool – responses will be in the specified language. ### 2. Example **Before (default English):** ``` Tool: chat Input: "Explain how to use Python dictionaries" Output: "Python dictionaries are key-value pairs that allow you to store and organize data..." ``` **After (with LOCALE=fr-FR):** ``` Tool: chat Input: "Explain how to use Python dictionaries" Output: "Les dictionnaires Python sont des paires clé-valeur qui permettent de stocker et d'organiser des données..." ``` ### 3. Affected Tools All MCP tools are affected by this configuration: - `chat` – General conversation - `codereview` – Code review - `analyze` – Code analysis - `debug` – Debugging - `refactor` – Refactoring - `thinkdeep` – Deep thinking - `consensus` – Model consensus - And all other tools... ## Best Practices ### 1. Language Choice - Use standard language codes (ISO 639-1 with ISO 3166-1 country codes) - Be specific with regional variants if needed (e.g., `zh-CN` vs `zh-TW`) ### 2. Consistency - Use the same language setting across your team for consistency - Document the chosen language in your team documentation ### 3. Testing - Test the configuration with different tools to ensure consistency ## Troubleshooting ### Issue: Language does not change **Solution:** 1. Check that the `LOCALE` variable is correctly set in `.env` 2. Fully restart the MCP server 3. Ensure there are no extra spaces in the value ### Issue: Partially translated responses **Explanation:** - AI models may sometimes mix languages - This depends on the multilingual capabilities of the model used - Technical terms may remain in English ### Issue: Configuration errors **Solution:** 1. Check the syntax of your `.env` file 2. Make sure there are no quotes around the value ## Advanced Customization ### Customizing the Language Instruction To customize the language instruction, modify the `get_language_instruction()` method in `tools/shared/base_tool.py`: ```python def get_language_instruction(self) -> str: import os locale = os.getenv("LOCALE", "").strip() if not locale: return "" # Custom instruction return f"Always respond in {locale} and use a professional tone.\n\n" ``` ### Per-Tool Customization You can also override the method in specific tools for custom behavior: ```python class MyCustomTool(SimpleTool): def get_language_instruction(self) -> str: import os locale = os.getenv("LOCALE", "").strip() if locale == "fr-FR": return "Respond in French with precise technical vocabulary.\n\n" elif locale == "zh-CN": return "请用中文回答，使用专业术语。\n\n" else: return super().get_language_instruction() ``` ## Integration with Other Features Localization works with all other MCP server features: - **Conversation threading** – Multilingual conversations are supported - **File processing** – File analysis is in the specified language - **Web search** – Search instructions remain functional - **Model selection** – Works with all supported models ================================================ FILE: docs/logging.md ================================================ # Logging ## Quick Start - Follow Logs The easiest way to monitor logs is to use the `-f` flag when starting the server: ```bash # Start server and automatically follow MCP logs ./run-server.sh -f ``` This will start the server and immediately begin tailing the MCP server logs. ## Log Files Logs are stored in the `logs/` directory within your project folder: - **`mcp_server.log`** - Main server operations, API calls, and errors - **`mcp_activity.log`** - Tool calls and conversation tracking Log files rotate automatically when they reach 20MB, keeping up to 10 rotated files. ## Viewing Logs To monitor MCP server activity: ```bash # Follow logs in real-time tail -f logs/mcp_server.log # View last 100 lines tail -n 100 logs/mcp_server.log # View activity logs (tool calls only) tail -f logs/mcp_activity.log # Search for specific patterns grep "ERROR" logs/mcp_server.log grep "tool_name" logs/mcp_activity.log ``` ## Log Level Set verbosity with `LOG_LEVEL` in your `.env` file: ```env # Options: DEBUG, INFO, WARNING, ERROR LOG_LEVEL=INFO ``` - **DEBUG**: Detailed information for debugging - **INFO**: General operational messages (default) - **WARNING**: Warning messages - **ERROR**: Only error messages ## Log Format Logs use a standardized format with timestamps: ``` 2024-06-14 10:30:45,123 - module.name - INFO - Message here ``` ## Tips - Use `./run-server.sh -f` for the easiest log monitoring experience - Activity logs show only tool-related events for cleaner output - Main server logs include all operational details - Logs persist across server restarts ================================================ FILE: docs/model_ranking.md ================================================ # Model Capability Ranking Auto mode needs a short, trustworthy list of models to suggest. The server computes a capability rank for every model at runtime using a simple recipe: 1. Start with the human-supplied `intelligence_score` (1–20). This is the anchor—multiply it by five to map onto the 0–100 scale the server uses. 2. Add a few light bonuses for hard capabilities: - **Context window:** up to +5 (log-scale bonus when the model exceeds ~1K tokens). - **Output budget:** +2 for ≥65K tokens, +1 for ≥32K. - **Extended thinking:** +3 when the provider supports it. - **Function calling / JSON / images:** +1 each when available. - **Custom endpoints:** −1 to nudge cloud-hosted defaults ahead unless tuned. 3. Clamp the final score to 0–100 so downstream callers can rely on the range. In code this looks like: ```python base = clamp(intelligence_score, 1, 20) * 5 ctx_bonus = min(5, max(0, log10(context_window) - 3)) output_bonus = 2 if max_output_tokens >= 65_000 else 1 if >= 32_000 else 0 feature_bonus = ( (3 if supports_extended_thinking else 0) + (1 if supports_function_calling else 0) + (1 if supports_json_mode else 0) + (1 if supports_images else 0) ) penalty = 1 if provider == CUSTOM else 0 effective_rank = clamp(base + ctx_bonus + output_bonus + feature_bonus - penalty, 0, 100) ``` The bonuses are intentionally small—the human intelligence score does most of the work so you can enforce organisational preferences easily. ## Picking an intelligence score A straightforward rubric that mirrors typical provider tiers: | Intelligence | Guidance | |--------------|-------------------------------------------------------------------------------------------| | 18–19 | Frontier reasoning models (Gemini 3.0 Pro, Gemini 2.5 Pro, GPT‑5.1 Codex, GPT‑5.2 Pro, GPT‑5.2, GPT‑5) | | 15–17 | Strong general models with large context (O3 Pro, DeepSeek R1) | | 12–14 | Balanced assistants (Claude Opus/Sonnet, Mistral Large) | | 9–11 | Fast distillations (Gemini Flash, GPT-5 Mini, Mistral medium) | | 6–8 | Local or efficiency-focused models (Llama 3 70B, Claude Haiku) | | ≤5 | Experimental/lightweight models | Record the reasoning for your scores so future updates stay consistent. ## How the rank is used The ranked list is cached per provider and consumed by: - Tool schemas (`model` parameter descriptions) when auto mode is active. - The `listmodels` tool’s “top models” sections. - Fallback messaging when a requested model is unavailable. Because the rank is computed after restriction filters, only allowed models appear in these summaries. ## Customising further If you need a different weighting you can: - Override `intelligence_score` in your provider or custom model config. - Subclass the provider and override `get_effective_capability_rank()`. - Post-process the rank via `get_capabilities_by_rank()` before surfacing it. Most teams find that adjusting `intelligence_score` alone is enough to keep auto mode honest without revisiting code. ================================================ FILE: docs/name-change.md ================================================ # PAL MCP Name Change PAL MCP was previously called Zen MCP. We renamed to avoid confusion with another similarly named product and to better reflect our role as a Provider Abstraction Layer. The software and workflows are the same. Due to the change of name, you may need to run `run-server.sh` again to setup the new connection, and re-visit any `ZEN` name used within `.env` and change it to `PAL`. ================================================ FILE: docs/testing.md ================================================ # Testing Guide This project includes comprehensive test coverage through unit tests and integration simulator tests. ## Running Tests ### Prerequisites - Environment set up: `./run-server.sh` - Use `./run-server.sh -f` to automatically follow logs after starting ### Unit Tests Run all unit tests with pytest: ```bash # Run all tests with verbose output python -m pytest -xvs # Run specific test file python -m pytest tests/test_providers.py -xvs ``` ### Simulator Tests Simulator tests replicate real-world Claude CLI interactions with the standalone MCP server. Unlike unit tests that test isolated functions, simulator tests validate the complete end-to-end flow including: - Actual MCP protocol communication - Standalone server interactions - Multi-turn conversations across tools - Log output validation **Important**: Simulator tests require `LOG_LEVEL=DEBUG` in your `.env` file to validate detailed execution logs. #### Monitoring Logs During Tests **Important**: The MCP stdio protocol interferes with stderr output during tool execution. Tool execution logs are written to local log files. This is a known limitation of the stdio-based MCP protocol. To monitor logs during test execution: ```bash # Start server and automatically follow logs ./run-server.sh -f # Or manually monitor main server logs (includes all tool execution details) tail -f -n 500 logs/mcp_server.log # Monitor MCP activity logs (tool calls and completions) tail -f logs/mcp_activity.log # Check log file sizes (logs rotate at 20MB) ls -lh logs/mcp_*.log* ``` **Log Rotation**: All log files are configured with automatic rotation at 20MB to prevent disk space issues. The server keeps: - 10 rotated files for mcp_server.log (200MB total) - 5 rotated files for mcp_activity.log (100MB total) **Why logs appear in files**: The MCP stdio_server captures stderr during tool execution to prevent interference with the JSON-RPC protocol communication. This means tool execution logs are written to files rather than displayed in console output. #### Running All Simulator Tests ```bash # Run all simulator tests python communication_simulator_test.py # Run with verbose output for debugging python communication_simulator_test.py --verbose # Keep server logs after tests for inspection python communication_simulator_test.py --keep-logs ``` #### Running Individual Tests To run a single simulator test in isolation (useful for debugging or test development): ```bash # Run a specific test by name python communication_simulator_test.py --individual basic_conversation # Examples of available tests: python communication_simulator_test.py --individual content_validation python communication_simulator_test.py --individual cross_tool_continuation python communication_simulator_test.py --individual memory_validation ``` #### Other Options ```bash # List all available simulator tests with descriptions python communication_simulator_test.py --list-tests # Run multiple specific tests (not all) python communication_simulator_test.py --tests basic_conversation content_validation ``` ### Code Quality Checks Before committing, ensure all linting passes: ```bash # Run all linting checks ruff check . black --check . isort --check-only . # Auto-fix issues ruff check . --fix black . isort . ``` ## What Each Test Suite Covers ### Unit Tests Test isolated components and functions: - **Provider functionality**: Model initialization, API interactions, capability checks - **Tool operations**: All MCP tools (chat, analyze, debug, etc.) - **Conversation memory**: Threading, continuation, history management - **File handling**: Path validation, token limits, deduplication - **Auto mode**: Model selection logic and fallback behavior ### HTTP Recording/Replay Tests (HTTP Transport Recorder) Tests for expensive API calls (like o3-pro) use custom recording/replay: - **Real API validation**: Tests against actual provider responses - **Cost efficiency**: Record once, replay forever - **Provider compatibility**: Validates fixes against real APIs - Uses HTTP Transport Recorder for httpx-based API calls - See [HTTP Recording/Replay Testing Guide](./vcr-testing.md) for details ### Simulator Tests Validate real-world usage scenarios by simulating actual Claude prompts: - **Basic conversations**: Multi-turn chat functionality with real prompts - **Cross-tool continuation**: Context preservation across different tools - **File deduplication**: Efficient handling of repeated file references - **Model selection**: Proper routing to configured providers - **Token allocation**: Context window management in practice - **Redis validation**: Conversation persistence and retrieval ## Contributing For detailed contribution guidelines, testing requirements, and code quality standards, please see our [Contributing Guide](./contributions.md). ### Quick Testing Reference ```bash # Run quality checks ./code_quality_checks.sh # Run unit tests python -m pytest -xvs # Run simulator tests (for tool changes) python communication_simulator_test.py ``` Remember: All tests must pass before submitting a PR. See the [Contributing Guide](./contributions.md) for complete requirements. ================================================ FILE: docs/tools/analyze.md ================================================ # Analyze Tool - Smart File Analysis **General-purpose code understanding and exploration through workflow-driven investigation** The `analyze` tool provides comprehensive code analysis and understanding capabilities, helping you explore codebases, understand architecture, and identify patterns across files and directories. This workflow tool guides Claude through systematic investigation of code structure, patterns, and architectural decisions across multiple steps, gathering comprehensive insights before providing expert analysis. ## Thinking Mode **Default is `medium` (8,192 tokens).** Use `high` for architecture analysis (comprehensive insights worth the cost) or `low` for quick file overviews (save ~6k tokens). ## How the Workflow Works The analyze tool implements a **structured workflow** for thorough code understanding: **Investigation Phase (Claude-Led):** 1. **Step 1**: Claude describes the analysis plan and begins examining code structure 2. **Step 2+**: Claude investigates architecture, patterns, dependencies, and design decisions 3. **Throughout**: Claude tracks findings, relevant files, insights, and confidence levels 4. **Completion**: Once analysis is comprehensive, Claude signals completion **Expert Analysis Phase:** After Claude completes the investigation (unless confidence is **certain**): - Complete analysis summary with all findings - Architectural insights and pattern identification - Strategic improvement recommendations - Final expert assessment based on investigation This workflow ensures methodical analysis before expert insights, resulting in deeper understanding and more valuable recommendations. ## Example Prompts **Basic Usage:** ``` "Use gemini to analyze main.py to understand how it works" "Get gemini to do an architecture analysis of the src/ directory" ``` ## Key Features - **Analyzes single files or entire directories** with intelligent file filtering - **Supports specialized analysis types**: architecture, performance, security, quality, general - **Uses file paths (not content) for clean terminal output** while processing full content - **Can identify patterns, anti-patterns, and refactoring opportunities** - **Large codebase support**: Handle massive codebases with 1M token context models - **Cross-file relationship mapping**: Understand dependencies and interactions - **Architecture visualization**: Describe system structure and component relationships - **Image support**: Analyze architecture diagrams, UML charts, flowcharts: `"Analyze this system diagram with gemini to understand the data flow and identify bottlenecks"` - **Web search capability**: Automatically requests Claude to perform web searches when fresh documentation, patterns, or best practices are needed, ensuring the analysis stays current ## Tool Parameters **Workflow Investigation Parameters (used during step-by-step process):** - `step`: Current investigation step description (required for each step) - `step_number`: Current step number in analysis sequence (required) - `total_steps`: Estimated total investigation steps (adjustable) - `next_step_required`: Whether another investigation step is needed - `findings`: Discoveries and insights collected in this step (required) - `files_checked`: All files examined during investigation - `relevant_files`: Files directly relevant to the analysis (required in step 1) - `relevant_context`: Methods/functions/classes central to analysis findings - `issues_found`: Issues or concerns identified with severity levels - `confidence`: Confidence level in analysis completeness (exploring/low/medium/high/certain) - `images`: Visual references for analysis context **Initial Configuration (used in step 1):** - `prompt`: What to analyze or look for (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `analysis_type`: architecture|performance|security|quality|general (default: general) - `output_format`: summary|detailed|actionable (default: detailed) - `temperature`: Temperature for analysis (0-1, default 0.2) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert analysis phase (default: true, set to false to use Claude only) - `continuation_id`: Continue previous analysis sessions ## Analysis Types **General Analysis (default):** - Overall code structure and organization - Key components and their responsibilities - Data flow and control flow - Design patterns and architectural decisions **Architecture Analysis:** - System-level design and component relationships - Module dependencies and coupling - Separation of concerns and layering - Scalability and maintainability considerations **Performance Analysis:** - Potential bottlenecks and optimization opportunities - Algorithmic complexity assessment - Memory usage patterns - I/O and database interaction efficiency **Security Analysis:** - Security patterns and potential vulnerabilities - Input validation and sanitization - Authentication and authorization mechanisms - Data protection and privacy considerations **Quality Analysis:** - Code quality metrics and maintainability - Testing coverage and patterns - Documentation completeness - Best practices adherence ## Usage Examples **Single File Analysis:** ``` "Analyze user_controller.py to understand the authentication flow with gemini" ``` **Directory Architecture Analysis:** ``` "Use pro to analyze the src/ directory architecture and identify the main components" ``` **Performance-Focused Analysis:** ``` "Analyze backend/api/ for performance bottlenecks with o3, focus on database queries" ``` **Security Assessment:** ``` "Use gemini pro to analyze the authentication module for security patterns and potential issues" ``` **Visual + Code Analysis:** ``` "Analyze this system architecture diagram along with the src/core/ implementation to understand the data flow" ``` **Large Codebase Analysis:** ``` "Analyze the entire project structure with gemini pro to understand how all components work together" ``` ## Output Formats **Summary Format:** - High-level overview with key findings - Main components and their purposes - Critical insights and recommendations **Detailed Format (default):** - Comprehensive analysis with specific examples - Code snippets and file references - Detailed explanations of patterns and structures **Actionable Format:** - Specific recommendations and next steps - Prioritized list of improvements - Implementation guidance and examples ## Best Practices - **Be specific about goals**: Clearly state what you want to understand or discover - **Use appropriate analysis types**: Choose the type that matches your needs - **Include related files**: Analyze modules together for better context understanding - **Leverage large context models**: Use Gemini Pro for comprehensive codebase analysis - **Combine with visual context**: Include architecture diagrams or documentation - **Use continuation**: Build on previous analysis for deeper understanding ## Advanced Features **Large Codebase Support:** With models like Gemini Pro (1M context), you can analyze extensive codebases: ``` "Analyze the entire microservices architecture across all service directories" ``` **Cross-File Relationship Mapping:** Understand how components interact across multiple files: ``` "Analyze the data processing pipeline across input/, processing/, and output/ directories" ``` **Pattern Recognition:** Identify design patterns, anti-patterns, and architectural decisions: ``` "Analyze src/ to identify all design patterns used and assess their implementation quality" ``` **Web Search Enhancement:** The tool can recommend searches for current best practices and documentation: ``` After analysis: "Recommended searches for Claude: 'FastAPI async best practices 2024', 'SQLAlchemy ORM performance optimization patterns'" ``` ## When to Use Analyze vs Other Tools - **Use `analyze`** for: Understanding code structure, exploring unfamiliar codebases, architecture assessment - **Use `codereview`** for: Finding bugs and security issues with actionable fixes - **Use `debug`** for: Diagnosing specific runtime errors or performance problems - **Use `refactor`** for: Getting specific refactoring recommendations and implementation plans - **Use `chat`** for: Open-ended discussions about code without structured analysis ================================================ FILE: docs/tools/apilookup.md ================================================ # API Lookup Tool The `apilookup` tool ensures you get **current, accurate API/SDK documentation** by forcing the AI to search for the latest information rather than relying on outdated training data. This is especially critical for OS-tied APIs (iOS, macOS, Android, etc.) where the AI's knowledge cutoff may be months or years old. Most importantly, it does this within in a sub-process / sub-agent, saving you precious tokens within your working context window. ## Why Use This Tool? ### Without PAL (Using Standard AI) ``` User: "How do I add glass look to a button in Swift?" AI: [Searches based on training data knowledge cutoff] "SwiftUI glass morphism frosted glass effect button iOS 18 2025" Result: You get outdated APIs for iOS 18, not the iOS 26 effect you're after ```
[API without PAL](https://github.com/user-attachments/assets/01a79dc9-ad16-4264-9ce1-76a56c3580ee)
### With PAL (Using apilookup) ``` User: "use apilookup how do I add glass look to a button in swift?" AI: Step 1 - Search: "what is the latest iOS version 2025" → Finds: iOS 26 is current Step 2 - Search: "iOS 26 SwiftUI glass effect button 2025" → Gets current APIs specific to iOS 26 Result: You get the correct, current APIs that work with today's iOS version ```
[API with PAL](https://github.com/user-attachments/assets/5c847326-4b66-41f7-8f30-f380453dce22)
## Key Features ### 1. **OS Version Detection** (Critical!) For any OS-tied request (iOS, macOS, Windows, Android, watchOS, tvOS), `apilookup` **MUST**: - First search for the current OS version ("what is the latest iOS version 2025") - **Never** rely on the AI's training data for version numbers - Only after confirming current version, search for APIs/SDKs for that specific version ### 2. **Authoritative Sources Only** Prioritizes official documentation: - Project documentation sites - GitHub repositories - Package registries (npm, PyPI, crates.io, Maven Central, etc.) - Official blogs and release notes ### 3. **Actionable, Concise Results** - Current version numbers and release dates - Breaking changes and migration notes - Code examples and configuration options - Deprecation warnings and security advisories ## When to Use - You need current API/SDK documentation or version info - You're working with OS-specific frameworks (SwiftUI, UIKit, Jetpack Compose, etc.) - You want to verify which version supports a feature - You need migration guides or breaking change notes - You're checking for deprecations or security advisories ## Usage Examples ### OS-Specific APIs ``` use apilookup how do I add glass look to a button in swift? use apilookup what's the latest way to handle permissions in Android? use apilookup how do I use the new macOS window management APIs? ``` ### Library/Framework Versions ``` use apilookup find the latest Stripe Python SDK version and note any breaking changes since v7 use apilookup what's the current AWS CDK release and list migration steps from v2 use apilookup check the latest React version and any new hooks introduced in 2025 ``` ### Feature Compatibility ``` use apilookup does the latest TypeScript support decorators natively? use apilookup what's the current status of Swift async/await on Linux? ``` ## How It Works 1. **Receives your query** with API/SDK/framework name 2. **Injects mandatory instructions** that force current-year searches 3. **For OS-tied requests**: Requires two-step search (OS version first, then API) 4. **Returns structured guidance** with instructions for web search 5. **AI executes searches** and provides authoritative, current documentation ## Output Format The tool returns JSON with: - `status`: "web_lookup_needed" - `instructions`: Detailed search strategy and requirements - `user_prompt`: Your original request The AI then performs the actual web searches and synthesizes the results into actionable documentation. ## Codex CLI Configuration Reminder If you use PAL through the Codex CLI, the assistant needs Codex's native web-search tool to fetch current documentation. After adding the PAL MCP entry to `~/.codex/config.toml`, confirm the file also contains: ```toml [tools] web_search = true ``` If `[tools]` is missing, append the block manually. Without this flag, `apilookup` will keep requesting web searches that Codex cannot execute, and you'll see repeated attempts at using `curl` incorrectly. ================================================ FILE: docs/tools/challenge.md ================================================ # challenge - Challenge an approach or validate ideas with confidence The `challenge` tool encourages thoughtful critical thinking instead of automatic agreement with the dreaded **You're absolutely right!** responses - especially when you're not. This tool wraps your comment with instructions that prompt critical thinking and honest analysis instead of blind agreement. ## Quick Example ``` challenge but do we even need all this extra caching because it'll just slow the app down? ``` ``` challenge I don't think this approach solves my original complaint ``` Normally, your favorite coding agent will enthusiastically reply with **“You’re absolutely right!”**—then proceed to reverse the _correct_ strategy entirely, without stopping to consider that you might actually be wrong, missing the bigger picture or ignoring architectural constraints. `challenge` fixes this. Claude can even _detect_ when you're challenging something and automatically invokes this tool to ensure thoughtful analysis instead of reflexive agreement. **Without PAL:** ![without_pal@2x](https://github.com/user-attachments/assets/64f3c9fb-7ca9-4876-b687-25e847edfd87) **With PAL:** ![with_pal@2x](https://github.com/user-attachments/assets/9d72f444-ba53-4ab1-83e5-250062c6ee70) ## Why Use Challenge? AI assistants sometimes tend to agree too readily. The challenge tool helps you: - Get genuine critical evaluation of your ideas - Challenge assumptions constructively - Receive honest feedback on proposals - Validate approaches with thoughtful analysis ================================================ FILE: docs/tools/chat.md ================================================ # Chat Tool - General Development Chat & Collaborative Thinking **Your thinking partner - bounce ideas, get second opinions, brainstorm collaboratively** The `chat` tool is your collaborative thinking partner for development conversations. It's designed to help you brainstorm, validate ideas, get second opinions, and explore alternatives in a conversational format. ## Thinking Mode **Default is `medium` (8,192 tokens).** Use `low` for quick questions to save tokens, or `high` for complex discussions when thoroughness matters. ## Example Prompt ``` I need to pick between Redis and Memcached for session storage and I need an expert opinion for the project I'm working on. Take a look at the code and get an idea of what this project does, pick one of the two options and then chat with gemini pro and continue discussing pros and cons to come to a final conclusion. I need a one word verdict in the end. ```
[Chat Redis or Memcached_web.webm](https://github.com/user-attachments/assets/41076cfe-dd49-4dfc-82f5-d7461b34705d)
**Another Example**: * We ask Claude code to pick one of two frameworks, then `chat` with `gemini` to make a final decision * Gemini responds, confirming choice. We use `continuation` to ask another question using the same conversation thread * Gemini responds with explanation. We use continuation again, using `/pal:continue (MCP)` command the second time
[Chat With Gemini_web.webm](https://github.com/user-attachments/assets/37bd57ca-e8a6-42f7-b5fb-11de271e95db)
## Key Features - **Collaborative thinking partner** for your analysis and planning - **Get second opinions** on your designs and approaches - **Brainstorm solutions** and explore alternatives together - **Structured code generation**: When using GPT-5.2 or Gemini 3.0 / 2.5 Pro, get complete, production-ready implementations saved to `pal_generated.code` for your CLI to review and apply - **Validate your checklists** and implementation plans - **General development questions** and explanations - **Technology comparisons** and best practices - **Architecture and design discussions** - **File reference support**: `"Use gemini to explain this algorithm with context from algorithm.py"` - **Image support**: Include screenshots, diagrams, UI mockups for visual analysis: `"Chat with gemini about this error dialog screenshot to understand the user experience issue"` - **Dynamic collaboration**: Models can request additional files or context during the conversation if needed for a more thorough response - **Web search awareness**: Automatically identifies when online research would help and instructs Claude to perform targeted searches using continuation IDs ## Tool Parameters - `prompt`: Your question or discussion topic (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `absolute_file_paths`: Optional absolute file or directory paths for additional context - `images`: Optional images for visual context (absolute paths) - `working_directory_absolute_path`: **Required** - Absolute path to an existing directory where generated code artifacts will be saved - `temperature`: Response creativity (0-1, default 0.5) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `continuation_id`: Continue previous conversations ## Structured Code Generation When using advanced reasoning models like **GPT-5.2 Pro** or **Gemini 3.0 Pro**, the chat tool can generate complete, production-ready code implementations in a structured format. ### How It Works 1. You ask your AI agent to implement a complex new feature using `chat` with a higher-reasoning model such as **GPT-5.2 Pro** or **Gemini 3.0 Pro** 2. The model generates structured implementation and shares the complete implementation with PAL 3. PAL saves the code to `pal_generated.code` and asks AI agent to implement the plan 4. AI agent continues from the previous context, reads the file, applies the implementation ### When Code Generation Activates The structured format activates for **substantial implementation work**: - Creating new features from scratch with multiple files or significant code - Major refactoring across multiple files or large sections - Implementing new modules, components, or subsystems - Large-scale updates affecting substantial portions of the codebase - Complete rewrites of functions, algorithms, or approaches For minor changes (small tweaks, bug fixes, algorithm improvements), the model responds normally with inline code blocks. ### Example Usage ``` chat with gpt-5.2-pro and ask it to make me a standalone, classic version of the Pacman game using pygame that I can run from the commandline. Give me a single script to execute in the end with any / all dependencies setup for me. Do everything using pygame, we have no external resources / images / audio at hand. Instead of ghosts, it'll be different geometric shapes moving around in the maze that Pacman can eat (so there are no baddies). Pacman gets to eat everything including bread-crumbs and large geometric shapes but make me the classic maze / walls that it navigates within using keyboard arrow keys. ``` See the [Configuration Guide](../configuration.md#code-generation-capability) for details on the `allow_code_generation` flag. ## Usage Examples **Basic Development Chat:** ``` "Chat with pal about the best approach for user authentication in my React app" ``` **Technology Comparison:** ``` "Use flash to discuss whether PostgreSQL or MongoDB would be better for my e-commerce platform" ``` **Architecture Discussion:** ``` "Chat with pro about microservices vs monolith architecture for my project, consider scalability and team size" ``` **File Context Analysis:** ``` "Use gemini to chat about the current authentication implementation in auth.py and suggest improvements" ``` **Visual Analysis:** ``` "Chat with gemini about this UI mockup screenshot - is the user flow intuitive?" ``` ## Best Practices - **Be specific about context**: Include relevant files or describe your project scope - **Ask for trade-offs**: Request pros/cons for better decision-making - **Use conversation continuation**: Build on previous discussions with `continuation_id` - **Leverage visual context**: Include diagrams, mockups, or screenshots when discussing UI/UX - **Encourage research**: When you suspect documentation has changed, explicitly ask the assistant to confirm by requesting a web search ## When to Use Chat vs Other Tools - **Use `chat`** for: Open-ended discussions, brainstorming, getting second opinions, technology comparisons - **Use `thinkdeep`** for: Extending specific analysis, challenging assumptions, deeper reasoning - **Use `analyze`** for: Understanding existing code structure and patterns - **Use `debug`** for: Specific error diagnosis and troubleshooting ================================================ FILE: docs/tools/clink.md ================================================ # Clink Tool - CLI-to-CLI Bridge **Spawn AI subagents, connect external CLIs, orchestrate isolated contexts – all without leaving your session** The `clink` tool transforms your CLI into a multi-agent orchestrator. Launch isolated Codex instances from _within_ Codex, delegate to Gemini's 1M context, or run specialized Claude agents—all while preserving conversation continuity. Instead of context-switching or token bloat, spawn fresh subagents that handle complex tasks in isolation and return only the results you need. > **CAUTION**: Clink launches real CLI agents with relaxed permission flags (Gemini ships with `--yolo`, Codex with `--dangerously-bypass-approvals-and-sandbox`, Claude with `--permission-mode acceptEdits`) so they can edit files and run tools autonomously via MCP. If that’s more access than you want, remove those flags—the CLI can still open/read files and report findings, it just won’t auto-apply edits. You can also tighten role prompts or system prompts with stop-words/guardrails, or disable clink entirely. Otherwise, keep the shipped presets confined to workspaces you fully trust. ## Why Use Clink (CLI + Link)? ### Codex-within-Codex: The Ultimate Context Management **The Problem**: You're deep in a Codex session debugging authentication. Now you need a comprehensive security audit, but that'll consume 50K tokens of context you can't spare. **The Solution**: Spawn a fresh Codex subagent in an isolated context: ```bash clink with codex codereviewer to audit auth/ for OWASP Top 10 vulnerabilities ``` The subagent: - Launches in a **pristine context** with full token budget - Performs deep analysis using its own MCP tools and web search - Returns **only the final security report** (not intermediate steps) - Your main session stays **laser-focused** on debugging **Works with any supported CLI**: Codex can spawn Codex / Claude Code / Gemini CLI subagents, or mix and match between different CLIs. --- ### Cross-CLI Orchestration **Scenario 1**: You're in Codex and need Gemini's 1M context window to analyze a massive legacy codebase. **Without clink**: Open new terminal → run `gemini` → lose conversation context → manually copy/paste findings → context mismatch hell. **With clink**: `"clink with gemini to map dependencies across this 500-file monorepo"` – Gemini processes, returns insights, conversation flows seamlessly. **Scenario 2**: Use [`consensus`](consensus.md) to debate features with multiple models, then hand off to Gemini for implementation. ``` "Use consensus with pro and gpt5 to decide whether to add dark mode or offline support next" [consensus runs, models deliberate, recommendation emerges] Use continuation with clink - implement the recommended feature ``` Gemini receives the full conversation context from `consensus` including the consensus prompt + replies, understands the chosen feature, technical constraints discussed, and can start implementation immediately. No re-explaining, no context loss - true conversation continuity across tools and models. ## Key Features - **Stay in one CLI**: No switching between terminal sessions or losing context - **Full conversation continuity**: Gemini's responses participate in the same conversation thread - **Role-based prompts**: Pre-configured roles for planning, code review, or general questions - **Full CLI capabilities**: Gemini can use its own web search, file tools, and latest features - **Token efficiency**: File references (not full content) to conserve tokens - **Cross-tool collaboration**: Combine with other PAL tools like `planner` → `clink` → `codereview` - **Free tier available**: Gemini offers 1,000 requests/day free with a personal Google account - great for cost savings across tools ## Available Roles **Default Role** - General questions, summaries, quick answers ``` Use clink to ask gemini about the latest React 19 features ``` **Planner Role** - Strategic planning with multi-phase approach ``` clink with gemini with planner role to map out our microservices migration strategy ``` **Code Reviewer Role** - Focused code analysis with severity levels ``` Use clink codereviewer role to review auth.py for security issues ``` You can make your own custom roles in `conf/cli_clients/` or tweak any of the shipped presets. ## Tool Parameters - `prompt`: Your question or task for the external CLI (required) - `cli_name`: Which CLI to use - `gemini` (default), `claude`, `codex`, or add your own in `conf/cli_clients/` - `role`: Preset role - `default`, `planner`, `codereviewer` (default: `default`) - `files`: Optional file paths for context (references only, CLI opens files itself) - `images`: Optional image paths for visual context - `continuation_id`: Continue previous clink conversations ## Usage Examples **Architecture Planning:** ``` Use clink with gemini planner to design a 3-phase rollout plan for our feature flags system ``` **Code Review with Context:** ``` clink to gemini codereviewer: Review payment_service.py for race conditions and concurrency issues ``` **Codex Code Review:** ``` "clink with codex cli and perform a full code review using the codereview role" ``` **Quick Research Question:** ``` "Ask gemini via clink: What are the breaking changes in TypeScript 5.5?" ``` **Multi-Tool Workflow:** ``` "Use planner to outline the refactor, then clink gemini planner for validation, then codereview to verify the implementation" ``` **Leveraging Gemini's Web Search:** ``` "Clink gemini to research current best practices for Kubernetes autoscaling in 2025" ``` ## How Clink Works 1. **Your request** - You ask your current CLI to use `clink` with a specific CLI and role 2. **Background execution** - PAL spawns the configured CLI (e.g., `gemini --output-format json`) 3. **Context forwarding** - Your prompt, files (as references), and conversation history are sent as part of the prompt 4. **CLI processing** - Gemini (or other CLI) uses its own tools: web search, file access, thinking modes 5. **Seamless return** - Results flow back into your conversation with full context preserved 6. **Continuation support** - Future tools and models can reference Gemini's findings via [continuation support](../context-revival.md) within PAL. ## Best Practices - **Pre-authenticate CLIs**: Install and configure Gemini CLI first (`npm install -g @google/gemini-cli`) - **Choose appropriate roles**: Use `planner` for strategy, `codereviewer` for code, `default` for general questions - **Leverage CLI strengths**: Gemini's 1M context for large codebases, web search for current docs - **Combine with PAL tools**: Chain `clink` with `planner`, `codereview`, `debug` for powerful workflows - **File efficiency**: Pass file paths, let the CLI decide what to read (saves tokens) ## Configuration Clink configurations live in `conf/cli_clients/`. We ship presets for the supported CLIs: - `gemini.json` – runs `gemini --telemetry false --yolo -o json` - `claude.json` – runs `claude --print --output-format json --permission-mode acceptEdits --model sonnet` - `codex.json` – runs `codex exec --json --dangerously-bypass-approvals-and-sandbox` > **CAUTION**: These flags intentionally bypass each CLI's safety prompts so they can edit files or launch tools autonomously via MCP. Only enable them in trusted sandboxes and tailor role prompts or CLI configs if you need more guardrails. Each preset points to role-specific prompts in `systemprompts/clink/`. Duplicate those files to add more roles or adjust CLI flags. > **Why `--yolo` for Gemini?** The Gemini CLI currently requires automatic approvals to execute its own tools (for example `run_shell_command`). Without the flag it errors with `Tool "run_shell_command" not found in registry`. See [issue #5382](https://github.com/google-gemini/gemini-cli/issues/5382) for more details. **Adding new CLIs**: Drop a JSON config into `conf/cli_clients/`, create role prompts in `systemprompts/clink/`, and register a parser/agent if the CLI outputs a new format. ## When to Use Clink vs Other Tools - **Use `clink`** for: Leveraging external CLI capabilities (Gemini's web search, 1M context), specialized CLI features, cross-CLI collaboration - **Use `chat`** for: Direct model-to-model conversations within PAL - **Use `planner`** for: PAL's native planning workflows with step validation - **Use `codereview`** for: PAL's structured code review with severity levels ## Setup Requirements Ensure the relevant CLI is installed and configured: - [Claude Code](https://www.anthropic.com/claude-code) - [Gemini CLI](https://github.com/google-gemini/gemini-cli) - [Codex CLI](https://docs.sourcegraph.com/codex) ## Related Guides - [Chat Tool](chat.md) - Direct model conversations - [Planner Tool](planner.md) - PAL's native planning workflows - [CodeReview Tool](codereview.md) - Structured code reviews - [Context Revival](../context-revival.md) - Continuing conversations across tools - [Advanced Usage](../advanced-usage.md) - Complex multi-tool workflows ================================================ FILE: docs/tools/codereview.md ================================================ # CodeReview Tool - Professional Code Review **Comprehensive code analysis with prioritized feedback through workflow-driven investigation** The `codereview` tool provides professional code review capabilities with actionable feedback, severity-based issue prioritization, and support for various review types from quick style checks to comprehensive security audits. This workflow tool guides Claude through systematic investigation steps with forced pauses between each step to ensure thorough code examination, issue identification, and quality assessment before providing expert analysis. ## Thinking Mode **Default is `medium` (8,192 tokens).** Use `high` for security-critical code (worth the extra tokens) or `low` for quick style checks (saves ~6k tokens). ## How the Workflow Works The codereview tool implements a **structured workflow** that ensures thorough code examination: **Investigation Phase (Claude-Led):** 1. **Step 1**: Claude describes the review plan and begins systematic analysis of code structure 2. **Step 2+**: Claude examines code quality, security implications, performance concerns, and architectural patterns 3. **Throughout**: Claude tracks findings, relevant files, issues, and confidence levels 4. **Completion**: Once review is comprehensive, Claude signals completion **Expert Analysis Phase:** After Claude completes the investigation (unless confidence is **certain**): - Complete review summary with all findings and evidence - Relevant files and code patterns identified - Issues categorized by severity levels - Final recommendations based on investigation **Special Note**: If you want Claude to perform the entire review without calling another model, you can include "don't use any other model" in your prompt, and Claude will complete the full workflow independently. ## Model Recommendation This tool particularly benefits from Gemini Pro or Flash models due to their 1M context window, which allows comprehensive analysis of large codebases. Claude's context limitations make it challenging to see the "big picture" in complex projects - this is a concrete example where utilizing a secondary model with larger context provides significant value beyond just experimenting with different AI capabilities. ## Example Prompts ``` Perform a codereview with gemini pro and review auth.py for security issues and potential vulnerabilities. I need an actionable plan but break it down into smaller quick-wins that we can implement and test rapidly ``` ## Pro Tip: Multiple Parallel Reviews **You can start more than one codereview session with Claude:** ``` Start separate sub-tasks for codereview one with o3 finding critical issues and one with flash finding low priority issues and quick-wins and give me the final single combined review highlighting only the critical issues ``` The above prompt will simultaneously run two separate `codereview` tools with two separate models and combine the output into a single summary for you to consume. ## Key Features - **Issues prioritized by severity** (🔴 CRITICAL → 🟢 LOW) - **Supports specialized reviews**: security, performance, quick - **Coding standards enforcement**: `"Use gemini to review src/ against PEP8 standards"` - **Severity filtering**: `"Get gemini to review auth/ - only report critical vulnerabilities"` - **Image support**: Review code from screenshots, error dialogs, or visual bug reports: `"Review this error screenshot and the related auth.py file for potential security issues"` - **Multi-file analysis**: Comprehensive review of entire directories or codebases - **Actionable feedback**: Specific recommendations with line numbers and code examples - **Language-specific expertise**: Tailored analysis for Python, JavaScript, Java, C#, Swift, and more - **Integration issue detection**: Identifies cross-file dependencies and architectural problems - **Security vulnerability scanning**: Focused on common security patterns and anti-patterns ## Tool Parameters **Workflow Investigation Parameters (used during step-by-step process):** - `step`: Current investigation step description (required for each step) - `step_number`: Current step number in review sequence (required) - `total_steps`: Estimated total investigation steps (adjustable) - `next_step_required`: Whether another investigation step is needed - `findings`: Discoveries and evidence collected in this step (required) - `files_checked`: All files examined during investigation - `relevant_files`: Files directly relevant to the review (required in step 1) - `relevant_context`: Methods/functions/classes central to review findings - `issues_found`: Issues identified with severity levels - `confidence`: Confidence level in review completeness (exploring/low/medium/high/certain) - `images`: Visual references for review context **Initial Review Configuration (used in step 1):** - `prompt`: User's summary of what the code does, expected behavior, constraints, and review objectives (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `review_type`: full|security|performance|quick (default: full) - `focus_on`: Specific aspects to focus on (e.g., "security vulnerabilities", "performance bottlenecks") - `standards`: Coding standards to enforce (e.g., "PEP8", "ESLint", "Google Style Guide") - `severity_filter`: critical|high|medium|low|all (default: all) - `temperature`: Temperature for consistency (0-1, default 0.2) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert analysis phase (default: true, set to false to use Claude only) - `continuation_id`: Continue previous review discussions ## Review Types **Full Review (default):** - Comprehensive analysis including bugs, security, performance, maintainability - Best for new features or significant code changes **Security Review:** - Focused on security vulnerabilities and attack vectors - Checks for common security anti-patterns - Best for authentication, authorization, data handling code **Performance Review:** - Analyzes performance bottlenecks and optimization opportunities - Memory usage, algorithmic complexity, resource management - Best for performance-critical code paths **Quick Review:** - Fast style and basic issue check - Lower token usage for rapid feedback - Best for code formatting and simple validation ## Severity Levels Issues are categorized and prioritized: - **🔴 CRITICAL**: Security vulnerabilities, crashes, data corruption - **🟠 HIGH**: Logic errors, performance issues, reliability problems - **🟡 MEDIUM**: Code smells, maintainability issues, minor bugs - **🟢 LOW**: Style issues, documentation, minor improvements ## Usage Examples **Basic Security Review:** ``` "Review the authentication module in auth/ for security vulnerabilities with gemini pro" ``` **Performance-Focused Review:** ``` "Use o3 to review backend/api.py for performance issues, focus on database queries and caching" ``` **Quick Style Check:** ``` "Quick review of utils.py with flash, only report critical and high severity issues" ``` **Standards Enforcement:** ``` "Review src/ directory against PEP8 standards with gemini, focus on code formatting and structure" ``` **Visual Context Review:** ``` "Review this authentication code along with the error dialog screenshot to understand the security implications" ``` ## Best Practices - **Provide context**: Describe what the code is supposed to do and any constraints - **Use appropriate review types**: Security for auth code, performance for critical paths - **Set severity filters**: Focus on critical issues for quick wins - **Include relevant files**: Review related modules together for better context - **Use parallel reviews**: Run multiple reviews with different models for comprehensive coverage - **Follow up on findings**: Use the continuation feature to discuss specific issues in detail ## Output Format Reviews include: - **Executive Summary**: Overview of code quality and main concerns - **Detailed Findings**: Specific issues with severity levels, line numbers, and recommendations - **Quick Wins**: Easy-to-implement improvements with high impact - **Long-term Improvements**: Structural changes for better maintainability - **Security Considerations**: Specific security recommendations when relevant ## When to Use CodeReview vs Other Tools - **Use `codereview`** for: Finding bugs, security issues, performance problems, code quality assessment - **Use `analyze`** for: Understanding code structure without finding issues - **Use `debug`** for: Diagnosing specific runtime errors or exceptions - **Use `refactor`** for: Identifying structural improvements and modernization opportunities ================================================ FILE: docs/tools/consensus.md ================================================ # Consensus Tool - Multi-Model Perspective Gathering **Get diverse expert opinions from multiple AI models on technical proposals and decisions** The `consensus` tool orchestrates multiple AI models to provide diverse perspectives on your proposals, enabling structured decision-making through for/against analysis and multi-model expert opinions. ## Thinking Mode **Default is `medium` (8,192 tokens).** Use `high` for complex architectural decisions or `max` for critical strategic choices requiring comprehensive analysis. ## Model Recommendation Consensus tool uses extended reasoning models by default, making it ideal for complex decision-making scenarios that benefit from multiple perspectives and deep analysis. ## How It Works The consensus tool orchestrates multiple AI models to provide diverse perspectives on your proposals: 1. **Assign stances**: Each model can take a specific viewpoint (supportive, critical, or neutral) 2. **Gather opinions**: Models analyze your proposal from their assigned perspective with built-in common-sense guardrails 3. **Synthesize results**: Claude combines all perspectives into a balanced recommendation 4. **Natural language**: Use simple descriptions like "supportive", "critical", or "against" - the tool handles synonyms automatically ## Watch In Action The following is a hypothetical example designed to demonstrate how one consensus can be built upon another (via [continuation](../context-revival.md)). In this scenario, we start with a _blinded_ consensus, where one model is tasked with taking a **for** stance and another with an **against** stance. This approach allows us to see how each model evaluates a particular option relative to the alternative. We then conduct a second consensus — all initiated by a single prompt and orchestrated by Claude Code in this video — to gather each model’s final conclusions.
[PAL Consensus Debate](https://github.com/user-attachments/assets/76a23dd5-887a-4382-9cf0-642f5cf6219e)
## Example Prompts **For/Against Analysis:** ``` Use pal consensus with flash taking a supportive stance and pro being critical to evaluate whether we should migrate from REST to GraphQL for our API ``` **Multi-Model Technical Decision:** ``` Get consensus from o3, flash, and pro on our new authentication architecture. Have o3 focus on security implications, flash on implementation speed, and pro stay neutral for overall assessment ``` **Natural Language Stance Assignment:** ``` Use consensus tool with gemini being "for" the proposal and grok being "against" to debate whether we should adopt microservices architecture ``` ``` I want to work on module X and Y, unsure which is going to be more popular with users of my app. Get a consensus from gemini supporting the idea for implementing X, grok opposing it, and flash staying neutral ``` ## Key Features - **Stance steering**: Assign specific perspectives (for/against/neutral) to each model with intelligent synonym handling - **Custom stance prompts**: Provide specific instructions for how each model should approach the analysis - **Ethical guardrails**: Models will refuse to support truly bad ideas regardless of assigned stance - **Unknown stance handling**: Invalid stances automatically default to neutral with warning - **Natural language support**: Use terms like "supportive", "critical", "oppose", "favor" - all handled intelligently - **Sequential processing**: Reliable execution avoiding MCP protocol issues - **Focus areas**: Specify particular aspects to emphasize (e.g., 'security', 'performance', 'user experience') - **File context support**: Include relevant files for informed decision-making - **Image support**: Analyze architectural diagrams, UI mockups, or design documents - **Conversation continuation**: Build on previous consensus analysis with additional rounds - **Web search capability**: Enhanced analysis with current best practices and documentation ## Tool Parameters - `prompt`: Detailed description of the proposal or decision to analyze (required) - `models`: List of model configurations with optional stance and custom instructions (required) - `files`: Context files for informed analysis (absolute paths) - `images`: Visual references like diagrams or mockups (absolute paths) - `focus_areas`: Specific aspects to emphasize - `temperature`: Control consistency (default: 0.2 for stable consensus) - `thinking_mode`: Analysis depth (minimal/low/medium/high/max) - `continuation_id`: Continue previous consensus discussions ## Model Configuration Examples **Basic For/Against:** ```json [ {"model": "flash", "stance": "for"}, {"model": "pro", "stance": "against"} ] ``` **Custom Stance Instructions:** ```json [ {"model": "o3", "stance": "for", "stance_prompt": "Focus on implementation benefits and user value"}, {"model": "flash", "stance": "against", "stance_prompt": "Identify potential risks and technical challenges"} ] ``` **Neutral Analysis:** ```json [ {"model": "pro", "stance": "neutral"}, {"model": "o3", "stance": "neutral"} ] ``` ## Usage Examples **Architecture Decision:** ``` "Get consensus from pro and o3 on whether to use microservices vs monolith for our e-commerce platform" ``` **Technology Migration:** ``` "Use consensus with flash supporting and pro opposing to evaluate migrating from MySQL to PostgreSQL" ``` **Feature Priority:** ``` "Get consensus from multiple models on whether to prioritize mobile app vs web dashboard development first" ``` **With Visual Context:** ``` "Use consensus to evaluate this new UI design mockup - have flash support it and pro be critical" ``` ## Best Practices - **Provide detailed context**: Include project constraints, requirements, and background - **Use balanced stances**: Mix supportive and critical perspectives for thorough analysis - **Specify focus areas**: Guide models to emphasize relevant aspects (security, performance, etc.) - **Include relevant files**: Provide code, documentation, or specifications for context - **Build on discussions**: Use continuation for follow-up analysis and refinement - **Leverage visual context**: Include diagrams, mockups, or design documents when relevant ## Ethical Guardrails The consensus tool includes built-in ethical safeguards: - Models won't support genuinely harmful proposals regardless of assigned stance - Unknown or invalid stances automatically default to neutral - Warning messages for potentially problematic requests - Focus on constructive technical decision-making ## When to Use Consensus vs Other Tools - **Use `consensus`** for: Multi-perspective analysis, structured debates, major technical decisions - **Use `chat`** for: Open-ended discussions and brainstorming - **Use `thinkdeep`** for: Extending specific analysis with deeper reasoning - **Use `analyze`** for: Understanding existing systems without debate ================================================ FILE: docs/tools/debug.md ================================================ # Debug Tool - Systematic Investigation & Expert Analysis **Step-by-step investigation followed by expert debugging assistance** The `debug` workflow guides Claude through a systematic investigation process where Claude performs methodical code examination, evidence collection, and hypothesis formation across multiple steps. Once the investigation is complete, the tool provides expert analysis from the selected AI model (optionally) based on all gathered findings. ## Example Prompts ``` Get gemini to debug why my API returns 400 errors randomly with the full stack trace: [paste traceback] ``` You can also ask it to debug on its own, no external model required (**recommended in most cases**). ``` Use debug tool to find out why the app is crashing, here are some app logs [paste app logs] and a crash trace: [paste crash trace] ``` ## How It Works The debug tool implements a **systematic investigation methodology** where Claude is guided through structured debugging steps: **Investigation Phase:** 1. **Step 1**: Claude describes the issue and begins thinking deeply about possible underlying causes, side-effects, and contributing factors 2. **Step 2+**: Claude examines relevant code, traces errors, tests hypotheses, and gathers evidence 3. **Throughout**: Claude tracks findings, relevant files, methods, and evolving hypotheses with confidence levels 4. **Backtracking**: Claude can revise previous steps when new insights emerge 5. **Completion**: Once investigation is thorough, Claude signals completion **Expert Analysis Phase:** After Claude completes the investigation, it automatically calls the selected AI model with (unless confidence is **certain**, in which case expert analysis is bypassed): - Complete investigation summary with all steps and findings - Relevant files and methods identified during investigation - Final hypothesis and confidence assessment - Error context and supporting evidence - Visual debugging materials if provided This structured approach ensures Claude performs methodical groundwork before expert analysis, resulting in significantly better debugging outcomes and more efficient token usage. **Special Note**: If you want Claude to perform the entire debugging investigation without calling another model, you can include "don't use any other model" in your prompt, and Claude will complete the full workflow independently. ## Key Features - **Multi-step investigation process** with evidence collection and hypothesis evolution - **Systematic code examination** with file and method tracking throughout investigation - **Confidence assessment and revision** capabilities for investigative steps - **Backtracking support** to revise previous steps when new insights emerge - **Expert analysis integration** that provides final debugging recommendations based on complete investigation - **Error context support**: Stack traces, logs, and runtime information - **Visual debugging**: Include error screenshots, stack traces, console output - **Conversation threading**: Continue investigations across multiple sessions - **Large context analysis**: Handle extensive log files and multiple related code files - **Multi-language support**: Debug issues across Python, JavaScript, Java, C#, Swift, and more - **Web search integration**: Identifies when additional research would help solve problems ## Tool Parameters **Investigation Step Parameters:** - `step`: Current investigation step description (required) - `step_number`: Current step number in investigation sequence (required) - `total_steps`: Estimated total investigation steps (adjustable as process evolves) - `next_step_required`: Whether another investigation step is needed - `findings`: Discoveries and evidence collected in this step (required) - `files_checked`: All files examined during investigation (tracks exploration path) - `relevant_files`: Files directly tied to the root cause or its effects - `relevant_methods`: Specific methods/functions involved in the issue - `hypothesis`: Current best guess about the underlying cause - `confidence`: Confidence level in current hypothesis (exploring/low/medium/high/certain) - `continuation_id`: Thread ID for continuing investigations across sessions - `images`: Visual debugging materials (error screenshots, logs, etc.) **Model Selection:** - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert analysis phase (default: true, set to false to use Claude only) ## Usage Examples **Error Debugging:** ``` Debug this TypeError: 'NoneType' object has no attribute 'split' in my parser.py ``` **With Stack Trace:** ``` Use gemini to debug why my API returns 500 errors with this stack trace: [paste full traceback] ``` **With File Context:** ``` Debug without using external model, the authentication failure in auth.py and user_model.py ``` **Performance Debugging:** ``` Debug without using external model to find out why the app is consuming excessive memory during bulk edit operations ``` **Runtime Environment Issues:** ``` Debug deployment issues with server startup failures, here's the runtime info: [environment details] ``` ## Investigation Methodology The debug tool enforces a thorough, structured investigation process: **Step-by-Step Investigation (Claude-Led):** 1. **Initial Problem Description:** Claude describes the issue and begins thinking about possible causes, side-effects, and contributing factors 2. **Code Examination:** Claude systematically examines relevant files, traces execution paths, and identifies suspicious patterns 3. **Evidence Collection:** Claude gathers findings, tracks files checked, and identifies methods/functions involved 4. **Hypothesis Formation:** Claude develops working theories about the root cause with confidence assessments 5. **Iterative Refinement:** Claude can backtrack and revise previous steps as understanding evolves 6. **Investigation Completion:** Claude signals when sufficient evidence has been gathered **Expert Analysis Phase (Another AI Model When Used):** Once investigation is complete, the selected AI model performs: - **Root Cause Analysis:** Deep analysis of all investigation findings and evidence - **Solution Recommendations:** Specific fixes with implementation guidance - **Prevention Strategies:** Measures to avoid similar issues in the future - **Testing Approaches:** Validation methods for proposed solutions **Key Benefits:** - **Methodical Evidence Collection:** Ensures no critical information is missed - **Progressive Understanding:** Hypotheses evolve as investigation deepens - **Complete Context:** Expert analysis receives full investigation history - **Efficient Token Usage:** Structured approach prevents redundant back-and-forth ## Debugging Categories **Runtime Errors:** - Exceptions and crashes - Null pointer/reference errors - Type errors and casting issues - Memory leaks and resource exhaustion **Logic Errors:** - Incorrect algorithm implementation - Off-by-one errors and boundary conditions - State management issues - Race conditions and concurrency bugs **Integration Issues:** - API communication failures - Database connection problems - Third-party service integration - Configuration and environment issues **Performance Problems:** - Slow response times - Memory usage spikes - CPU-intensive operations - I/O bottlenecks ## Best Practices **For Investigation Steps:** - **Be thorough in step descriptions**: Explain what you're examining and why - **Track all files examined**: Include even files that don't contain the bug (tracks investigation path) - **Document findings clearly**: Summarize discoveries, suspicious patterns, and evidence - **Evolve hypotheses**: Update theories as investigation progresses - **Use backtracking wisely**: Revise previous steps when new insights emerge - **Include visual evidence**: Screenshots, error dialogs, console output **For Initial Problem Description:** - **Provide complete error context**: Full stack traces, error messages, and logs - **Describe expected vs actual behavior**: Clear symptom description - **Include environment details**: Runtime versions, configuration, deployment context - **Mention previous attempts**: What debugging steps have already been tried - **Be specific about occurrence**: When, where, and how the issue manifests ## Advanced Features **Large Log Analysis:** With models like Gemini Pro (1M context), you can include extensive log files for comprehensive analysis: ``` "Debug application crashes using these large log files: app.log, error.log, system.log" ``` **Multi-File Investigation:** Analyze multiple related files simultaneously to understand complex issues: ``` "Debug the data processing pipeline issues across processor.py, validator.py, and output_handler.py" ``` **Web Search Integration:** The tool can recommend specific searches for error messages, known issues, or documentation: ``` After analysis: "Recommended searches for Claude: 'Django 4.2 migration error specific_error_code', 'PostgreSQL connection pool exhaustion solutions'" ``` ## When to Use Debug vs Other Tools - **Use `debug`** for: Specific runtime errors, exceptions, crashes, performance issues requiring systematic investigation - **Use `codereview`** for: Finding potential bugs in code without specific errors or symptoms - **Use `analyze`** for: Understanding code structure and flow without troubleshooting specific issues - **Use `precommit`** for: Validating changes before commit to prevent introducing bugs ## Investigation Example **Step 1:** "The user authentication fails intermittently with no error logs. I need to investigate the auth flow and identify where failures might occur silently." **Step 2:** "Examined auth.py and found three potential failure points: token validation, database connectivity, and session management. No obvious bugs yet but need to trace execution flow." **Step 3:** "Found suspicious async/await pattern in session_manager.py lines 45-67. The await might be missing exception handling. This could explain silent failures." **Completion:** Investigation reveals likely root cause in exception handling, ready for expert analysis with full context. ================================================ FILE: docs/tools/docgen.md ================================================ # DocGen Tool - Comprehensive Documentation Generation **Generates comprehensive documentation with complexity analysis through workflow-driven investigation** The `docgen` tool creates thorough documentation by analyzing your code structure, understanding function complexity, and documenting gotchas and unexpected behaviors that developers need to know. This workflow tool guides Claude through systematic investigation of code functionality, architectural patterns, and documentation needs across multiple steps before generating comprehensive documentation with complexity analysis and call flow information. ## How the Workflow Works The docgen tool implements a **structured workflow** for comprehensive documentation generation: **Investigation Phase (Claude-Led):** 1. **Step 1 (Discovery)**: Claude discovers ALL files needing documentation and reports exact count 2. **Step 2+ (Documentation)**: Claude documents files one-by-one with complete coverage validation 3. **Throughout**: Claude tracks progress with counters and enforces modern documentation styles 4. **Completion**: Only when all files are documented (num_files_documented = total_files_to_document) **Documentation Generation Phase:** After Claude completes the investigation: - Complete documentation strategy with style consistency - Function/method documentation with complexity analysis - Call flow and dependency documentation - Gotchas and unexpected behavior documentation - Final polished documentation following project standards This workflow ensures methodical analysis before documentation generation, resulting in more comprehensive and valuable documentation. ## Model Recommendation Documentation generation excels with analytical models like Gemini Pro or O3, which can understand complex code relationships, identify non-obvious behaviors, and generate thorough documentation that covers gotchas and edge cases. The combination of large context windows and analytical reasoning enables generation of documentation that helps prevent integration issues and developer confusion. ## Example Prompts **Basic Usage:** ``` "Use pal to generate documentation for the UserManager class" "Document the authentication module with complexity analysis using gemini pro" "Add comprehensive documentation to all methods in src/payment_processor.py" ``` ## Key Features - **Systematic file-by-file approach** - Complete documentation with progress tracking and validation - **Modern documentation styles** - Enforces /// for Objective-C/Swift, /** */ for Java/JavaScript, etc. - **Complexity analysis** - Big O notation for algorithms and performance characteristics - **Call flow documentation** - Dependencies and method relationships - **Counter-based completion** - Prevents stopping until all files are documented - **Large file handling** - Systematic portion-by-portion documentation for comprehensive coverage - **Final verification scan** - Mandatory check to ensure no functions are missed - **Bug tracking** - Surfaces code issues without altering logic - **Configuration parameters** - Control complexity analysis, call flow, and inline comments ## Tool Parameters **Workflow Parameters (used during step-by-step process):** - `step`: Current step description - discovery phase (step 1) or documentation phase (step 2+) - `step_number`: Current step number in documentation sequence (required) - `total_steps`: Dynamically calculated as 1 + total_files_to_document - `next_step_required`: Whether another step is needed - `findings`: Discoveries about code structure and documentation needs (required) - `relevant_files`: Files being actively documented in current step - `num_files_documented`: Counter tracking completed files (required) - `total_files_to_document`: Total count of files needing documentation (required) **Configuration Parameters (required fields):** - `document_complexity`: Include Big O complexity analysis (default: true) - `document_flow`: Include call flow and dependency information (default: true) - `update_existing`: Update existing documentation when incorrect/incomplete (default: true) - `comments_on_complex_logic`: Add inline comments for complex algorithmic steps (default: true) ## Usage Examples **Class Documentation:** ``` "Generate comprehensive documentation for the PaymentProcessor class including complexity analysis" ``` **Module Documentation:** ``` "Document all functions in the authentication module with call flow information" ``` **API Documentation:** ``` "Create documentation for the REST API endpoints in api/users.py with parameter gotchas" ``` **Algorithm Documentation:** ``` "Document the sorting algorithm in utils/sort.py with Big O analysis and edge cases" ``` **Library Documentation:** ``` "Add comprehensive documentation to the utility library with usage examples and warnings" ``` ## Documentation Standards **Function/Method Documentation:** - Parameter types and descriptions - Return value documentation with types - Algorithmic complexity analysis (Big O notation) - Call flow and dependency information - Purpose and behavior explanation - Exception types and conditions **Gotchas and Edge Cases:** - Parameter combinations that produce unexpected results - Hidden dependencies on global state or environment - Order-dependent operations where sequence matters - Performance implications and bottlenecks - Thread safety considerations - Platform-specific behavior differences **Code Quality Documentation:** - Inline comments for complex logic - Design pattern explanations - Architectural decision rationale - Usage examples and best practices ## Documentation Features Generated **Complexity Analysis:** - Time complexity (Big O notation) - Space complexity when relevant - Worst-case, average-case, and best-case scenarios - Performance characteristics and bottlenecks **Call Flow Documentation:** - Which methods/functions this code calls - Which methods/functions call this code - Key dependencies and interactions - Side effects and state modifications - Data flow through functions **Gotchas Documentation:** - Non-obvious parameter interactions - Hidden state dependencies - Silent failure conditions - Resource management requirements - Version compatibility issues - Platform-specific behaviors ## Incremental Documentation Approach **Key Benefits:** - **Immediate value delivery** - Code becomes more maintainable right away - **Iterative improvement** - Pattern recognition across multiple analysis rounds - **Quality validation** - Testing documentation effectiveness during workflow - **Reduced cognitive load** - Focus on one function/method at a time **Workflow Process:** 1. **Analyze and Document**: Examine each function and immediately add documentation 2. **Continue Analyzing**: Move to next function while building understanding 3. **Refine and Standardize**: Review and improve previously added documentation ## Language Support **Modern Documentation Style Enforcement:** - **Python**: Triple-quote docstrings with type hints - **Objective-C**: /// comments - **Swift**: /// comments - **JavaScript/TypeScript**: /** */ JSDoc style - **Java**: /** */ Javadoc style - **C#**: /// XML documentation comments - **C/C++**: /// for documentation comments - **Go**: // comments above functions/types - **Rust**: /// for documentation comments ## Documentation Quality Features **Comprehensive Coverage:** - All public methods and functions - Complex private methods requiring explanation - Class and module-level documentation - Configuration and setup requirements **Developer-Focused:** - Clear explanations of non-obvious behavior - Usage examples for complex APIs - Warning about common pitfalls - Integration guidance and best practices **Maintainable Format:** - Consistent documentation style - Appropriate level of detail - Cross-references and links - Version and compatibility notes ## Best Practices - **Use systematic approach**: Tool now documents all files with progress tracking and validation - **Trust the counters**: Tool prevents premature completion until all files are documented - **Large files handled**: Tool automatically processes large files in systematic portions - **Modern styles enforced**: Tool ensures correct documentation style per language - **Configuration matters**: Enable complexity analysis and call flow for comprehensive docs - **Bug tracking**: Tool surfaces issues without altering code - review findings after completion ## When to Use DocGen vs Other Tools - **Use `docgen`** for: Creating comprehensive documentation, adding missing docs, improving existing documentation - **Use `analyze`** for: Understanding code structure without generating documentation - **Use `codereview`** for: Reviewing code quality including documentation completeness - **Use `refactor`** for: Restructuring code before documentation (cleaner code = better docs) ================================================ FILE: docs/tools/listmodels.md ================================================ # ListModels Tool - List Available Models **Display all available AI models organized by provider** The `listmodels` tool shows which providers are configured, available models, their aliases, context windows, and capabilities. This is useful for understanding what models can be used and their characteristics. ## Usage ``` "Use pal to list available models" ``` ## Key Features - **Provider organization**: Shows all configured providers and their status - **Model capabilities**: Context windows, thinking mode support, and special features - **Alias mapping**: Shows shorthand names and their full model mappings - **Configuration status**: Indicates which providers are available based on API keys - **Context window information**: Helps you choose models based on your content size needs - **Capability overview**: Understanding which models support extended thinking, vision, etc. ## Output Information The tool displays: **Provider Status:** - Which providers are configured and available - API key status (without revealing the actual keys) - Provider priority order **Model Details:** - Full model names and their aliases - Context window sizes (tokens) - Special capabilities (thinking modes, vision support, etc.) - Provider-specific features **Capability Summary:** - Which models support extended thinking - Vision-capable models for image analysis - Models with largest context windows - Fastest models for quick tasks ## Example Output ``` 📋 Available Models by Provider 🔹 Google (Gemini) - ✅ Configured • pro (gemini-2.5-pro) - 1M context, thinking modes • flash (gemini-2.0-flash-experimental) - 1M context, ultra-fast 🔹 OpenAI - ✅ Configured • o3 (o3) - 200K context, strong reasoning • o3-mini (o3-mini) - 200K context, balanced • o4-mini (o4-mini) - 200K context, latest reasoning 🔹 Custom/Local - ✅ Configured • local-llama (llama3.2) - 128K context, local inference • Available at: http://localhost:11434/v1 🔹 OpenRouter - ❌ Not configured Set OPENROUTER_API_KEY to enable access to Claude, GPT-4, and more models ``` ## When to Use ListModels - **Model selection**: When you're unsure which models are available - **Capability checking**: To verify what features each model supports - **Configuration validation**: To confirm your API keys are working - **Context planning**: To choose models based on content size requirements - **Performance optimization**: To select the right model for speed vs quality trade-offs ## Configuration Dependencies The available models depend on your configuration: **API Keys Required:** - `GEMINI_API_KEY` - Enables Gemini Pro and Flash models - `OPENAI_API_KEY` - Enables OpenAI O3, O4-mini, and GPT models - `OPENROUTER_API_KEY` - Enables access to multiple providers through OpenRouter - `CUSTOM_API_URL` - Enables local/custom models (Ollama, vLLM, etc.) **Model Restrictions:** If you've set model usage restrictions via environment variables, the tool will show: - Which models are allowed vs restricted - Active restriction policies - How to modify restrictions ## Tool Parameters This tool requires no parameters - it simply queries the server configuration and displays all available information. ## Best Practices - **Check before planning**: Use this tool to understand your options before starting complex tasks - **Verify configuration**: Confirm your API keys are working as expected - **Choose appropriate models**: Match model capabilities to your specific needs - **Understand limits**: Be aware of context windows when working with large files ## When to Use ListModels vs Other Tools - **Use `listmodels`** for: Understanding available options and model capabilities - **Use `chat`** for: General discussions about which model to use for specific tasks - **Use `version`** for: Server configuration and version information - **Use other tools** for: Actual analysis, debugging, or development work ================================================ FILE: docs/tools/planner.md ================================================ # Planner Tool - Interactive Step-by-Step Planning **Break down complex projects into manageable, structured plans through step-by-step thinking** The `planner` tool helps you break down complex ideas, problems, or projects into multiple manageable steps. Perfect for system design, migration strategies, architectural planning, and feature development with branching and revision capabilities. ## How It Works The planner tool enables step-by-step thinking with incremental plan building: 1. **Start with step 1**: Describe the task or problem to plan 2. **Continue building**: Add subsequent steps, building the plan piece by piece 3. **Revise when needed**: Update earlier decisions as new insights emerge 4. **Branch alternatives**: Explore different approaches when multiple options exist 5. **Continue across sessions**: Resume planning later with full context ## Example Prompts #### Pro Tip Claude supports `sub-tasks` where it will spawn and run separate background tasks. You can ask Claude to run PAL's planner with two separate ideas. Then when it's done, use PAL's `consensus` tool to pass the entire plan and get expert perspective from two powerful AI models on which one to work on first! Like performing **AB** testing in one-go without the wait! ``` Create two separate sub-tasks: in one, using planner tool show me how to add natural language support to my cooking app. In the other sub-task, use planner to plan how to add support for voice notes to my cooking app. Once done, start a consensus by sharing both plans to o3 and flash to give me the final verdict. Which one do I implement first? ``` ``` Use pal's planner and show me how to add real-time notifications to our mobile app ``` ``` Using the planner tool, show me how to add CoreData sync to my app, include any sub-steps ``` ## Key Features - **Step-by-step breakdown**: Build plans incrementally with full context awareness - **Branching support**: Explore alternative approaches when needed - **Revision capabilities**: Update earlier decisions as new insights emerge - **Multi-session continuation**: Resume planning across multiple sessions with context - **Dynamic adjustment**: Modify step count and approach as planning progresses - **Visual presentation**: ASCII charts, diagrams, and structured formatting - **Professional output**: Clean, structured plans without emojis or time estimates ## More Examples ``` Using planner, plan the architecture for a new real-time chat system with 100k concurrent users ``` ``` Create a plan using pal for migrating our React app from JavaScript to TypeScript ``` ``` Develop a plan using pal for implementing CI/CD pipelines across our development teams ``` ## Best Practices - **Start broad, then narrow**: Begin with high-level strategy, then add implementation details - **Include constraints**: Consider technical, organizational, and resource limitations - **Plan for validation**: Include testing and verification steps - **Think about dependencies**: Identify what needs to happen before each step - **Consider alternatives**: Note when multiple approaches are viable - **Enable continuation**: Use continuation_id for multi-session planning ## Continue With a New Plan Like all other tools in PAL, you can `continue` with a new plan using the output from a previous plan by simply saying ``` Continue with pal's consensus tool and find out what o3:for and flash:against think of the plan ``` You can mix and match and take one output and feed it into another, continuing from where you left off using a different tool / model combination. ================================================ FILE: docs/tools/precommit.md ================================================ # PreCommit Tool - Pre-Commit Validation **Comprehensive review of staged/unstaged git changes across multiple repositories through workflow-driven investigation** The `precommit` tool provides thorough validation of git changes before committing, ensuring code quality, requirement compliance, and preventing regressions across multiple repositories. This workflow tool guides Claude through systematic investigation of git changes, repository status, and file modifications across multiple steps before providing expert validation. ## Thinking Mode **Default is `medium` (8,192 tokens).** Use `high` or `max` for critical releases when thorough validation justifies the token cost. ## How the Workflow Works The precommit tool implements a **structured workflow** for comprehensive change validation: **Investigation Phase (Claude-Led):** 1. **Step 1**: Claude states validation strategy using direct statements ("I will examine..." not "Let me examine...") 2. **Step 2**: Claude examines changes, diffs, dependencies with MANDATORY deeper investigation 3. **Step 3+**: Claude performs final verification (minimum 3 steps enforced) 4. **Throughout**: Claude tracks findings, relevant files, and issues with CRITICAL step validation 5. **Completion**: Only after minimum steps, Claude signals completion and creates changeset file **For Continuations**: When using `continuation_id` with external validation, Claude will immediately gather git changes and proceed to expert analysis without minimum step requirements. **Expert Validation Phase:** After Claude completes the investigation (unless precommit_type is **internal**): - Complete summary of all changes and their context - Potential issues and regressions identified - Requirement compliance assessment - Final recommendations for safe commit **Special Notes**: - Default validation type is **external** (uses expert model for additional review) - To skip expert validation, explicitly request "don't use any other model" or set precommit_type to "internal" - **CRITICAL**: Minimum 3 steps are enforced - tool will prevent setting `next_step_required=false` before final step - **MANDATORY**: Changeset file (pal_precommit.changeset) must be created for external validation ## Model Recommendation Pre-commit validation benefits significantly from models with extended context windows like Gemini Pro, which can analyze extensive changesets across multiple files and repositories simultaneously. This comprehensive view enables detection of cross-file dependencies, architectural inconsistencies, and integration issues that might be missed when reviewing changes in isolation due to context constraints. ## Visual Example
**Prompt Used:** ``` Now use gemini and perform a review and precommit and ensure original requirements are met, no duplication of code or logic, everything should work as expected ``` How beautiful is that? Claude used `precommit` twice and `codereview` once and actually found and fixed two critical errors before commit! ### Real-world Example Here's an example where Claude was made to go through changes *it* was asked to make. Its recommendation to **Publish all Changes** was made after a rigorous set of prompts where it examined the small set of changes from different angles (performance, bugs, anti-patterns etc). After confirming that unit tests were passing, it went ahead with an initial review with O3; came back clean - all good to go. O3 was happy, everything looked great. Then it shared the changes and relevant code with Gemini 3.0 Pro - the following is the outcome: ``` ⏺ pal:precommit (MCP)(step: "FINAL VALIDATION COMPLETE. All changes have been thoroughly analyzed and are ready for commit. The modifications successfully fix 4 GitHub CI test failures with high-quality, defensive implementations that maintain architectural integrity and coding standards. No issues or regressions detected.", step_number: 4, total_steps: 4, next_step_required: false, findings: "COMMIT RECOMMENDATION: ✅ APPROVED\n\nSUMMARY OF CHANGES:\n✅ Fixed 4 GitHub CI test failures:\n - test_openrouter_with_restrictions (Mock iteration issue) \n - test_execute_with_branching (Missing metadata)\n - test_execute_with_revision (Missing metadata)\n - test_metadata_preserved (Provider type AttributeError)\n\n✅ IMPLEMENTATION QUALITY VERIFIED:\n - Defensive programming with proper type checking\n - Error handling with meaningful fallbacks\n - Backward compatibility maintained\n - Coding standards followed (no hasattr/getattr in changes)\n - Architectural patterns preserved\n\n✅ COMPREHENSIVE TESTING:\n - All originally failing tests pass\n - Affected functionality: 37/37 tests pass\n - Provider systems: 16/16 tests pass \n - Module imports work correctly\n - No regressions detected\n\n✅ CHANGE IMPACT ASSESSMENT:\n - Minimal, targeted changes\n - No security or performance concerns\n - Maintains API compatibility\n - Changes are well-documented with clear comments\n\nRECOMMENDATION: These changes are safe to commit and will resolve the GitHub CI failures without any negative impact.", ⎿  { "status": "calling_expert_analysis", "step_number": 4, … +169 lines (ctrl+r to expand) ⏺ 🔍 Pre-Commit Validation Complete Based on comprehensive analysis using both O3 and Gemini 3.0 Pro models, I can provide the following assessment: ✅ APPROVED WITH CRITICAL FIX REQUIRED The changes successfully fix all 4 GitHub CI test failures and are well-implemented with defensive programming patterns. However, the expert analysis identified one critical issue that must be addressed before commit: 🚨 [HIGH PRIORITY] Critical Issue Found Inconsistent Provider Handling in tools/shared/base_tool.py - Location: tools/shared/base_tool.py inside _create_continuation_offer_response - Problem: The defensive provider handling pattern was applied in 2 locations but missed a 3rd location - Impact: Risk of AttributeError: 'str' object has no attribute 'get_provider_type' in continuation scenarios - Fix Required: Apply the same defensive pattern to the missed location ``` A subtle but critical issue spotted by Gemini that both Claude + O3 missed. This is the power of running these workflows with PAL. ## Example Prompts ``` Use pal and perform a thorough precommit ensuring there aren't any new regressions or bugs introduced ``` ## Key Features - **Recursive repository discovery** - finds all git repos including nested ones - **Validates changes against requirements** - ensures implementation matches intent - **Detects incomplete changes** - finds added functions never called, missing tests, etc. - **Multi-repo support** - reviews changes across multiple repositories in one go - **Configurable scope** - review staged, unstaged, or compare against branches - **Security focused** - catches exposed secrets, vulnerabilities in new code - **Smart truncation** - handles large diffs without exceeding context limits - **Cross-file dependency analysis** - identifies breaking changes across modules - **Test coverage validation** - ensures new code has appropriate test coverage - **Regression detection** - compares against requirements to prevent scope creep ## Tool Parameters **Workflow Investigation Parameters (used during step-by-step process):** - `step`: Technical brief to another engineer using direct statements (required, FORBIDDEN: large code snippets) - `step_number`: Current step number in validation sequence (required, starts at 1) - `total_steps`: Estimated total investigation steps (minimum 3 enforced) - `next_step_required`: Whether another investigation step is needed (CRITICAL: must be true until final step) - `findings`: Specific discoveries and evidence from actual investigation (required, no vague language) - `files_checked`: All files examined during investigation - `relevant_files`: Files directly relevant to the changes - `relevant_context`: Methods/functions/classes affected by changes - `issues_found`: Issues identified with severity levels - `precommit_type`: Type of validation to perform (external/internal, default: external - ALWAYS use external unless explicitly told otherwise) - `images`: Screenshots of requirements, design mockups for validation **Initial Configuration (used in step 1):** - `path`: Starting directory to search for repos (REQUIRED for step 1, must be absolute path) - `prompt`: The original user request description for the changes (required for context) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `compare_to`: Compare against a branch/tag instead of local changes (optional) - `severity_filter`: critical|high|medium|low|all (default: all) - `include_staged`: Include staged changes in the review (default: true) - `include_unstaged`: Include uncommitted changes in the review (default: true) - `focus_on`: Specific aspects to focus on - `temperature`: Temperature for response (default: 0.2) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert validation phase (default: true, set to false to use Claude only) - `continuation_id`: Continue previous validation discussions ## Usage Examples **Basic Pre-commit Validation:** ``` "Use pal precommit to validate my changes before committing" ``` **Security-Focused Validation:** ``` "Perform precommit security review with gemini pro on the authentication changes" ``` **Multi-Repository Validation:** ``` "Validate changes across all repositories in this workspace with o3" ``` **Against Specific Branch:** ``` "Compare current changes against main branch with precommit using gemini pro" ``` **With Requirements Context:** ``` "Precommit validation ensuring the new payment feature meets requirements in FEATURE_SPEC.md" ``` ## Validation Scope The tool automatically discovers and validates: **Repository Discovery:** - Searches recursively for all `.git` directories - Handles nested repositories and submodules - Configurable search depth to prevent excessive recursion **Change Analysis:** - Staged changes (`git diff --cached`) - Unstaged changes (`git diff`) - Untracked files that should be added - Deleted files and their impact **Cross-Repository Impact:** - Shared dependencies between repositories - API contract changes that affect other repos - Configuration changes with system-wide impact ## Validation Categories **Completeness Checks:** - New functions/classes have corresponding tests - Documentation updated for API changes - Configuration files updated as needed - Migration scripts for database changes **Quality Assurance:** - Code follows project standards - No obvious bugs or logical errors - Performance implications considered - Security vulnerabilities addressed **Requirement Compliance:** - Implementation matches original requirements - No scope creep or unauthorized changes - All acceptance criteria met - Edge cases properly handled **Integration Safety:** - Breaking changes properly documented - Backward compatibility maintained where required - Dependencies correctly updated - Environment-specific changes validated ## Best Practices - **Provide clear context**: Include the original requirements or feature description - **Use for significant changes**: Most valuable for features, refactoring, or security updates - **Review before final commit**: Catch issues before they enter the main branch - **Include visual context**: Screenshots of requirements or expected behavior - **Focus validation scope**: Use `focus_on` parameter for specific concerns - **Multi-stage validation**: Use continuation for iterative improvement ## Output Format Validation results include: - **Change Summary**: Overview of what was modified across repositories - **Requirement Compliance**: How well changes match original intent - **Completeness Assessment**: Missing tests, documentation, or related changes - **Security Review**: Potential vulnerabilities or exposed secrets - **Integration Impact**: Cross-repository and cross-module effects - **Recommendations**: Specific actions before committing ## When to Use PreCommit vs Other Tools - **Use `precommit`** for: Validating changes before git commit, ensuring requirement compliance - **Use `codereview`** for: General code quality assessment without git context - **Use `debug`** for: Diagnosing specific runtime issues - **Use `analyze`** for: Understanding existing code without validation context ================================================ FILE: docs/tools/refactor.md ================================================ # Refactor Tool - Intelligent Code Refactoring **Comprehensive refactoring analysis with top-down decomposition strategy through workflow-driven investigation** The `refactor` tool provides intelligent code refactoring recommendations with a focus on top-down decomposition and systematic code improvement. This workflow tool enforces systematic investigation of code smells, decomposition opportunities, and modernization possibilities across multiple steps, ensuring thorough analysis before providing expert refactoring recommendations with precise implementation guidance. ## Thinking Mode **Default is `medium` (8,192 tokens).** Use `high` for complex legacy systems (worth the investment for thorough refactoring plans) or `max` for extremely complex codebases requiring deep analysis. ## How the Workflow Works The refactor tool implements a **structured workflow** for systematic refactoring analysis: **Investigation Phase (Claude-Led):** 1. **Step 1**: Claude describes the refactoring plan and begins analyzing code structure 2. **Step 2+**: Claude examines code smells, decomposition opportunities, and modernization possibilities 3. **Throughout**: Claude tracks findings, relevant files, refactoring opportunities, and confidence levels 4. **Completion**: Once investigation is thorough, Claude signals completion **Expert Analysis Phase:** After Claude completes the investigation (unless confidence is **complete**): - Complete refactoring opportunity summary - Prioritized recommendations by impact - Precise implementation guidance with line numbers - Final expert assessment for refactoring strategy This workflow ensures methodical investigation before expert recommendations, resulting in more targeted and valuable refactoring plans. ## Model Recommendation The refactor tool excels with models that have large context windows like Gemini Pro (1M tokens), which can analyze entire files and complex codebases simultaneously. This comprehensive view enables detection of cross-file dependencies, architectural patterns, and refactoring opportunities that might be missed when reviewing code in smaller chunks due to context constraints. ## Example Prompts ``` "Use gemini pro to decompose my_crazy_big_class.m into smaller extensions" "Using pal's refactor decompose the all_in_one_sync_code.swift into maintainable extensions" ``` 💡**Example of a powerful prompt** to get the best out of both Claude + Flash's 1M Context: ``` "First, think about how the authentication module works, find related classes and find any code smells, then using pal's refactor ask flash to confirm your findings but ask it to find additional code smells and any other quick-wins and then fix these issues" ``` This results in Claude first performing its own expert analysis, encouraging it to think critically and identify links within the project code. It then prompts `flash` to review the same code with a hint—preventing it from duplicating Claude's findings and encouraging it to explore other areas that Claude did *not* discover. ## Key Features - **Intelligent prioritization** - Will refuse to work on low priority issues if code is unwieldy large and requires decomposition first, helps identify poorly managed classes and files that need structural improvements before detail work - **Top-down decomposition strategy** - Analyzes file → class → function levels systematically - **Four refactor types**: `codesmells` (detect anti-patterns), `decompose` (break down large components), `modernize` (update language features), `organization` (improve structure) - **Precise line-number references** - Provides exact line numbers for Claude to implement changes - **Language-specific guidance** - Tailored suggestions for Python, JavaScript, Java, C#, Swift, and more - **Style guide integration** - Uses existing project files as pattern references - **Conservative approach** - Careful dependency analysis to prevent breaking changes - **Multi-file analysis** - Understands cross-file relationships and dependencies - **Priority sequencing** - Recommends implementation order for refactoring changes - **Image support**: Analyze code architecture diagrams, legacy system charts: `"Refactor this legacy module using gemini pro with the current architecture diagram"` ## Refactor Types (Progressive Priority System) **1. `decompose` (CRITICAL PRIORITY)** - Context-aware decomposition with adaptive thresholds: **AUTOMATIC decomposition** (CRITICAL severity - blocks all other refactoring): - Files >15,000 LOC, Classes >3,000 LOC, Functions >500 LOC **EVALUATE decomposition** (contextual severity - intelligent assessment): - Files >5,000 LOC, Classes >1,000 LOC, Functions >150 LOC - Only recommends if genuinely improves maintainability - Respects legacy stability, domain complexity, performance constraints - Considers legitimate cases where size is justified (algorithms, state machines, generated code) **2. `codesmells`** - Applied only after decomposition is complete: - Detect long methods, complex conditionals, duplicate code, magic numbers, poor naming **3. `modernize`** - Applied only after decomposition is complete: - Update to modern language features (f-strings, async/await, etc.) **4. `organization`** - Applied only after decomposition is complete: - Improve logical grouping, separation of concerns, module structure **Progressive Analysis:** The tool performs a top-down check (worse → bad → better) and refuses to work on lower-priority issues if critical decomposition is needed first. It understands that massive files and classes create cognitive overload that must be addressed before detail work can be effective. Legacy code that cannot be safely decomposed is handled with higher tolerance thresholds and context-sensitive exemptions. ## Tool Parameters **Workflow Investigation Parameters (used during step-by-step process):** - `step`: Current investigation step description (required for each step) - `step_number`: Current step number in refactoring sequence (required) - `total_steps`: Estimated total investigation steps (adjustable) - `next_step_required`: Whether another investigation step is needed - `findings`: Discoveries and refactoring opportunities in this step (required) - `files_checked`: All files examined during investigation - `relevant_files`: Files directly needing refactoring (required in step 1) - `relevant_context`: Methods/functions/classes requiring refactoring - `issues_found`: Refactoring opportunities with severity and type - `confidence`: Confidence level in analysis completeness (exploring/incomplete/partial/complete) - `hypothesis`: Current assessment of refactoring priorities **Initial Configuration (used in step 1):** - `prompt`: Description of refactoring goals, context, and specific areas of focus (required) - `refactor_type`: codesmells|decompose|modernize|organization (default: codesmells) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `focus_areas`: Specific areas to focus on (e.g., 'performance', 'readability', 'maintainability', 'security') - `style_guide_examples`: Optional existing code files to use as style/pattern reference (absolute paths) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert analysis phase (default: true, set to false to use Claude only) - `continuation_id`: Thread continuation ID for multi-turn conversations ## Usage Examples **Decomposition Analysis:** ``` "Analyze UserController.java for decomposition opportunities - it's becoming unwieldy" ``` **Code Smell Detection:** ``` "Use gemini to identify code smells in the authentication module with high thinking mode" ``` **Modernization:** ``` "Modernize legacy_parser.py to use modern Python features following examples/modern_patterns.py" ``` **Organization Improvement:** ``` "Refactor src/utils/ for better organization, focus on maintainability and readability" ``` **Legacy System Refactoring:** ``` "Use pro with max thinking to analyze this 10,000-line legacy file for decomposition strategy" ``` ## Refactoring Strategy **Top-Down Analysis:** 1. **File Level**: Identify oversized files that need splitting 2. **Class Level**: Find classes with too many responsibilities 3. **Function Level**: Locate functions that are too complex or long 4. **Code Quality**: Address smells, modernization, and organization **Context-Aware Decisions:** - **Domain Complexity**: Some domains legitimately require larger classes - **Performance Constraints**: Critical path code may resist decomposition - **Legacy Stability**: Old, working code may need gentler refactoring - **Test Coverage**: Refactoring recommendations consider testability **Breaking Change Prevention:** - Analyzes dependencies before suggesting splits - Recommends gradual migration strategies - Identifies public API impact - Suggests backward compatibility approaches ## Best Practices - **Start with decomposition**: Address structural issues before cosmetic improvements - **Provide clear context**: Explain the codebase purpose and constraints - **Use appropriate refactor types**: Match the type to your primary concern - **Include style examples**: Reference existing well-structured code in your project - **Focus on high-impact areas**: Target the most problematic or frequently modified code - **Plan implementation order**: Follow the tool's sequencing recommendations - **Consider test coverage**: Ensure adequate tests before major structural changes ## Output Format Refactoring analysis includes: - **Priority Assessment**: What needs attention first and why - **Decomposition Strategy**: Specific file/class/function splitting recommendations - **Implementation Plan**: Step-by-step refactoring sequence - **Line-Number References**: Exact locations for changes - **Dependency Analysis**: Impact assessment and migration strategies - **Risk Assessment**: Potential breaking changes and mitigation strategies ## Advanced Features **Adaptive Thresholds:** The tool adjusts size thresholds based on context: - **Generated Code**: Higher tolerance for large files - **Algorithm Implementation**: Recognizes when size is justified - **Legacy Systems**: More conservative recommendations - **Test Files**: Different standards for test vs production code **Cross-File Refactoring:** Analyzes multiple files together to understand: - Shared responsibilities that could be extracted - Dependencies that complicate refactoring - Opportunities for new abstractions - Impact of changes across the codebase ## When to Use Refactor vs Other Tools - **Use `refactor`** for: Structural improvements, decomposition, modernization, code organization - **Use `codereview`** for: Finding bugs and security issues with immediate fixes - **Use `analyze`** for: Understanding code without making change recommendations - **Use `debug`** for: Solving specific runtime issues rather than structural problems ================================================ FILE: docs/tools/secaudit.md ================================================ # Secaudit Tool - Comprehensive Security Audit **Systematic OWASP-based security assessment with compliance evaluation through workflow-driven investigation** The `secaudit` tool provides comprehensive security auditing capabilities with systematic OWASP Top 10 assessment, compliance framework evaluation, and threat modeling. This workflow tool guides Claude through methodical security investigation steps with forced pauses between each step to ensure thorough vulnerability assessment, security pattern analysis, and compliance verification before providing expert analysis. **Important**: AI models may not identify all security vulnerabilities. Always perform additional manual security reviews, penetration testing, and verification. ## How the Workflow Works The secaudit tool implements a **structured 6-step security workflow** that ensures comprehensive security assessment: **Investigation Phase (Claude-Led):** 1. **Step 1**: Security Scope Analysis - Claude identifies application type, tech stack, attack surface, and compliance requirements 2. **Step 2**: Authentication & Authorization Assessment - Analyzes auth mechanisms, session management, and access controls 3. **Step 3**: Input Validation & Data Security - Reviews input handling, data protection, and injection vulnerabilities 4. **Step 4**: OWASP Top 10 (2021) Review - Systematic assessment of all OWASP categories with specific findings 5. **Step 5**: Dependencies & Infrastructure - Security analysis of third-party components and deployment configurations 6. **Step 6**: Compliance & Risk Assessment - Evaluation against specified compliance frameworks and risk prioritization **Expert Analysis Phase:** After Claude completes the investigation (unless confidence is **certain**): - Complete security assessment summary with all vulnerabilities and evidence - OWASP Top 10 systematic findings with severity classifications - Compliance framework gap analysis and remediation recommendations - Risk-prioritized remediation roadmap based on threat level and business impact **Special Note**: If you want Claude to perform the entire security audit without calling another model, you can include "don't use any other model" in your prompt, and Claude will complete the full workflow independently. ## Model Recommendation This tool particularly benefits from Gemini Pro or O3 models due to their advanced reasoning capabilities and large context windows, which allow comprehensive security analysis across complex codebases. Security audits require understanding subtle attack vectors and cross-component interactions that benefit from deeper analytical capabilities. ## Example Prompts ``` Perform a secaudit with o3 on this e-commerce web application focusing on payment processing security and PCI DSS compliance ``` ``` Use secaudit to conduct a comprehensive security audit of the authentication system, threat level high, focus on enterprise security patterns and HIPAA compliance ``` ## Pro Tip: Multi-Scope Security Assessment **You can run parallel security audits for different application components:** ``` Start separate sub-tasks, in one start a secaudit for critical payment processing components focusing on PCI DSS with gemini pro, and in the other for user management focusing on OWASP authentication vulnerabilities with o4-mini, then combine into a unified security remediation plan using planner ``` ## Key Features - **OWASP Top 10 (2021) systematic assessment** with specific vulnerability identification - **Multi-compliance framework support**: SOC2, PCI DSS, HIPAA, GDPR, FedRAMP - **Threat-level aware analysis**: Critical, high, medium, low threat classifications - **Technology-specific security patterns**: Web apps, APIs, mobile, cloud, enterprise systems - **Risk-based prioritization**: Business impact and exploitability assessment - **Audit focus customization**: Comprehensive, authentication, data protection, infrastructure - **Image support**: Security analysis from architecture diagrams, network topology, or security findings - **Multi-file security analysis**: Cross-component vulnerability identification - **Compliance gap analysis**: Specific framework requirements with remediation guidance - **Attack surface mapping**: Entry points, data flows, and privilege boundaries - **Security control effectiveness**: Evaluation of existing security measures ## Tool Parameters **Workflow Investigation Parameters (used during step-by-step process):** - `step`: Current security investigation step description (required for each step) - `step_number`: Current step number in audit sequence (required) - `total_steps`: Estimated total investigation steps (typically 4-6, adjustable) - `next_step_required`: Whether another investigation step is needed - `findings`: Security discoveries and evidence collected in this step (required) - `files_checked`: All files examined during security investigation - `relevant_files`: Files directly relevant to security assessment (required in step 1) - `relevant_context`: Methods/functions/classes central to security findings - `issues_found`: Security issues identified with severity levels - `confidence`: Confidence level in security assessment completeness (exploring/low/medium/high/certain) - `images`: Architecture diagrams, security documentation, or visual references **Initial Security Configuration (used in step 1):** - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `security_scope`: Application context, technology stack, and security boundary definition (required) - `threat_level`: low|medium|high|critical (default: medium) - determines assessment depth and urgency - `compliance_requirements`: List of compliance frameworks to assess against (e.g., ["PCI DSS", "SOC2"]) - `audit_focus`: comprehensive|authentication|data_protection|infrastructure|api_security (default: comprehensive) - `severity_filter`: critical|high|medium|low|all (default: all) - `temperature`: Temperature for analytical consistency (0-1, default 0.2) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert security analysis phase (default: true) - `continuation_id`: Continue previous security audit discussions ## Audit Focus Areas **Comprehensive (default):** - Full OWASP Top 10 assessment with all security domains - Authentication, authorization, data protection, infrastructure - Best for complete security posture evaluation **Authentication:** - Focused on identity, access management, and session security - Multi-factor authentication, password policies, privilege escalation - Best for user management and access control systems **Data Protection:** - Encryption, data handling, privacy controls, and compliance - Input validation, output encoding, data classification - Best for applications handling sensitive or regulated data **Infrastructure:** - Deployment security, configuration management, dependency security - Network security, container security, cloud security posture - Best for DevOps and infrastructure security assessment **API Security:** - REST/GraphQL security, rate limiting, API authentication - Input validation, authorization patterns, API gateway security - Best for API-first applications and microservices ## Threat Levels Security assessment depth and urgency: - **🔴 CRITICAL**: Mission-critical systems, high-value targets, regulatory requirements - **🟠 HIGH**: Business-critical applications, customer data handling, financial systems - **🟡 MEDIUM**: Standard business applications, internal tools, moderate risk exposure - **🟢 LOW**: Development environments, non-sensitive applications, proof-of-concepts ## Compliance Frameworks Supported compliance assessments: - **SOC2**: Security, availability, processing integrity, confidentiality, privacy - **PCI DSS**: Payment card industry data security standards - **HIPAA**: Healthcare information privacy and security - **GDPR**: General data protection regulation compliance - **FedRAMP**: Federal risk and authorization management program - **ISO27001**: Information security management systems - **NIST**: Cybersecurity framework controls ## OWASP Top 10 (2021) Coverage Systematic assessment includes: 1. **A01 Broken Access Control**: Authorization flaws and privilege escalation 2. **A02 Cryptographic Failures**: Encryption and data protection issues 3. **A03 Injection**: SQL, NoSQL, OS, and LDAP injection vulnerabilities 4. **A04 Insecure Design**: Security design flaws and threat modeling gaps 5. **A05 Security Misconfiguration**: Configuration and hardening issues 6. **A06 Vulnerable Components**: Third-party and dependency vulnerabilities 7. **A07 Identification & Authentication Failures**: Authentication bypass and session management 8. **A08 Software & Data Integrity Failures**: Supply chain and integrity violations 9. **A09 Security Logging & Monitoring Failures**: Detection and response capabilities 10. **A10 Server-Side Request Forgery**: SSRF and related vulnerabilities ## Usage Examples **Comprehensive E-commerce Security Audit:** ``` "Conduct a comprehensive secaudit with gemini pro for our Node.js e-commerce platform, threat level high, compliance requirements PCI DSS and SOC2, focus on payment processing security" ``` **Authentication System Security Review:** ``` "Use o3 to perform secaudit on authentication microservice, focus on authentication, threat level critical, check for OWASP A07 and multi-factor authentication implementation" ``` **API Security Assessment:** ``` "Secaudit our REST API gateway with gemini pro, audit focus api_security, compliance requirements GDPR, threat level medium" ``` **Infrastructure Security Review:** ``` "Perform secaudit on Kubernetes deployment manifests with o3, focus infrastructure, threat level high, include container security and network policies" ``` **Quick Security Scan:** ``` "Fast secaudit of user registration flow with flash, focus authentication, severity filter critical and high only" ``` ## Best Practices - **Define clear security scope**: Specify application type, tech stack, and security boundaries - **Set appropriate threat levels**: Match assessment depth to risk exposure and criticality - **Include compliance requirements**: Specify relevant frameworks for regulatory alignment - **Use parallel audits**: Run separate assessments for different components or compliance frameworks - **Provide architectural context**: Include system diagrams, data flow documentation, or deployment topology - **Focus audit scope**: Use audit_focus for targeted assessments of specific security domains - **Follow up on findings**: Use continuation feature to dive deeper into specific vulnerabilities ## Output Format Security audits include: - **Executive Security Summary**: Overall security posture and critical findings - **OWASP Top 10 Assessment**: Systematic review of each category with specific findings - **Compliance Gap Analysis**: Framework-specific requirements and current compliance status - **Risk-Prioritized Findings**: Vulnerabilities ordered by exploitability and business impact - **Remediation Roadmap**: Phased approach to security improvements with quick wins - **Security Architecture Recommendations**: Structural improvements for long-term security posture ## When to Use Secaudit vs Other Tools - **Use `secaudit`** for: Comprehensive security assessment, compliance evaluation, OWASP-based vulnerability analysis - **Use `codereview`** for: General code quality with some security considerations - **Use `analyze`** for: Understanding security architecture without vulnerability assessment - **Use `debug`** for: Investigating specific security incidents or exploit attempts - **Use `precommit`** for: Pre-deployment security validation and change impact assessment ================================================ FILE: docs/tools/testgen.md ================================================ # TestGen Tool - Comprehensive Test Generation **Generates thorough test suites with edge case coverage through workflow-driven investigation** The `testgen` tool creates comprehensive test suites by analyzing your code paths, understanding intricate dependencies, and identifying realistic edge cases and failure scenarios that need test coverage. This workflow tool guides Claude through systematic investigation of code functionality, critical paths, edge cases, and integration points across multiple steps before generating comprehensive tests with realistic failure mode analysis. ## Thinking Mode **Default is `medium` (8,192 tokens) for extended thinking models.** Use `high` for complex systems with many interactions or `max` for critical systems requiring exhaustive test coverage. ## How the Workflow Works The testgen tool implements a **structured workflow** for comprehensive test generation: **Investigation Phase (Claude-Led):** 1. **Step 1**: Claude describes the test generation plan and begins analyzing code functionality 2. **Step 2+**: Claude examines critical paths, edge cases, error handling, and integration points 3. **Throughout**: Claude tracks findings, test scenarios, and coverage gaps 4. **Completion**: Once investigation is thorough, Claude signals completion **Test Generation Phase:** After Claude completes the investigation: - Complete test scenario catalog with all edge cases - Framework-specific test generation - Realistic failure mode coverage - Final test suite with comprehensive coverage This workflow ensures methodical analysis before test generation, resulting in more thorough and valuable test suites. ## Model Recommendation Test generation excels with extended reasoning models like Gemini Pro or O3, which can analyze complex code paths, understand intricate dependencies, and identify comprehensive edge cases. The combination of large context windows and advanced reasoning enables generation of thorough test suites that cover realistic failure scenarios and integration points that shorter-context models might overlook. ## Example Prompts **Basic Usage:** ``` "Use pal to generate tests for User.login() method" "Generate comprehensive tests for the sorting method in src/new_sort.py using o3" "Create tests for edge cases not already covered in our tests using gemini pro" ``` ## Key Features - **Multi-agent workflow** analyzing code paths and identifying realistic failure modes - **Generates framework-specific tests** following project conventions - **Supports test pattern following** when examples are provided - **Dynamic token allocation** (25% for test examples, 75% for main code) - **Prioritizes smallest test files** for pattern detection - **Can reference existing test files**: `"Generate tests following patterns from tests/unit/"` - **Specific code coverage** - target specific functions/classes rather than testing everything - **Image support**: Test UI components, analyze visual requirements: `"Generate tests for this login form using the UI mockup screenshot"` - **Edge case identification**: Systematic discovery of boundary conditions and error states - **Realistic failure mode analysis**: Understanding what can actually go wrong in production - **Integration test support**: Tests that cover component interactions and system boundaries ## Tool Parameters **Workflow Investigation Parameters (used during step-by-step process):** - `step`: Current investigation step description (required for each step) - `step_number`: Current step number in test generation sequence (required) - `total_steps`: Estimated total investigation steps (adjustable) - `next_step_required`: Whether another investigation step is needed - `findings`: Discoveries about functionality and test scenarios (required) - `files_checked`: All files examined during investigation - `relevant_files`: Files directly needing tests (required in step 1) - `relevant_context`: Methods/functions/classes requiring test coverage - `confidence`: Confidence level in test plan completeness (exploring/low/medium/high/certain) **Initial Configuration (used in step 1):** - `prompt`: Description of what to test, testing objectives, and specific scope/focus areas (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `test_examples`: Optional existing test files or directories to use as style/pattern reference (absolute paths) - `thinking_mode`: minimal|low|medium|high|max (default: medium, Gemini only) - `use_assistant_model`: Whether to use expert test generation phase (default: true, set to false to use Claude only) ## Usage Examples **Method-Specific Tests:** ``` "Generate tests for User.login() method covering authentication success, failure, and edge cases" ``` **Class Testing:** ``` "Use pro to generate comprehensive tests for PaymentProcessor class with max thinking mode" ``` **Following Existing Patterns:** ``` "Generate tests for new authentication module following patterns from tests/unit/auth/" ``` **UI Component Testing:** ``` "Generate tests for this login form component using the UI mockup screenshot" ``` **Algorithm Testing:** ``` "Create thorough tests for the sorting algorithm in utils/sort.py, focus on edge cases and performance" ``` **Integration Testing:** ``` "Generate integration tests for the payment processing pipeline from order creation to completion" ``` ## Test Generation Strategy **Code Path Analysis:** - Identifies all execution paths through the code - Maps conditional branches and loops - Discovers error handling paths - Analyzes state transitions **Edge Case Discovery:** - Boundary value analysis (empty, null, max values) - Invalid input scenarios - Race conditions and timing issues - Resource exhaustion cases **Failure Mode Analysis:** - External dependency failures - Network and I/O errors - Authentication and authorization failures - Data corruption scenarios **Framework Detection:** The tool automatically detects and generates tests for: - **Python**: pytest, unittest, nose2 - **JavaScript**: Jest, Mocha, Jasmine, Vitest - **Java**: JUnit 4/5, TestNG, Mockito - **C#**: NUnit, MSTest, xUnit - **Swift**: XCTest - **Go**: testing package - **And more**: Adapts to project conventions ## Test Categories Generated **Unit Tests:** - Function/method behavior validation - Input/output verification - Error condition handling - State change verification **Integration Tests:** - Component interaction testing - API endpoint validation - Database integration - External service mocking **Edge Case Tests:** - Boundary conditions - Invalid inputs - Resource limits - Concurrent access **Performance Tests:** - Response time validation - Memory usage checks - Load handling - Scalability verification ## Best Practices - **Be specific about scope**: Target specific functions/classes rather than requesting tests for everything - **Provide test examples**: Include existing test files for pattern consistency - **Focus on critical paths**: Prioritize testing of business-critical functionality - **Include visual context**: Screenshots or mockups for UI component testing - **Describe testing objectives**: Explain what aspects are most important to test - **Consider test maintenance**: Request readable, maintainable test code ## Test Quality Features **Realistic Test Data:** - Generates meaningful test data that represents real-world scenarios - Avoids trivial test cases that don't add value - Creates data that exercises actual business logic **Comprehensive Coverage:** - Happy path scenarios - Error conditions and exceptions - Edge cases and boundary conditions - Integration points and dependencies **Maintainable Code:** - Clear test names that describe what's being tested - Well-organized test structure - Appropriate use of setup/teardown - Minimal test data and mocking ## Advanced Features **Pattern Following:** When test examples are provided, the tool analyzes: - Naming conventions and structure - Assertion patterns and style - Mocking and setup approaches - Test data organization **Large Context Analysis:** With models like Gemini Pro, the tool can: - Analyze extensive codebases for comprehensive test coverage - Understand complex interactions across multiple modules - Generate integration tests that span multiple components **Visual Testing:** For UI components and visual elements: - Generate tests based on visual requirements - Create accessibility testing scenarios - Test responsive design behaviors ## When to Use TestGen vs Other Tools - **Use `testgen`** for: Creating comprehensive test suites, filling test coverage gaps, testing new features - **Use `debug`** for: Diagnosing specific test failures or runtime issues - **Use `codereview`** for: Reviewing existing test quality and coverage - **Use `analyze`** for: Understanding existing test structure without generating new tests ================================================ FILE: docs/tools/thinkdeep.md ================================================ # ThinkDeep Tool - Extended Reasoning Partner **Get a second opinion to augment Claude's own extended thinking** The `thinkdeep` tool provides extended reasoning capabilities, offering a second perspective to augment Claude's analysis. It's designed to challenge assumptions, find edge cases, and provide alternative approaches to complex problems. ## Thinking Mode **Default is `high` (16,384 tokens) for deep analysis.** Claude will automatically choose the best mode based on complexity - use `low` for quick validations, `medium` for standard problems, `high` for complex issues (default), or `max` for extremely complex challenges requiring deepest analysis. ## Example Prompt ``` Think deeper about my authentication design with pro using max thinking mode and brainstorm to come up with the best architecture for my project ``` ## Key Features - **Uses Gemini's specialized thinking models** for enhanced reasoning capabilities - **Provides a second opinion** on Claude's analysis - **Challenges assumptions** and identifies edge cases Claude might miss - **Offers alternative perspectives** and approaches - **Validates architectural decisions** and design patterns - **File reference support**: `"Use gemini to think deeper about my API design with reference to api/routes.py"` - **Image support**: Analyze architectural diagrams, flowcharts, design mockups: `"Think deeper about this system architecture diagram with gemini pro using max thinking mode"` - **Enhanced Critical Evaluation (v2.10.0)**: After Gemini's analysis, Claude is prompted to critically evaluate the suggestions, consider context and constraints, identify risks, and synthesize a final recommendation - ensuring a balanced, well-considered solution - **Web search capability**: Automatically identifies areas where current documentation or community solutions would strengthen the analysis and instructs Claude to perform targeted searches ## Tool Parameters - `prompt`: Your current thinking/analysis to extend and validate (required) - `model`: auto|pro|flash|flash-2.0|flashlite|o3|o3-mini|o4-mini|gpt4.1|gpt5.2|gpt5.1-codex|gpt5.1-codex-mini|gpt5|gpt5-mini|gpt5-nano (default: server default) - `problem_context`: Additional context about the problem or goal - `focus_areas`: Specific aspects to focus on (architecture, performance, security, etc.) - `files`: Optional file paths or directories for additional context (absolute paths) - `images`: Optional images for visual analysis (absolute paths) - `temperature`: Temperature for creative thinking (0-1, default 0.7) - `thinking_mode`: minimal|low|medium|high|max (default: high, Gemini only) - `continuation_id`: Continue previous conversations ## Usage Examples **Architecture Design:** ``` "Think deeper about my microservices authentication strategy with pro using max thinking mode" ``` **With File Context:** ``` "Use gemini to think deeper about my API design with reference to api/routes.py and models/user.py" ``` **Visual Analysis:** ``` "Think deeper about this system architecture diagram with gemini pro - identify potential bottlenecks" ``` **Problem Solving:** ``` "I'm considering using GraphQL vs REST for my API. Think deeper about the trade-offs with o3 using high thinking mode" ``` **Code Review Enhancement:** ``` "Think deeper about the security implications of this authentication code with pro" ``` ## Best Practices - **Provide detailed context**: Share your current thinking, constraints, and objectives - **Be specific about focus areas**: Mention what aspects need deeper analysis - **Include relevant files**: Reference code, documentation, or configuration files - **Use appropriate thinking modes**: Higher modes for complex problems, lower for quick validations - **Leverage visual context**: Include diagrams or mockups for architectural discussions - **Build on discussions**: Use continuation to extend previous analyses ## Enhanced Critical Evaluation Process The `thinkdeep` tool includes a unique two-stage process: 1. **Gemini's Analysis**: Extended reasoning with specialized thinking capabilities 2. **Claude's Critical Evaluation**: Claude reviews Gemini's suggestions, considers: - Context and constraints of your specific situation - Potential risks and implementation challenges - Trade-offs and alternatives - Final synthesized recommendation This ensures you get both deep reasoning and practical, context-aware advice. ## When to Use ThinkDeep vs Other Tools - **Use `thinkdeep`** for: Extending specific analysis, challenging assumptions, architectural decisions - **Use `chat`** for: Open-ended brainstorming and general discussions - **Use `analyze`** for: Understanding existing code without extending analysis - **Use `codereview`** for: Finding specific bugs and security issues ================================================ FILE: docs/tools/tracer.md ================================================ # Tracer Tool - Static Code Analysis Prompt Generator **Creates detailed analysis prompts for call-flow mapping and dependency tracing** The `tracer` tool is a specialized prompt-generation tool that creates structured analysis requests for Claude to perform comprehensive static code analysis. Rather than passing entire projects to another model, this tool generates focused prompts that Claude can use to efficiently trace execution flows and map dependencies within the codebase. ## Two Analysis Modes **`precision` Mode**: For methods/functions - Traces execution flow, call chains, and usage patterns - Detailed branching analysis and side effects - Shows when and how functions are called throughout the system **`dependencies` Mode**: For classes/modules/protocols - Maps bidirectional dependencies and structural relationships - Identifies coupling and architectural dependencies - Shows how components interact and depend on each other ## Key Features - **Generates comprehensive analysis prompts** instead of performing analysis directly - **Faster and more efficient** than full project analysis by external models - **Creates structured instructions** for call-flow graph generation - **Provides detailed formatting requirements** for consistent output - **Supports any programming language** with automatic convention detection - **Output can be used as input** into another tool, such as `chat` along with related code files to perform logical call-flow analysis - **Image support**: Analyze visual call flow diagrams, sequence diagrams: `"Generate tracer analysis for this payment flow using the sequence diagram"` ## Tool Parameters - `prompt`: Detailed description of what to trace and WHY you need this analysis (required) - `trace_mode`: precision|dependencies (required) - `images`: Optional images of system architecture diagrams, flow charts, or visual references (absolute paths) ## Usage Examples **Method Execution Tracing:** ``` "Use pal tracer to analyze how UserAuthManager.authenticate is used and why" ``` → Uses `precision` mode to trace the method's execution flow **Class Dependency Mapping:** ``` "Use pal to generate a dependency trace for the PaymentProcessor class to understand its relationships" ``` → Uses `dependencies` mode to map structural relationships **With Visual Context:** ``` "Generate tracer analysis for the authentication flow using this sequence diagram" ``` **Complex System Analysis:** ``` "Create a tracer prompt to understand how the OrderProcessor.processPayment method flows through the entire system" ``` ## Precision Mode Output When using `precision` mode for methods/functions, the tool generates prompts that will help Claude create: **Call Chain Analysis:** - Where the method is defined - All locations where it's called - Direct and indirect callers - Call hierarchy and depth **Execution Flow Mapping:** - Step-by-step execution path - Branching conditions and logic - Side effects and state changes - Return value usage **Usage Pattern Analysis:** - Frequency and context of calls - Parameter passing patterns - Error handling approaches - Performance implications ## Dependencies Mode Output When using `dependencies` mode for classes/modules, the tool generates prompts that will help Claude create: **Structural Relationships:** - Inheritance hierarchies - Composition and aggregation - Interface implementations - Module imports and exports **Bidirectional Dependencies:** - What the component depends on - What depends on the component - Circular dependencies - Coupling strength analysis **Architectural Impact:** - Layer violations - Dependency inversion opportunities - Refactoring impact assessment - Testability implications ## Example Generated Prompts **For Precision Mode:** ``` Analyze the execution flow and usage of the `authenticate` method in UserAuthManager: 1. **Method Location**: Find where UserAuthManager.authenticate is defined 2. **Call Sites**: Identify all locations where this method is called 3. **Execution Flow**: Trace the step-by-step execution path 4. **Side Effects**: Document state changes and external interactions 5. **Return Handling**: Show how return values are used by callers Format the analysis as: - Method signature and location - Call hierarchy (direct and indirect callers) - Execution flow diagram - Side effects and dependencies - Usage patterns and frequency ``` **For Dependencies Mode:** ``` Map the structural dependencies for PaymentProcessor class: 1. **Direct Dependencies**: What PaymentProcessor directly imports/uses 2. **Reverse Dependencies**: What classes/modules depend on PaymentProcessor 3. **Inheritance Relationships**: Parent classes and implemented interfaces 4. **Composition**: Objects that PaymentProcessor contains or creates Format the analysis as: - Dependency graph (incoming and outgoing) - Architectural layer analysis - Coupling assessment - Refactoring impact evaluation ``` ## Best Practices - **Be specific about goals**: Clearly state what you need to understand and why - **Describe context**: Mention if you're debugging, refactoring, or learning the codebase - **Choose appropriate mode**: Use `precision` for method flows, `dependencies` for architecture - **Include visual context**: Reference diagrams or documentation when available - **Follow up with analysis**: Use the generated prompt with `chat` or `analyze` tools ## Integration with Other Tools The `tracer` tool works best when combined with other analysis tools: **Tracer + Chat:** ``` 1. Generate analysis prompt with tracer 2. Use the prompt with chat tool and relevant code files 3. Get detailed call-flow or dependency analysis ``` **Tracer + Analyze:** ``` 1. Use tracer to create structured analysis prompt 2. Apply the prompt using analyze tool for systematic code exploration 3. Get architectural insights and dependency mapping ``` ## When to Use Tracer vs Other Tools - **Use `tracer`** for: Creating structured analysis prompts, systematic code exploration planning - **Use `analyze`** for: Direct code analysis without prompt generation - **Use `debug`** for: Specific runtime error investigation - **Use `chat`** for: Open-ended code discussions and exploration ================================================ FILE: docs/tools/version.md ================================================ # Version Tool - Server Information **Get server version, configuration details, and list of available tools** The `version` tool provides information about the PAL MCP Server version, configuration details, and system capabilities. This is useful for debugging, understanding server capabilities, and verifying your installation. ## Usage ``` "Get pal to show its version" ``` ## Key Features - **Server version information**: Current version and build details - **Configuration overview**: Active settings and capabilities - **Tool inventory**: Complete list of available tools and their status - **System health**: Basic server status and connectivity verification - **Debug information**: Helpful details for troubleshooting ## Output Information The tool provides: **Version Details:** - Server version number - Build timestamp and commit information - MCP protocol version compatibility - Python runtime version **Configuration Summary:** - Active providers and their status - Default model configuration - Feature flags and settings - Environment configuration overview **Tool Availability:** - Complete list of available tools - Tool version information - Capability status for each tool **System Information:** - Server uptime and status - Memory and resource usage (if available) - Conversation memory status - Server process information ## Example Output ``` 🔧 PAL MCP Server Information 📋 Version: 2.15.0 🏗️ Build: 2024-01-15T10:30:00Z (commit: abc123f) 🔌 MCP Protocol: 1.0.0 🐍 Python Runtime: 3.11.7 ⚙️ Configuration: • Default Model: auto • Providers: Google ✅, OpenAI ✅, Custom ✅ • Conversation Memory: Active ✅ • Web Search: Enabled 🛠️ Available Tools (12): • chat - General development chat & collaborative thinking • thinkdeep - Extended reasoning partner • consensus - Multi-model perspective gathering • codereview - Professional code review • precommit - Pre-commit validation • debug - Expert debugging assistant • analyze - Smart file analysis • refactor - Intelligent code refactoring • tracer - Static code analysis prompt generator • testgen - Comprehensive test generation • listmodels - List available models • version - Server information 🔍 System Status: • Server Uptime: 2h 35m • Memory Storage: Active • Server Process: Running ``` ## When to Use Version Tool - **Troubleshooting**: When experiencing issues with the server or tools - **Configuration verification**: To confirm your setup is correct - **Support requests**: To provide system information when asking for help - **Update checking**: To verify you're running the latest version - **Capability discovery**: To understand what features are available ## Debug Information The version tool can help diagnose common issues: **Connection Problems:** - Verify server is running and responsive - Check MCP protocol compatibility - Confirm tool availability **Configuration Issues:** - Validate provider setup - Check API key configuration status - Verify feature enablement **Performance Troubleshooting:** - Server uptime and stability - Resource usage patterns - Memory storage health ## Tool Parameters This tool requires no parameters - it provides comprehensive server information automatically. ## Best Practices - **Include in bug reports**: Always include version output when reporting issues - **Check after updates**: Verify version information after server updates - **Monitor system health**: Use periodically to check server status - **Validate configuration**: Confirm settings match your expectations ## When to Use Version vs Other Tools - **Use `version`** for: Server diagnostics, configuration verification, troubleshooting - **Use `listmodels`** for: Model availability and capability information - **Use other tools** for: Actual development and analysis tasks - **Use with support**: Essential information for getting help with issues ================================================ FILE: docs/troubleshooting.md ================================================ # Troubleshooting Guide ## Quick Debugging Steps If you're experiencing issues with the PAL MCP Server, follow these steps: ### 1. Check MCP Connection Open Claude Desktop and type `/mcp` to see if pal is connected: - ✅ If pal appears in the list, the connection is working - ❌ If not listed or shows an error, continue to step 2 ### 2. Launch Claude with Debug Mode Close Claude Desktop and restart with debug logging: ```bash # macOS/Linux claude --debug # Windows (in WSL2) claude.exe --debug ``` Look for error messages in the console output, especially: - API key errors - Python/environment issues - File permission errors ### 3. Verify API Keys Check that your API keys are properly set: ```bash # Check your .env file cat .env # Ensure at least one key is set: # GEMINI_API_KEY=your-key-here # OPENAI_API_KEY=your-key-here ``` If you need to update your API keys, edit the `.env` file and then restart Claude for changes to take effect. ### 4. Check Server Logs View the server logs for detailed error information: ```bash # View recent logs tail -n 100 logs/mcp_server.log # Follow logs in real-time tail -f logs/mcp_server.log # Or use the -f flag when starting to automatically follow logs ./run-server.sh -f # Search for errors grep "ERROR" logs/mcp_server.log ``` See [Logging Documentation](logging.md) for more details on accessing logs. ### 5. Common Issues **"Connection failed" in Claude Desktop** - Ensure the server path is correct in your Claude config - Run `./run-server.sh` to verify setup and see configuration - Check that Python is installed: `python3 --version` **"API key environment variable is required"** - Add your API key to the `.env` file - Restart Claude Desktop after updating `.env` **File path errors** - Always use absolute paths: `/Users/you/project/file.py` - Never use relative paths: `./file.py` **Python module not found** - Run `./run-server.sh` to reinstall dependencies - Check virtual environment is activated: should see `.pal_venv` in the Python path ### 6. Environment Issues **Virtual Environment Problems** ```bash # Reset environment completely rm -rf .pal_venv ./run-server.sh ``` **Permission Issues** ```bash # Ensure script is executable chmod +x run-server.sh ``` ### 7. Still Having Issues? If the problem persists after trying these steps: 1. **Reproduce the issue** - Note the exact steps that cause the problem 2. **Collect logs** - Save relevant error messages from Claude debug mode and server logs 3. **Open a GitHub issue** with: - Your operating system - Python version: `python3 --version` - Error messages from logs - Steps to reproduce - What you've already tried ## Windows Users **Important**: Windows users must use WSL2. Install it with: ```powershell wsl --install -d Ubuntu ``` Then follow the standard setup inside WSL2. ================================================ FILE: docs/vcr-testing.md ================================================ # HTTP Transport Recorder for Testing A custom HTTP recorder for testing expensive API calls (like o3-pro) with real responses. ## Overview The HTTP Transport Recorder captures and replays HTTP interactions at the transport layer, enabling: - Cost-efficient testing of expensive APIs (record once, replay forever) - Deterministic tests with real API responses - Seamless integration with httpx and OpenAI SDK - Automatic PII sanitization for secure recordings ## Quick Start ```python from tests.transport_helpers import inject_transport # Simple one-line setup with automatic transport injection def test_expensive_api_call(monkeypatch): inject_transport(monkeypatch, "tests/openai_cassettes/my_test.json") # Make API calls - automatically recorded/replayed with PII sanitization result = await chat_tool.execute({"prompt": "2+2?", "model": "o3-pro"}) ``` ## How It Works 1. **First run** (cassette doesn't exist): Records real API calls 2. **Subsequent runs** (cassette exists): Replays saved responses 3. **Re-record**: Delete cassette file and run again ## Usage in Tests The `transport_helpers.inject_transport()` function simplifies test setup: ```python from tests.transport_helpers import inject_transport async def test_with_recording(monkeypatch): # One-line setup - handles all transport injection complexity inject_transport(monkeypatch, "tests/openai_cassettes/my_test.json") # Use API normally - recording/replay happens transparently result = await chat_tool.execute({"prompt": "2+2?", "model": "o3-pro"}) ``` For manual setup, see `test_o3_pro_output_text_fix.py`. ## Automatic PII Sanitization All recordings are automatically sanitized to remove sensitive data: - **API Keys & Tokens**: Bearer tokens, API keys, and auth headers - **Personal Data**: Email addresses, IP addresses, phone numbers - **URLs**: Sensitive query parameters and paths - **Custom Patterns**: Add your own sanitization rules Sanitization is enabled by default in `RecordingTransport`. To disable: ```python transport = TransportFactory.create_transport(cassette_path, sanitize=False) ``` ## File Structure ``` tests/ ├── openai_cassettes/ # Recorded API interactions │ └── *.json # Cassette files ├── http_transport_recorder.py # Transport implementation ├── pii_sanitizer.py # Automatic PII sanitization ├── transport_helpers.py # Simplified transport injection ├── sanitize_cassettes.py # Batch sanitization script └── test_o3_pro_output_text_fix.py # Example usage ``` ## Sanitizing Existing Cassettes Use the `sanitize_cassettes.py` script to clean existing recordings: ```bash # Sanitize all cassettes (creates backups) python tests/sanitize_cassettes.py # Sanitize specific cassette python tests/sanitize_cassettes.py tests/openai_cassettes/my_test.json # Skip backup creation python tests/sanitize_cassettes.py --no-backup ``` The script will: - Create timestamped backups of original files - Apply comprehensive PII sanitization - Preserve JSON structure and functionality ## Cost Management - **One-time cost**: Initial recording only - **Zero ongoing cost**: Replays are free - **CI-friendly**: No API keys needed for replay ## Re-recording When API changes require new recordings: ```bash # Delete specific cassette rm tests/openai_cassettes/my_test.json # Run test with real API key python -m pytest tests/test_o3_pro_output_text_fix.py ``` ## Implementation Details - **RecordingTransport**: Captures real HTTP calls with automatic PII sanitization - **ReplayTransport**: Serves saved responses from cassettes - **TransportFactory**: Auto-selects mode based on cassette existence - **PIISanitizer**: Comprehensive sanitization of sensitive data (integrated by default) **Security Note**: While recordings are automatically sanitized, always review new cassette files before committing. The sanitizer removes known patterns of sensitive data, but domain-specific secrets may need custom rules. For implementation details, see: - `tests/http_transport_recorder.py` - Core transport implementation - `tests/pii_sanitizer.py` - Sanitization patterns and logic - `tests/transport_helpers.py` - Simplified test integration ================================================ FILE: docs/wsl-setup.md ================================================ # WSL (Windows Subsystem for Linux) Setup Guide This guide provides detailed instructions for setting up PAL MCP Server on Windows using WSL. ## Prerequisites for WSL ```bash # Update WSL and ensure you have a recent Ubuntu distribution sudo apt update && sudo apt upgrade -y # Install required system dependencies sudo apt install -y python3-venv python3-pip curl git # Install Node.js and npm (required for Claude Code CLI) curl -fsSL https://deb.nodesource.com/setup_lts.x | sudo -E bash - sudo apt install -y nodejs # Install Claude Code CLI globally npm install -g @anthropic-ai/claude-code ``` ## WSL-Specific Installation Steps 1. **Clone the repository in your WSL environment** (not in Windows filesystem): ```bash # Navigate to your home directory or preferred location in WSL cd ~ # Clone the repository git clone https://github.com/BeehiveInnovations/pal-mcp-server.git cd pal-mcp-server ``` 2. **Run the setup script**: ```bash # Make the script executable and run it chmod +x run-server.sh ./run-server.sh ``` 3. **Verify Claude Code can find the MCP server**: ```bash # List configured MCP servers claude mcp list # You should see 'pal' listed in the output # If not, the setup script will provide the correct configuration ``` ## Troubleshooting WSL Issues ### Python Environment Issues ```bash # If you encounter Python virtual environment issues sudo apt install -y python3.12-venv python3.12-dev # Ensure pip is up to date python3 -m pip install --upgrade pip ``` ### Path Issues - Always use the full WSL path for MCP configuration (e.g., `/home/YourName/pal-mcp-server/`) - The setup script automatically detects WSL and configures the correct paths ### Claude Code Connection Issues ```bash # If Claude Code can't connect to the MCP server, check the configuration cat ~/.claude.json | grep -A 10 "pal" # The configuration should show the correct WSL path to the Python executable # Example: "/home/YourName/pal-mcp-server/.pal_venv/bin/python" ``` ### Performance Tip For best performance, keep your pal-mcp-server directory in the WSL filesystem (e.g., `~/pal-mcp-server`) rather than in the Windows filesystem (`/mnt/c/...`). ================================================ FILE: examples/claude_config_macos.json ================================================ { "comment": "macOS configuration using standalone server", "comment2": "Run './run-server.sh' to set up the environment and get exact paths", "comment3": "Use './run-server.sh -c' to display the correct configuration", "mcpServers": { "pal": { "command": "/path/to/pal-mcp-server/.pal_venv/bin/python", "args": ["/path/to/pal-mcp-server/server.py"] } } } ================================================ FILE: examples/claude_config_wsl.json ================================================ { "comment": "Windows configuration using WSL with standalone server", "comment2": "Run './run-server.sh' in WSL to set up the environment and get exact paths", "comment3": "Use './run-server.sh -c' to display the correct configuration", "mcpServers": { "pal": { "command": "wsl.exe", "args": [ "/path/to/pal-mcp-server/.pal_venv/bin/python", "/path/to/pal-mcp-server/server.py" ] } } } ================================================ FILE: pal-mcp-server ================================================ #!/bin/bash # Wrapper script for Gemini CLI compatibility # Get the directory of this script DIR="$(cd "$(dirname "$0")" && pwd)" # Change to the pal-mcp-server directory cd "$DIR" # Execute the Python server with all arguments passed through exec .pal_venv/bin/python server.py "$@" ================================================ FILE: providers/__init__.py ================================================ """Model provider abstractions for supporting multiple AI providers.""" from .azure_openai import AzureOpenAIProvider from .base import ModelProvider from .gemini import GeminiModelProvider from .openai import OpenAIModelProvider from .openai_compatible import OpenAICompatibleProvider from .openrouter import OpenRouterProvider from .registry import ModelProviderRegistry from .shared import ModelCapabilities, ModelResponse __all__ = [ "ModelProvider", "ModelResponse", "ModelCapabilities", "ModelProviderRegistry", "AzureOpenAIProvider", "GeminiModelProvider", "OpenAIModelProvider", "OpenAICompatibleProvider", "OpenRouterProvider", ] ================================================ FILE: providers/azure_openai.py ================================================ """Azure OpenAI provider built on the OpenAI-compatible implementation.""" from __future__ import annotations import logging from dataclasses import asdict, replace try: # pragma: no cover - optional dependency from openai import AzureOpenAI except ImportError: # pragma: no cover AzureOpenAI = None # type: ignore[assignment] from utils.env import get_env, suppress_env_vars from .openai import OpenAIModelProvider from .openai_compatible import OpenAICompatibleProvider from .registries.azure import AzureModelRegistry from .shared import ModelCapabilities, ModelResponse, ProviderType, TemperatureConstraint logger = logging.getLogger(__name__) class AzureOpenAIProvider(OpenAICompatibleProvider): """Thin Azure wrapper that reuses the OpenAI-compatible request pipeline.""" FRIENDLY_NAME = "Azure OpenAI" DEFAULT_API_VERSION = "2024-02-15-preview" # The OpenAI-compatible base expects subclasses to expose capabilities via # ``get_all_model_capabilities``. Azure deployments are user-defined, so we # build the catalogue dynamically from environment configuration instead of # relying on a static ``MODEL_CAPABILITIES`` map. MODEL_CAPABILITIES: dict[str, ModelCapabilities] = {} def __init__( self, api_key: str, *, azure_endpoint: str | None = None, api_version: str | None = None, deployments: dict[str, object] | None = None, **kwargs, ) -> None: # Let the OpenAI-compatible base handle shared configuration such as # timeouts, restriction-aware allowlists, and logging. ``base_url`` maps # directly onto Azure's endpoint URL. super().__init__(api_key, base_url=azure_endpoint, **kwargs) if not azure_endpoint: azure_endpoint = get_env("AZURE_OPENAI_ENDPOINT") if not azure_endpoint: raise ValueError("Azure OpenAI endpoint is required via parameter or AZURE_OPENAI_ENDPOINT") self.azure_endpoint = azure_endpoint.rstrip("/") self.api_version = api_version or get_env("AZURE_OPENAI_API_VERSION", self.DEFAULT_API_VERSION) registry_specs = self._load_registry_entries() override_specs = self._normalise_deployments(deployments or {}) if deployments else {} self._model_specs = self._merge_specs(registry_specs, override_specs) if not self._model_specs: raise ValueError( "Azure OpenAI provider requires at least one configured deployment. " "Populate conf/azure_models.json or set AZURE_MODELS_CONFIG_PATH." ) self._capabilities = self._build_capabilities_map() self._deployment_map = {name: spec["deployment"] for name, spec in self._model_specs.items()} self._deployment_alias_lookup = { deployment.lower(): canonical for canonical, deployment in self._deployment_map.items() } self._canonical_lookup = {name.lower(): name for name in self._model_specs.keys()} self._invalidate_capability_cache() # ------------------------------------------------------------------ # Capability helpers # ------------------------------------------------------------------ def get_all_model_capabilities(self) -> dict[str, ModelCapabilities]: return dict(self._capabilities) def get_provider_type(self) -> ProviderType: return ProviderType.AZURE def get_capabilities(self, model_name: str) -> ModelCapabilities: # type: ignore[override] lowered = model_name.lower() if lowered in self._deployment_alias_lookup: canonical = self._deployment_alias_lookup[lowered] return super().get_capabilities(canonical) canonical = self._canonical_lookup.get(lowered) if canonical: return super().get_capabilities(canonical) return super().get_capabilities(model_name) def validate_model_name(self, model_name: str) -> bool: # type: ignore[override] lowered = model_name.lower() if lowered in self._deployment_alias_lookup or lowered in self._canonical_lookup: return True return super().validate_model_name(model_name) def _build_capabilities_map(self) -> dict[str, ModelCapabilities]: capabilities: dict[str, ModelCapabilities] = {} for canonical_name, spec in self._model_specs.items(): template_capability: ModelCapabilities | None = spec.get("capability") overrides = spec.get("overrides", {}) if template_capability: cloned = replace(template_capability) else: template = OpenAIModelProvider.MODEL_CAPABILITIES.get(canonical_name) if template: friendly = template.friendly_name.replace("OpenAI", "Azure OpenAI", 1) cloned = replace( template, provider=ProviderType.AZURE, friendly_name=friendly, aliases=list(template.aliases), ) else: deployment_name = spec.get("deployment", "") cloned = ModelCapabilities( provider=ProviderType.AZURE, model_name=canonical_name, friendly_name=f"Azure OpenAI ({canonical_name})", description=f"Azure deployment '{deployment_name}' for {canonical_name}", aliases=[], ) if overrides: overrides = dict(overrides) temp_override = overrides.get("temperature_constraint") if isinstance(temp_override, str): overrides["temperature_constraint"] = TemperatureConstraint.create(temp_override) aliases_override = overrides.get("aliases") if isinstance(aliases_override, str): overrides["aliases"] = [alias.strip() for alias in aliases_override.split(",") if alias.strip()] provider_override = overrides.get("provider") if provider_override: overrides.pop("provider", None) try: cloned = replace(cloned, **overrides) except TypeError: base_data = asdict(cloned) base_data.update(overrides) base_data["provider"] = ProviderType.AZURE temp_value = base_data.get("temperature_constraint") if isinstance(temp_value, str): base_data["temperature_constraint"] = TemperatureConstraint.create(temp_value) cloned = ModelCapabilities(**base_data) if cloned.provider != ProviderType.AZURE: cloned.provider = ProviderType.AZURE capabilities[canonical_name] = cloned return capabilities def _load_registry_entries(self) -> dict[str, dict]: try: registry = AzureModelRegistry() except Exception as exc: # pragma: no cover - registry failure should not crash provider logger.warning("Unable to load Azure model registry: %s", exc) return {} entries: dict[str, dict] = {} for model_name, capability, extra in registry.iter_entries(): deployment = extra.get("deployment") if not deployment: logger.warning("Azure model '%s' missing deployment in registry", model_name) continue entries[model_name] = {"deployment": deployment, "capability": capability} return entries @staticmethod def _merge_specs( registry_specs: dict[str, dict], override_specs: dict[str, dict], ) -> dict[str, dict]: specs: dict[str, dict] = {} for canonical, entry in registry_specs.items(): specs[canonical] = { "deployment": entry.get("deployment"), "capability": entry.get("capability"), "overrides": {}, } for canonical, entry in override_specs.items(): spec = specs.get(canonical, {"deployment": None, "capability": None, "overrides": {}}) deployment = entry.get("deployment") if deployment: spec["deployment"] = deployment overrides = {k: v for k, v in entry.items() if k not in {"deployment"}} overrides.pop("capability", None) if overrides: spec["overrides"].update(overrides) specs[canonical] = spec return {k: v for k, v in specs.items() if v.get("deployment")} @staticmethod def _normalise_deployments(mapping: dict[str, object]) -> dict[str, dict]: normalised: dict[str, dict] = {} for canonical, spec in mapping.items(): canonical_name = (canonical or "").strip() if not canonical_name: continue deployment_name: str | None = None overrides: dict[str, object] = {} if isinstance(spec, str): deployment_name = spec.strip() elif isinstance(spec, dict): deployment_name = spec.get("deployment") or spec.get("deployment_name") overrides = {k: v for k, v in spec.items() if k not in {"deployment", "deployment_name"}} if not deployment_name: continue normalised[canonical_name] = {"deployment": deployment_name.strip(), **overrides} return normalised # ------------------------------------------------------------------ # Azure-specific configuration # ------------------------------------------------------------------ @property def client(self): # type: ignore[override] """Instantiate the Azure OpenAI client on first use.""" if self._client is None: if AzureOpenAI is None: raise ImportError( "Azure OpenAI support requires the 'openai' package. Install it with `pip install openai`." ) import httpx proxy_env_vars = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] with suppress_env_vars(*proxy_env_vars): try: timeout_config = self.timeout_config http_client = httpx.Client(timeout=timeout_config, follow_redirects=True) client_kwargs = { "api_key": self.api_key, "azure_endpoint": self.azure_endpoint, "api_version": self.api_version, "http_client": http_client, } if self.DEFAULT_HEADERS: client_kwargs["default_headers"] = self.DEFAULT_HEADERS.copy() logger.debug( "Initializing Azure OpenAI client endpoint=%s api_version=%s timeouts=%s", self.azure_endpoint, self.api_version, timeout_config, ) self._client = AzureOpenAI(**client_kwargs) except Exception as exc: logger.error("Failed to create Azure OpenAI client: %s", exc) raise return self._client # ------------------------------------------------------------------ # Request delegation # ------------------------------------------------------------------ def generate_content( self, prompt: str, model_name: str, system_prompt: str | None = None, temperature: float = 0.3, max_output_tokens: int | None = None, images: list[str] | None = None, **kwargs, ) -> ModelResponse: canonical_name, deployment_name = self._resolve_canonical_and_deployment(model_name) # Delegate to the shared OpenAI-compatible implementation using the # deployment name – Azure requires the deployment identifier in the # ``model`` field. The returned ``ModelResponse`` is normalised so # downstream consumers continue to see the canonical model name. raw_response = super().generate_content( prompt=prompt, model_name=deployment_name, system_prompt=system_prompt, temperature=temperature, max_output_tokens=max_output_tokens, images=images, **kwargs, ) capabilities = self._capabilities.get(canonical_name) friendly_name = capabilities.friendly_name if capabilities else self.FRIENDLY_NAME return ModelResponse( content=raw_response.content, usage=raw_response.usage, model_name=canonical_name, friendly_name=friendly_name, provider=ProviderType.AZURE, metadata={**raw_response.metadata, "deployment": deployment_name}, ) def _resolve_canonical_and_deployment(self, model_name: str) -> tuple[str, str]: resolved_canonical = self._resolve_model_name(model_name) if resolved_canonical not in self._deployment_map: # The base resolver may hand back the deployment alias. Try to map it # back to a canonical entry. for canonical, deployment in self._deployment_map.items(): if deployment.lower() == resolved_canonical.lower(): return canonical, deployment raise ValueError(f"Model '{model_name}' is not configured for Azure OpenAI") return resolved_canonical, self._deployment_map[resolved_canonical] def _parse_allowed_models(self) -> set[str] | None: # type: ignore[override] # Support both AZURE_ALLOWED_MODELS (inherited behaviour) and the # clearer AZURE_OPENAI_ALLOWED_MODELS alias. explicit = get_env("AZURE_OPENAI_ALLOWED_MODELS") if explicit: models = {m.strip().lower() for m in explicit.split(",") if m.strip()} if models: logger.info("Configured allowed models for Azure OpenAI: %s", sorted(models)) self._allowed_alias_cache = {} return models return super()._parse_allowed_models() ================================================ FILE: providers/base.py ================================================ """Base interfaces and common behaviour for model providers.""" import logging import time from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Callable, Optional if TYPE_CHECKING: from tools.models import ToolModelCategory from .shared import ModelCapabilities, ModelResponse, ProviderType logger = logging.getLogger(__name__) class ModelProvider(ABC): """Abstract base class for all model backends in the MCP server. Role Defines the interface every provider must implement so the registry, restriction service, and tools have a uniform surface for listing models, resolving aliases, and executing requests. Responsibilities * expose static capability metadata for each supported model via :class:`ModelCapabilities` * accept user prompts, forward them to the underlying SDK, and wrap responses in :class:`ModelResponse` * report tokenizer counts for budgeting and validation logic * advertise provider identity (``ProviderType``) so restriction policies can map environment configuration onto providers * validate whether a model name or alias is recognised by the provider Shared helpers like temperature validation, alias resolution, and restriction-aware ``list_models`` live here so concrete subclasses only need to supply their catalogue and wire up SDK-specific behaviour. """ # All concrete providers must define their supported models MODEL_CAPABILITIES: dict[str, Any] = {} def __init__(self, api_key: str, **kwargs): """Initialize the provider with API key and optional configuration.""" self.api_key = api_key self.config = kwargs self._sorted_capabilities_cache: Optional[list[tuple[str, ModelCapabilities]]] = None # ------------------------------------------------------------------ # Provider identity & capability surface # ------------------------------------------------------------------ @abstractmethod def get_provider_type(self) -> ProviderType: """Return the concrete provider identity.""" def get_capabilities(self, model_name: str) -> ModelCapabilities: """Resolve capability metadata for a model name. This centralises the alias resolution → lookup → restriction check pipeline so providers only override the pieces they genuinely need to customise. Subclasses usually only override ``_lookup_capabilities`` to integrate a registry or dynamic source, or ``_finalise_capabilities`` to tweak the returned object. Args: model_name: Canonical model name or its alias """ resolved_model_name = self._resolve_model_name(model_name) capabilities = self._lookup_capabilities(resolved_model_name, model_name) if capabilities is None: self._raise_unsupported_model(model_name) self._ensure_model_allowed(capabilities, resolved_model_name, model_name) return self._finalise_capabilities(capabilities, resolved_model_name, model_name) def get_all_model_capabilities(self) -> dict[str, ModelCapabilities]: """Return statically declared capabilities when available.""" model_map = getattr(self, "MODEL_CAPABILITIES", None) if isinstance(model_map, dict) and model_map: return {k: v for k, v in model_map.items() if isinstance(v, ModelCapabilities)} return {} def get_capabilities_by_rank(self) -> list[tuple[str, ModelCapabilities]]: """Return model capabilities sorted by effective capability rank.""" if self._sorted_capabilities_cache is not None: return list(self._sorted_capabilities_cache) model_configs = self.get_all_model_capabilities() if not model_configs: self._sorted_capabilities_cache = [] return [] items = list(model_configs.items()) items.sort(key=lambda item: (-item[1].get_effective_capability_rank(), item[0])) self._sorted_capabilities_cache = items return list(items) def _invalidate_capability_cache(self) -> None: """Clear cached sorted capability data (call after dynamic updates).""" self._sorted_capabilities_cache = None def list_models( self, *, respect_restrictions: bool = True, include_aliases: bool = True, lowercase: bool = False, unique: bool = False, ) -> list[str]: """Return formatted model names supported by this provider.""" model_configs = self.get_all_model_capabilities() if not model_configs: return [] restriction_service = None if respect_restrictions: from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if restriction_service: allowed_configs = {} for model_name, config in model_configs.items(): if restriction_service.is_allowed(self.get_provider_type(), model_name): allowed_configs[model_name] = config model_configs = allowed_configs if not model_configs: return [] return ModelCapabilities.collect_model_names( model_configs, include_aliases=include_aliases, lowercase=lowercase, unique=unique, ) # ------------------------------------------------------------------ # Request execution # ------------------------------------------------------------------ @abstractmethod def generate_content( self, prompt: str, model_name: str, system_prompt: Optional[str] = None, temperature: float = 0.3, max_output_tokens: Optional[int] = None, **kwargs, ) -> ModelResponse: """Generate content using the model. This is the core method that all providers must implement to generate responses from their models. Providers should handle model-specific capabilities and constraints appropriately. Args: prompt: The main user prompt/query to send to the model model_name: Canonical model name or its alias that the provider supports system_prompt: Optional system instructions to prepend to the prompt for establishing context, behavior, or role temperature: Controls randomness in generation (0.0=deterministic, 1.0=creative), default 0.3. Some models may not support temperature control max_output_tokens: Optional maximum number of tokens to generate in the response. If not specified, uses the model's default limit **kwargs: Additional provider-specific parameters that vary by implementation (e.g., thinking_mode for Gemini, top_p for OpenAI, images for vision models) Returns: ModelResponse: Standardized response object containing: - content: The generated text response - usage: Token usage statistics (input/output/total) - model_name: The model that was actually used - friendly_name: Human-readable provider/model identifier - provider: The ProviderType enum value - metadata: Provider-specific metadata (finish_reason, safety info, etc.) Raises: ValueError: If the model is not supported, parameters are invalid, or the model is restricted by policy RuntimeError: If the API call fails after retries """ def count_tokens(self, text: str, model_name: str) -> int: """Estimate token usage for a piece of text.""" resolved_model = self._resolve_model_name(model_name) if not text: return 0 estimated = max(1, len(text) // 4) logger.debug("Estimating %s tokens for model %s via character heuristic", estimated, resolved_model) return estimated def close(self) -> None: """Clean up any resources held by the provider.""" return # ------------------------------------------------------------------ # Retry helpers # ------------------------------------------------------------------ def _is_error_retryable(self, error: Exception) -> bool: """Return True when an error warrants another attempt. Subclasses with structured provider errors should override this hook. The default implementation only retries obvious transient failures such as timeouts or 5xx responses detected via string inspection. """ error_str = str(error).lower() if "429" in error_str or "rate limit" in error_str: return False retryable_indicators = [ "timeout", "connection", "temporary", "unavailable", "retry", "reset", "refused", "broken pipe", "tls", "handshake", "network", "500", "502", "503", "504", ] return any(indicator in error_str for indicator in retryable_indicators) def _run_with_retries( self, operation: Callable[[], Any], *, max_attempts: int, delays: Optional[list[float]] = None, log_prefix: str = "", ): """Execute ``operation`` with retry semantics. Args: operation: Callable returning the provider result. max_attempts: Maximum number of attempts (>=1). delays: Optional list of sleep durations between attempts. log_prefix: Optional identifier for log clarity. Returns: Whatever ``operation`` returns. Raises: The last exception when all retries fail or the error is not retryable. """ if max_attempts < 1: raise ValueError("max_attempts must be >= 1") attempts = max_attempts delays = delays or [] last_exc: Optional[Exception] = None for attempt_index in range(attempts): try: return operation() except Exception as exc: # noqa: BLE001 - bubble exact provider errors last_exc = exc attempt_number = attempt_index + 1 # Decide whether to retry based on subclass hook retryable = self._is_error_retryable(exc) if not retryable or attempt_number >= attempts: raise delay_idx = min(attempt_index, len(delays) - 1) if delays else -1 delay = delays[delay_idx] if delay_idx >= 0 else 0.0 if delay > 0: logger.warning( "%s retryable error (attempt %s/%s): %s. Retrying in %ss...", log_prefix or self.__class__.__name__, attempt_number, attempts, exc, delay, ) time.sleep(delay) else: logger.warning( "%s retryable error (attempt %s/%s): %s. Retrying...", log_prefix or self.__class__.__name__, attempt_number, attempts, exc, ) # Should never reach here because loop either returns or raises raise last_exc if last_exc else RuntimeError("Retry loop exited without result") # ------------------------------------------------------------------ # Validation hooks # ------------------------------------------------------------------ def validate_model_name(self, model_name: str) -> bool: """ Return ``True`` when the model resolves to an allowed capability. Args: model_name: Canonical model name or its alias """ try: self.get_capabilities(model_name) except ValueError: return False return True def validate_parameters(self, model_name: str, temperature: float, **kwargs) -> None: """ Validate model parameters against capabilities. Args: model_name: Canonical model name or its alias """ capabilities = self.get_capabilities(model_name) if not capabilities.temperature_constraint.validate(temperature): constraint_desc = capabilities.temperature_constraint.get_description() raise ValueError(f"Temperature {temperature} is invalid for model {model_name}. {constraint_desc}") # ------------------------------------------------------------------ # Preference / registry hooks # ------------------------------------------------------------------ def get_preferred_model(self, category: "ToolModelCategory", allowed_models: list[str]) -> Optional[str]: """Get the preferred model from this provider for a given category.""" return None def get_model_registry(self) -> Optional[dict[str, Any]]: """Return the model registry backing this provider, if any.""" return None # ------------------------------------------------------------------ # Capability lookup pipeline # ------------------------------------------------------------------ def _lookup_capabilities( self, canonical_name: str, requested_name: Optional[str] = None, ) -> Optional[ModelCapabilities]: """Return ``ModelCapabilities`` for the canonical model name.""" return self.get_all_model_capabilities().get(canonical_name) def _ensure_model_allowed( self, capabilities: ModelCapabilities, canonical_name: str, requested_name: str, ) -> None: """Raise ``ValueError`` if the model violates restriction policy.""" try: from utils.model_restrictions import get_restriction_service except Exception: # pragma: no cover - only triggered if service import breaks return restriction_service = get_restriction_service() if not restriction_service: return if restriction_service.is_allowed(self.get_provider_type(), canonical_name, requested_name): return raise ValueError( f"{self.get_provider_type().value} model '{canonical_name}' is not allowed by restriction policy." ) def _finalise_capabilities( self, capabilities: ModelCapabilities, canonical_name: str, requested_name: str, ) -> ModelCapabilities: """Allow subclasses to adjust capability metadata before returning.""" return capabilities def _raise_unsupported_model(self, model_name: str) -> None: """Raise the canonical unsupported-model error.""" raise ValueError(f"Unsupported model '{model_name}' for provider {self.get_provider_type().value}.") def _resolve_model_name(self, model_name: str) -> str: """Resolve model shorthand to full name. This implementation uses the hook methods to support different model configuration sources. Args: model_name: Canonical model name or its alias Returns: Resolved model name """ # Get model configurations from the hook method model_configs = self.get_all_model_capabilities() # First check if it's already a base model name (case-sensitive exact match) if model_name in model_configs: return model_name # Check case-insensitively for both base models and aliases model_name_lower = model_name.lower() # Check base model names case-insensitively for base_model in model_configs: if base_model.lower() == model_name_lower: return base_model # Check aliases from the model configurations alias_map = ModelCapabilities.collect_aliases(model_configs) for base_model, aliases in alias_map.items(): if any(alias.lower() == model_name_lower for alias in aliases): return base_model # If not found, return as-is return model_name ================================================ FILE: providers/custom.py ================================================ """Custom API provider implementation.""" import logging from utils.env import get_env from .openai_compatible import OpenAICompatibleProvider from .registries.custom import CustomEndpointModelRegistry from .registries.openrouter import OpenRouterModelRegistry from .shared import ModelCapabilities, ProviderType class CustomProvider(OpenAICompatibleProvider): """Adapter for self-hosted or local OpenAI-compatible endpoints. Role Provide a uniform bridge between the MCP server and user-managed OpenAI-compatible services (Ollama, vLLM, LM Studio, bespoke gateways). By subclassing :class:`OpenAICompatibleProvider` it inherits request and token handling, while the custom registry exposes locally defined model metadata. Notable behaviour * Uses :class:`OpenRouterModelRegistry` to load model definitions and aliases so custom deployments share the same metadata pipeline as OpenRouter itself. * Normalises version-tagged model names (``model:latest``) and applies restriction policies just like cloud providers, ensuring consistent behaviour across environments. """ FRIENDLY_NAME = "Custom API" # Model registry for managing configurations and aliases _registry: CustomEndpointModelRegistry | None = None def __init__(self, api_key: str = "", base_url: str = "", **kwargs): """Initialize Custom provider for local/self-hosted models. This provider supports any OpenAI-compatible API endpoint including: - Ollama (typically no API key required) - vLLM (may require API key) - LM Studio (may require API key) - Text Generation WebUI (may require API key) - Enterprise/self-hosted APIs (typically require API key) Args: api_key: API key for the custom endpoint. Can be empty string for providers that don't require authentication (like Ollama). Falls back to CUSTOM_API_KEY environment variable if not provided. base_url: Base URL for the custom API endpoint (e.g., 'http://localhost:11434/v1'). Falls back to CUSTOM_API_URL environment variable if not provided. **kwargs: Additional configuration passed to parent OpenAI-compatible provider Raises: ValueError: If no base_url is provided via parameter or environment variable """ # Fall back to environment variables only if not provided if not base_url: base_url = get_env("CUSTOM_API_URL", "") or "" if not api_key: api_key = get_env("CUSTOM_API_KEY", "") or "" if not base_url: raise ValueError( "Custom API URL must be provided via base_url parameter or CUSTOM_API_URL environment variable" ) # For Ollama and other providers that don't require authentication, # set a dummy API key to avoid OpenAI client header issues if not api_key: api_key = "dummy-key-for-unauthenticated-endpoint" logging.debug("Using dummy API key for unauthenticated custom endpoint") logging.info(f"Initializing Custom provider with endpoint: {base_url}") self._alias_cache: dict[str, str] = {} super().__init__(api_key, base_url=base_url, **kwargs) # Initialize model registry if CustomProvider._registry is None: CustomProvider._registry = CustomEndpointModelRegistry() # Log loaded models and aliases only on first load models = self._registry.list_models() aliases = self._registry.list_aliases() logging.info(f"Custom provider loaded {len(models)} models with {len(aliases)} aliases") # ------------------------------------------------------------------ # Capability surface # ------------------------------------------------------------------ def _lookup_capabilities( self, canonical_name: str, requested_name: str | None = None, ) -> ModelCapabilities | None: """Return capabilities for models explicitly marked as custom.""" builtin = super()._lookup_capabilities(canonical_name, requested_name) if builtin is not None: return builtin registry_entry = self._registry.resolve(canonical_name) if registry_entry: registry_entry.provider = ProviderType.CUSTOM return registry_entry logging.debug( "Custom provider cannot resolve model '%s'; ensure it is declared in custom_models.json", canonical_name, ) return None def get_provider_type(self) -> ProviderType: """Identify this provider for restriction and logging logic.""" return ProviderType.CUSTOM # ------------------------------------------------------------------ # Registry helpers # ------------------------------------------------------------------ def _resolve_model_name(self, model_name: str) -> str: """Resolve registry aliases and strip version tags for local models.""" cache_key = model_name.lower() if cache_key in self._alias_cache: return self._alias_cache[cache_key] config = self._registry.resolve(model_name) if config: if config.model_name != model_name: logging.debug("Resolved model alias '%s' to '%s'", model_name, config.model_name) resolved = config.model_name self._alias_cache[cache_key] = resolved self._alias_cache.setdefault(resolved.lower(), resolved) return resolved if ":" in model_name: base_model = model_name.split(":")[0] logging.debug(f"Stripped version tag from '{model_name}' -> '{base_model}'") base_config = self._registry.resolve(base_model) if base_config: logging.debug("Resolved base model '%s' to '%s'", base_model, base_config.model_name) resolved = base_config.model_name self._alias_cache[cache_key] = resolved self._alias_cache.setdefault(resolved.lower(), resolved) return resolved self._alias_cache[cache_key] = base_model return base_model logging.debug(f"Model '{model_name}' not found in registry, using as-is") # Attempt to resolve via OpenRouter registry so aliases still map cleanly openrouter_registry = OpenRouterModelRegistry() openrouter_config = openrouter_registry.resolve(model_name) if openrouter_config: resolved = openrouter_config.model_name self._alias_cache[cache_key] = resolved self._alias_cache.setdefault(resolved.lower(), resolved) return resolved self._alias_cache[cache_key] = model_name return model_name def get_all_model_capabilities(self) -> dict[str, ModelCapabilities]: """Expose registry capabilities for models marked as custom.""" if not self._registry: return {} capabilities = {} for model in self._registry.list_models(): config = self._registry.resolve(model) if config: capabilities[model] = config return capabilities ================================================ FILE: providers/dial.py ================================================ """DIAL (Data & AI Layer) model provider implementation.""" import logging import threading from typing import ClassVar, Optional from utils.env import get_env from .openai_compatible import OpenAICompatibleProvider from .registries.dial import DialModelRegistry from .registry_provider_mixin import RegistryBackedProviderMixin from .shared import ModelCapabilities, ModelResponse, ProviderType logger = logging.getLogger(__name__) class DIALModelProvider(RegistryBackedProviderMixin, OpenAICompatibleProvider): """Client for the DIAL (Data & AI Layer) aggregation service. DIAL exposes several third-party models behind a single OpenAI-compatible endpoint. This provider wraps the service, publishes capability metadata for the known deployments, and centralises retry/backoff settings tailored to DIAL's latency characteristics. """ FRIENDLY_NAME = "DIAL" REGISTRY_CLASS = DialModelRegistry MODEL_CAPABILITIES: ClassVar[dict[str, ModelCapabilities]] = {} # Retry configuration for API calls MAX_RETRIES = 4 RETRY_DELAYS = [1, 3, 5, 8] # seconds def __init__(self, api_key: str, **kwargs): """Initialize DIAL provider with API key and host. Args: api_key: DIAL API key for authentication **kwargs: Additional configuration options """ self._ensure_registry() # Get DIAL API host from environment or kwargs dial_host = kwargs.get("base_url") or get_env("DIAL_API_HOST") or "https://core.dialx.ai" # DIAL uses /openai endpoint for OpenAI-compatible API if not dial_host.endswith("/openai"): dial_host = f"{dial_host.rstrip('/')}/openai" kwargs["base_url"] = dial_host # Get API version from environment or use default self.api_version = get_env("DIAL_API_VERSION", "2024-12-01-preview") or "2024-12-01-preview" # Add DIAL-specific headers # DIAL uses Api-Key header instead of Authorization: Bearer # Reference: https://dialx.ai/dial_api#section/Authorization self.DEFAULT_HEADERS = { "Api-Key": api_key, } # Store the actual API key for use in Api-Key header self._dial_api_key = api_key # Pass a placeholder API key to OpenAI client - we'll override the auth header in httpx # The actual authentication happens via the Api-Key header in the httpx client super().__init__("placeholder-not-used", **kwargs) # Cache for deployment-specific clients to avoid recreating them on each request self._deployment_clients = {} # Lock to ensure thread-safe client creation self._client_lock = threading.Lock() # Create a SINGLE shared httpx client for the provider instance import httpx # Create custom event hooks to remove Authorization header def remove_auth_header(request): """Remove Authorization header that OpenAI client adds.""" # httpx headers are case-insensitive, so we need to check all variations headers_to_remove = [] for header_name in request.headers: if header_name.lower() == "authorization": headers_to_remove.append(header_name) for header_name in headers_to_remove: del request.headers[header_name] self._http_client = httpx.Client( timeout=self.timeout_config, verify=True, follow_redirects=True, headers=self.DEFAULT_HEADERS.copy(), # Include DIAL headers including Api-Key limits=httpx.Limits( max_keepalive_connections=5, max_connections=10, keepalive_expiry=30.0, ), event_hooks={"request": [remove_auth_header]}, ) logger.info(f"Initialized DIAL provider with host: {dial_host} and api-version: {self.api_version}") def get_provider_type(self) -> ProviderType: """Get the provider type.""" return ProviderType.DIAL def _get_deployment_client(self, deployment: str): """Get or create a cached client for a specific deployment. This avoids recreating OpenAI clients on every request, improving performance. Reuses the shared HTTP client for connection pooling. Args: deployment: The deployment/model name Returns: OpenAI client configured for the specific deployment """ # Check if client already exists without locking for performance if deployment in self._deployment_clients: return self._deployment_clients[deployment] # Use lock to ensure thread-safe client creation with self._client_lock: # Double-check pattern: check again inside the lock if deployment not in self._deployment_clients: from openai import OpenAI # Build deployment-specific URL base_url = str(self.client.base_url) if base_url.endswith("/"): base_url = base_url[:-1] # Remove /openai suffix if present to reconstruct properly if base_url.endswith("/openai"): base_url = base_url[:-7] deployment_url = f"{base_url}/openai/deployments/{deployment}" # Create and cache the client, REUSING the shared http_client # Use placeholder API key - Authorization header will be removed by http_client event hook self._deployment_clients[deployment] = OpenAI( api_key="placeholder-not-used", base_url=deployment_url, http_client=self._http_client, # Pass the shared client with Api-Key header default_query={"api-version": self.api_version}, # Add api-version as query param ) return self._deployment_clients[deployment] def generate_content( self, prompt: str, model_name: str, system_prompt: Optional[str] = None, temperature: float = 0.3, max_output_tokens: Optional[int] = None, images: Optional[list[str]] = None, **kwargs, ) -> ModelResponse: """Generate content using DIAL's deployment-specific endpoint. DIAL uses Azure OpenAI-style deployment endpoints: /openai/deployments/{deployment}/chat/completions Args: prompt: The main user prompt/query to send to the model model_name: Model name or alias (e.g., "o3", "sonnet-4.1", "gemini-2.5-pro") system_prompt: Optional system instructions to prepend to the prompt for context/behavior temperature: Sampling temperature for randomness (0.0=deterministic, 1.0=creative), default 0.3 Note: O3/O4 models don't support temperature and will ignore this parameter max_output_tokens: Optional maximum number of tokens to generate in the response images: Optional list of image paths or data URLs to include with the prompt (for vision-capable models) **kwargs: Additional OpenAI-compatible parameters (top_p, frequency_penalty, presence_penalty, seed, stop) Returns: ModelResponse: Contains the generated content, token usage stats, model metadata, and finish reason """ # Validate model name against allow-list if not self.validate_model_name(model_name): raise ValueError(f"Model '{model_name}' not in allowed models list. Allowed models: {self.allowed_models}") # Validate parameters and fetch capabilities self.validate_parameters(model_name, temperature) capabilities = self.get_capabilities(model_name) # Prepare messages messages = [] if system_prompt: messages.append({"role": "system", "content": system_prompt}) # Build user message content user_message_content = [] if prompt: user_message_content.append({"type": "text", "text": prompt}) if images and capabilities.supports_images: for img_path in images: processed_image = self._process_image(img_path) if processed_image: user_message_content.append(processed_image) elif images: logger.warning(f"Model {model_name} does not support images, ignoring {len(images)} image(s)") # Add user message. If only text, content will be a string, otherwise a list. if len(user_message_content) == 1 and user_message_content[0]["type"] == "text": messages.append({"role": "user", "content": prompt}) else: messages.append({"role": "user", "content": user_message_content}) # Resolve model name resolved_model = self._resolve_model_name(model_name) # Build completion parameters completion_params = { "model": resolved_model, "messages": messages, "stream": False, } # Determine temperature support from capabilities supports_temperature = capabilities.supports_temperature # Add temperature parameter if supported if supports_temperature: completion_params["temperature"] = temperature # Add max tokens if specified and model supports it if max_output_tokens and supports_temperature: completion_params["max_tokens"] = max_output_tokens # Add additional parameters for key, value in kwargs.items(): if key in ["top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "stream"]: if not supports_temperature and key in ["top_p", "frequency_penalty", "presence_penalty", "stream"]: continue completion_params[key] = value # DIAL-specific: Get cached client for deployment endpoint deployment_client = self._get_deployment_client(resolved_model) attempt_counter = {"value": 0} def _attempt() -> ModelResponse: attempt_counter["value"] += 1 response = deployment_client.chat.completions.create(**completion_params) content = response.choices[0].message.content usage = self._extract_usage(response) return ModelResponse( content=content, usage=usage, model_name=model_name, friendly_name=self.FRIENDLY_NAME, provider=self.get_provider_type(), metadata={ "finish_reason": response.choices[0].finish_reason, "model": response.model, "id": response.id, "created": response.created, }, ) try: return self._run_with_retries( operation=_attempt, max_attempts=self.MAX_RETRIES, delays=self.RETRY_DELAYS, log_prefix=f"DIAL API ({resolved_model})", ) except Exception as exc: attempts = max(attempt_counter["value"], 1) if attempts == 1: raise ValueError(f"DIAL API error for model {resolved_model}: {exc}") from exc raise ValueError(f"DIAL API error for model {resolved_model} after {attempts} attempts: {exc}") from exc def close(self) -> None: """Clean up HTTP clients when provider is closed.""" logger.info("Closing DIAL provider HTTP clients...") # Clear the deployment clients cache # Note: We don't need to close individual OpenAI clients since they # use the shared httpx.Client which we close separately self._deployment_clients.clear() # Close the shared HTTP client if hasattr(self, "_http_client"): try: self._http_client.close() logger.debug("Closed shared HTTP client") except Exception as e: logger.warning(f"Error closing shared HTTP client: {e}") # Also close the client created by the superclass (OpenAICompatibleProvider) # as it holds its own httpx.Client instance that is not used by DIAL's generate_content if hasattr(self, "client") and self.client and hasattr(self.client, "close"): try: self.client.close() logger.debug("Closed superclass's OpenAI client") except Exception as e: logger.warning(f"Error closing superclass's OpenAI client: {e}") ================================================ FILE: providers/gemini.py ================================================ """Gemini model provider implementation.""" import base64 import logging from typing import TYPE_CHECKING, ClassVar, Optional if TYPE_CHECKING: from tools.models import ToolModelCategory from google import genai from google.genai import types from utils.env import get_env from utils.image_utils import validate_image from .base import ModelProvider from .registries.gemini import GeminiModelRegistry from .registry_provider_mixin import RegistryBackedProviderMixin from .shared import ModelCapabilities, ModelResponse, ProviderType logger = logging.getLogger(__name__) class GeminiModelProvider(RegistryBackedProviderMixin, ModelProvider): """First-party Gemini integration built on the official Google SDK. The provider advertises detailed thinking-mode budgets, handles optional custom endpoints, and performs image pre-processing before forwarding a request to the Gemini APIs. """ REGISTRY_CLASS = GeminiModelRegistry MODEL_CAPABILITIES: ClassVar[dict[str, ModelCapabilities]] = {} # Thinking mode configurations - percentages of model's max_thinking_tokens # These percentages work across all models that support thinking THINKING_BUDGETS = { "minimal": 0.005, # 0.5% of max - minimal thinking for fast responses "low": 0.08, # 8% of max - light reasoning tasks "medium": 0.33, # 33% of max - balanced reasoning (default) "high": 0.67, # 67% of max - complex analysis "max": 1.0, # 100% of max - full thinking budget } def __init__(self, api_key: str, **kwargs): """Initialize Gemini provider with API key and optional base URL.""" self._ensure_registry() super().__init__(api_key, **kwargs) self._client = None self._token_counters = {} # Cache for token counting self._base_url = kwargs.get("base_url", None) # Optional custom endpoint self._timeout_override = self._resolve_http_timeout() self._invalidate_capability_cache() # ------------------------------------------------------------------ # Capability surface # ------------------------------------------------------------------ # ------------------------------------------------------------------ # Client access # ------------------------------------------------------------------ @property def client(self): """Lazy initialization of Gemini client.""" if self._client is None: http_options_kwargs: dict[str, object] = {} if self._base_url: http_options_kwargs["base_url"] = self._base_url if self._timeout_override is not None: http_options_kwargs["timeout"] = self._timeout_override if http_options_kwargs: http_options = types.HttpOptions(**http_options_kwargs) logger.debug( "Initializing Gemini client with options: base_url=%s timeout=%s", http_options_kwargs.get("base_url"), http_options_kwargs.get("timeout"), ) self._client = genai.Client(api_key=self.api_key, http_options=http_options) else: self._client = genai.Client(api_key=self.api_key) return self._client def _resolve_http_timeout(self) -> Optional[float]: """Compute timeout override from shared custom timeout environment variables.""" timeouts: list[float] = [] for env_var in [ "CUSTOM_CONNECT_TIMEOUT", "CUSTOM_READ_TIMEOUT", "CUSTOM_WRITE_TIMEOUT", "CUSTOM_POOL_TIMEOUT", ]: raw_value = get_env(env_var) if raw_value: try: timeouts.append(float(raw_value)) except (TypeError, ValueError): logger.warning("Invalid %s value '%s'; ignoring.", env_var, raw_value) if timeouts: # Use the largest timeout to best approximate long-running requests resolved = max(timeouts) logger.debug("Using custom Gemini HTTP timeout: %ss", resolved) return resolved return None # ------------------------------------------------------------------ # Request execution # ------------------------------------------------------------------ def generate_content( self, prompt: str, model_name: str, system_prompt: Optional[str] = None, temperature: float = 1.0, max_output_tokens: Optional[int] = None, thinking_mode: str = "medium", images: Optional[list[str]] = None, **kwargs, ) -> ModelResponse: """ Generate content using Gemini model. Args: prompt: The main user prompt/query to send to the model model_name: Canonical model name or its alias (e.g., "gemini-2.5-pro", "flash", "pro") system_prompt: Optional system instructions to prepend to the prompt for context/behavior temperature: Controls randomness in generation (0.0=deterministic, 1.0=creative), default 0.3 max_output_tokens: Optional maximum number of tokens to generate in the response thinking_mode: Thinking budget level for models that support it ("minimal", "low", "medium", "high", "max"), default "medium" images: Optional list of image paths or data URLs to include with the prompt (for vision models) **kwargs: Additional keyword arguments (reserved for future use) Returns: ModelResponse: Contains the generated content, token usage stats, model metadata, and safety information """ # Validate parameters and fetch capabilities self.validate_parameters(model_name, temperature) capabilities = self.get_capabilities(model_name) capability_map = self.get_all_model_capabilities() resolved_model_name = self._resolve_model_name(model_name) # Prepare content parts (text and potentially images) parts = [] # Add system and user prompts as text if system_prompt: full_prompt = f"{system_prompt}\n\n{prompt}" else: full_prompt = prompt parts.append({"text": full_prompt}) # Add images if provided and model supports vision if images and capabilities.supports_images: for image_path in images: try: image_part = self._process_image(image_path) if image_part: parts.append(image_part) except Exception as e: logger.warning(f"Failed to process image {image_path}: {e}") # Continue with other images and text continue elif images and not capabilities.supports_images: logger.warning(f"Model {resolved_model_name} does not support images, ignoring {len(images)} image(s)") # Create contents structure contents = [{"parts": parts}] # Gemini 3 Pro Preview currently rejects medium thinking budgets; bump to high. effective_thinking_mode = thinking_mode if resolved_model_name == "gemini-3-pro-preview" and thinking_mode == "medium": logger.debug( "Overriding thinking mode 'medium' with 'high' for %s due to launch limitation", resolved_model_name, ) effective_thinking_mode = "high" # Prepare generation config generation_config = types.GenerateContentConfig( temperature=temperature, candidate_count=1, ) # Add max output tokens if specified if max_output_tokens: generation_config.max_output_tokens = max_output_tokens # Add thinking configuration for models that support it if capabilities.supports_extended_thinking and effective_thinking_mode in self.THINKING_BUDGETS: # Get model's max thinking tokens and calculate actual budget model_config = capability_map.get(resolved_model_name) if model_config and model_config.max_thinking_tokens > 0: max_thinking_tokens = model_config.max_thinking_tokens actual_thinking_budget = int(max_thinking_tokens * self.THINKING_BUDGETS[effective_thinking_mode]) generation_config.thinking_config = types.ThinkingConfig(thinking_budget=actual_thinking_budget) # Retry logic with progressive delays max_retries = 4 # Total of 4 attempts retry_delays = [1, 3, 5, 8] # Progressive delays: 1s, 3s, 5s, 8s attempt_counter = {"value": 0} def _attempt() -> ModelResponse: attempt_counter["value"] += 1 response = self.client.models.generate_content( model=resolved_model_name, contents=contents, config=generation_config, ) usage = self._extract_usage(response) finish_reason_str = "UNKNOWN" is_blocked_by_safety = False safety_feedback_details = None if response.candidates: candidate = response.candidates[0] try: finish_reason_enum = candidate.finish_reason if finish_reason_enum: try: finish_reason_str = finish_reason_enum.name except AttributeError: finish_reason_str = str(finish_reason_enum) else: finish_reason_str = "STOP" except AttributeError: finish_reason_str = "STOP" if not response.text: try: safety_ratings = candidate.safety_ratings if safety_ratings: for rating in safety_ratings: try: if rating.blocked: is_blocked_by_safety = True category_name = "UNKNOWN" probability_name = "UNKNOWN" try: category_name = rating.category.name except (AttributeError, TypeError): pass try: probability_name = rating.probability.name except (AttributeError, TypeError): pass safety_feedback_details = ( f"Category: {category_name}, Probability: {probability_name}" ) break except (AttributeError, TypeError): continue except (AttributeError, TypeError): pass elif response.candidates is not None and len(response.candidates) == 0: is_blocked_by_safety = True finish_reason_str = "SAFETY" safety_feedback_details = "Prompt blocked, reason unavailable" try: prompt_feedback = response.prompt_feedback if prompt_feedback and prompt_feedback.block_reason: try: block_reason_name = prompt_feedback.block_reason.name except AttributeError: block_reason_name = str(prompt_feedback.block_reason) safety_feedback_details = f"Prompt blocked, reason: {block_reason_name}" except (AttributeError, TypeError): pass return ModelResponse( content=response.text, usage=usage, model_name=resolved_model_name, friendly_name="Gemini", provider=ProviderType.GOOGLE, metadata={ "thinking_mode": effective_thinking_mode if capabilities.supports_extended_thinking else None, "finish_reason": finish_reason_str, "is_blocked_by_safety": is_blocked_by_safety, "safety_feedback": safety_feedback_details, }, ) try: return self._run_with_retries( operation=_attempt, max_attempts=max_retries, delays=retry_delays, log_prefix=f"Gemini API ({resolved_model_name})", ) except Exception as exc: attempts = max(attempt_counter["value"], 1) error_msg = ( f"Gemini API error for model {resolved_model_name} after {attempts} attempt" f"{'s' if attempts > 1 else ''}: {exc}" ) raise RuntimeError(error_msg) from exc def get_provider_type(self) -> ProviderType: """Get the provider type.""" return ProviderType.GOOGLE def _extract_usage(self, response) -> dict[str, int]: """Extract token usage from Gemini response.""" usage = {} # Try to extract usage metadata from response # Note: The actual structure depends on the SDK version and response format try: metadata = response.usage_metadata if metadata: # Extract token counts with explicit None checks input_tokens = None output_tokens = None try: value = metadata.prompt_token_count if value is not None: input_tokens = value usage["input_tokens"] = value except (AttributeError, TypeError): pass try: value = metadata.candidates_token_count if value is not None: output_tokens = value usage["output_tokens"] = value except (AttributeError, TypeError): pass # Calculate total only if both values are available and valid if input_tokens is not None and output_tokens is not None: usage["total_tokens"] = input_tokens + output_tokens except (AttributeError, TypeError): # response doesn't have usage_metadata pass return usage def _is_error_retryable(self, error: Exception) -> bool: """Determine if an error should be retried based on structured error codes. Uses Gemini API error structure instead of text pattern matching for reliability. Args: error: Exception from Gemini API call Returns: True if error should be retried, False otherwise """ error_str = str(error).lower() # Check for 429 errors first - these need special handling if "429" in error_str or "quota" in error_str or "resource_exhausted" in error_str: # For Gemini, check for specific non-retryable error indicators # These typically indicate permanent failures or quota/size limits non_retryable_indicators = [ "quota exceeded", "resource exhausted", "context length", "token limit", "request too large", "invalid request", "quota_exceeded", "resource_exhausted", ] # Also check if this is a structured error from Gemini SDK try: # Try to access error details if available error_details = None try: error_details = error.details except AttributeError: try: error_details = error.reason except AttributeError: pass if error_details: error_details_str = str(error_details).lower() # Check for non-retryable error codes/reasons if any(indicator in error_details_str for indicator in non_retryable_indicators): logger.debug(f"Non-retryable Gemini error: {error_details}") return False except Exception: pass # Check main error string for non-retryable patterns if any(indicator in error_str for indicator in non_retryable_indicators): logger.debug(f"Non-retryable Gemini error based on message: {error_str[:200]}...") return False # If it's a 429/quota error but doesn't match non-retryable patterns, it might be retryable rate limiting logger.debug(f"Retryable Gemini rate limiting error: {error_str[:100]}...") return True # For non-429 errors, check if they're retryable retryable_indicators = [ "timeout", "connection", "network", "temporary", "unavailable", "retry", "internal error", "408", # Request timeout "500", # Internal server error "502", # Bad gateway "503", # Service unavailable "504", # Gateway timeout "ssl", # SSL errors "handshake", # Handshake failures ] return any(indicator in error_str for indicator in retryable_indicators) def _process_image(self, image_path: str) -> Optional[dict]: """Process an image for Gemini API.""" try: # Use base class validation image_bytes, mime_type = validate_image(image_path) # For data URLs, extract the base64 data directly if image_path.startswith("data:"): # Extract base64 data from data URL _, data = image_path.split(",", 1) return {"inline_data": {"mime_type": mime_type, "data": data}} else: # For file paths, encode the bytes image_data = base64.b64encode(image_bytes).decode() return {"inline_data": {"mime_type": mime_type, "data": image_data}} except ValueError as e: logger.warning(str(e)) return None except Exception as e: logger.error(f"Error processing image {image_path}: {e}") return None def get_preferred_model(self, category: "ToolModelCategory", allowed_models: list[str]) -> Optional[str]: """Get Gemini's preferred model for a given category from allowed models. Args: category: The tool category requiring a model allowed_models: Pre-filtered list of models allowed by restrictions Returns: Preferred model name or None """ from tools.models import ToolModelCategory if not allowed_models: return None capability_map = self.get_all_model_capabilities() # Helper to find best model from candidates def find_best(candidates: list[str]) -> Optional[str]: """Return best model from candidates (sorted for consistency).""" return sorted(candidates, reverse=True)[0] if candidates else None if category == ToolModelCategory.EXTENDED_REASONING: # For extended reasoning, prefer models with thinking support # First try Pro models that support thinking pro_thinking = [ m for m in allowed_models if "pro" in m and m in capability_map and capability_map[m].supports_extended_thinking ] if pro_thinking: return find_best(pro_thinking) # Then any model that supports thinking any_thinking = [ m for m in allowed_models if m in capability_map and capability_map[m].supports_extended_thinking ] if any_thinking: return find_best(any_thinking) # Finally, just prefer Pro models even without thinking pro_models = [m for m in allowed_models if "pro" in m] if pro_models: return find_best(pro_models) elif category == ToolModelCategory.FAST_RESPONSE: # Prefer Flash models for speed flash_models = [m for m in allowed_models if "flash" in m] if flash_models: return find_best(flash_models) # Default for BALANCED or as fallback # Prefer Flash for balanced use, then Pro, then anything flash_models = [m for m in allowed_models if "flash" in m] if flash_models: return find_best(flash_models) pro_models = [m for m in allowed_models if "pro" in m] if pro_models: return find_best(pro_models) # Ultimate fallback to best available model return find_best(allowed_models) # Load registry data at import time for registry consumers GeminiModelProvider._ensure_registry() ================================================ FILE: providers/openai.py ================================================ """OpenAI model provider implementation.""" import logging from typing import TYPE_CHECKING, ClassVar, Optional if TYPE_CHECKING: from tools.models import ToolModelCategory from .openai_compatible import OpenAICompatibleProvider from .registries.openai import OpenAIModelRegistry from .registry_provider_mixin import RegistryBackedProviderMixin from .shared import ModelCapabilities, ProviderType logger = logging.getLogger(__name__) class OpenAIModelProvider(RegistryBackedProviderMixin, OpenAICompatibleProvider): """Implementation that talks to api.openai.com using rich model metadata. In addition to the built-in catalogue, the provider can surface models defined in ``conf/custom_models.json`` (for organisations running their own OpenAI-compatible gateways) while still respecting restriction policies. """ REGISTRY_CLASS = OpenAIModelRegistry MODEL_CAPABILITIES: ClassVar[dict[str, ModelCapabilities]] = {} def __init__(self, api_key: str, **kwargs): """Initialize OpenAI provider with API key.""" self._ensure_registry() # Set default OpenAI base URL, allow override for regions/custom endpoints kwargs.setdefault("base_url", "https://api.openai.com/v1") super().__init__(api_key, **kwargs) self._invalidate_capability_cache() # ------------------------------------------------------------------ # Capability surface # ------------------------------------------------------------------ def _lookup_capabilities( self, canonical_name: str, requested_name: Optional[str] = None, ) -> Optional[ModelCapabilities]: """Look up OpenAI capabilities from built-ins or the custom registry.""" self._ensure_registry() builtin = super()._lookup_capabilities(canonical_name, requested_name) if builtin is not None: return builtin try: from .registries.openrouter import OpenRouterModelRegistry registry = OpenRouterModelRegistry() config = registry.get_model_config(canonical_name) if config and config.provider == ProviderType.OPENAI: return config except Exception as exc: # pragma: no cover - registry failures are non-critical logger.debug(f"Could not resolve custom OpenAI model '{canonical_name}': {exc}") return None def _finalise_capabilities( self, capabilities: ModelCapabilities, canonical_name: str, requested_name: str, ) -> ModelCapabilities: """Ensure registry-sourced models report the correct provider type.""" if capabilities.provider != ProviderType.OPENAI: capabilities.provider = ProviderType.OPENAI return capabilities def _raise_unsupported_model(self, model_name: str) -> None: raise ValueError(f"Unsupported OpenAI model: {model_name}") # ------------------------------------------------------------------ # Provider identity # ------------------------------------------------------------------ def get_provider_type(self) -> ProviderType: """Get the provider type.""" return ProviderType.OPENAI # ------------------------------------------------------------------ # Provider preferences # ------------------------------------------------------------------ def get_preferred_model(self, category: "ToolModelCategory", allowed_models: list[str]) -> Optional[str]: """Get OpenAI's preferred model for a given category from allowed models. Args: category: The tool category requiring a model allowed_models: Pre-filtered list of models allowed by restrictions Returns: Preferred model name or None """ from tools.models import ToolModelCategory if not allowed_models: return None # Helper to find first available from preference list def find_first(preferences: list[str]) -> Optional[str]: """Return first available model from preference list.""" for model in preferences: if model in allowed_models: return model return None if category == ToolModelCategory.EXTENDED_REASONING: # Prefer models with extended thinking support # GPT-5.1 Codex first for coding tasks preferred = find_first( [ "gpt-5.1-codex", "gpt-5.2", "gpt-5-codex", "gpt-5.2-pro", "o3-pro", "gpt-5", "o3", ] ) return preferred if preferred else allowed_models[0] elif category == ToolModelCategory.FAST_RESPONSE: # Prefer fast, cost-efficient models # GPT-5.2 models for speed, GPT-5.1-Codex after (premium pricing but cached) preferred = find_first( [ "gpt-5.2", "gpt-5.1-codex-mini", "gpt-5", "gpt-5-mini", "gpt-5-codex", "o4-mini", "o3-mini", ] ) return preferred if preferred else allowed_models[0] else: # BALANCED or default # Prefer balanced performance/cost models # Include GPT-5.2 family for latest capabilities preferred = find_first( [ "gpt-5.2", "gpt-5.1-codex", "gpt-5", "gpt-5-codex", "gpt-5.2-pro", "gpt-5-mini", "o4-mini", "o3-mini", ] ) return preferred if preferred else allowed_models[0] # Load registry data at import time so dependent providers (Azure) can reuse it OpenAIModelProvider._ensure_registry() ================================================ FILE: providers/openai_compatible.py ================================================ """Base class for OpenAI-compatible API providers.""" import copy import ipaddress import logging from typing import Optional from urllib.parse import urlparse from openai import OpenAI from utils.env import get_env, suppress_env_vars from utils.image_utils import validate_image from .base import ModelProvider from .shared import ( ModelCapabilities, ModelResponse, ProviderType, ) class OpenAICompatibleProvider(ModelProvider): """Shared implementation for OpenAI API lookalikes. The class owns HTTP client configuration (timeouts, proxy hardening, custom headers) and normalises the OpenAI SDK responses into :class:`~providers.shared.ModelResponse`. Concrete subclasses only need to provide capability metadata and any provider-specific request tweaks. """ DEFAULT_HEADERS = {} FRIENDLY_NAME = "OpenAI Compatible" def __init__(self, api_key: str, base_url: str = None, **kwargs): """Initialize the provider with API key and optional base URL. Args: api_key: API key for authentication base_url: Base URL for the API endpoint **kwargs: Additional configuration options including timeout """ self._allowed_alias_cache: dict[str, str] = {} super().__init__(api_key, **kwargs) self._client = None self.base_url = base_url self.organization = kwargs.get("organization") self.allowed_models = self._parse_allowed_models() # Configure timeouts - especially important for custom/local endpoints self.timeout_config = self._configure_timeouts(**kwargs) # Validate base URL for security if self.base_url: self._validate_base_url() # Warn if using external URL without authentication if self.base_url and not self._is_localhost_url() and not api_key: logging.warning( f"Using external URL '{self.base_url}' without API key. " "This may be insecure. Consider setting an API key for authentication." ) def _ensure_model_allowed( self, capabilities: ModelCapabilities, canonical_name: str, requested_name: str, ) -> None: """Respect provider-specific allowlists before default restriction checks.""" super()._ensure_model_allowed(capabilities, canonical_name, requested_name) if self.allowed_models is not None: requested = requested_name.lower() canonical = canonical_name.lower() if requested not in self.allowed_models and canonical not in self.allowed_models: allowed = False for allowed_entry in list(self.allowed_models): normalized_resolved = self._allowed_alias_cache.get(allowed_entry) if normalized_resolved is None: try: resolved_name = self._resolve_model_name(allowed_entry) except Exception: continue if not resolved_name: continue normalized_resolved = resolved_name.lower() self._allowed_alias_cache[allowed_entry] = normalized_resolved if normalized_resolved == canonical: # Canonical match discovered via alias resolution – mark as allowed and # memoise the canonical entry for future lookups. allowed = True self._allowed_alias_cache[canonical] = canonical self.allowed_models.add(canonical) break if not allowed: raise ValueError( f"Model '{requested_name}' is not allowed by restriction policy. Allowed models: {sorted(self.allowed_models)}" ) def _parse_allowed_models(self) -> Optional[set[str]]: """Parse allowed models from environment variable. Returns: Set of allowed model names (lowercase) or None if not configured """ # Get provider-specific allowed models provider_type = self.get_provider_type().value.upper() env_var = f"{provider_type}_ALLOWED_MODELS" models_str = get_env(env_var, "") or "" if models_str: # Parse and normalize to lowercase for case-insensitive comparison models = {m.strip().lower() for m in models_str.split(",") if m.strip()} if models: logging.info(f"Configured allowed models for {self.FRIENDLY_NAME}: {sorted(models)}") self._allowed_alias_cache = {} return models # Log info if no allow-list configured for proxy providers if self.get_provider_type() not in [ProviderType.GOOGLE, ProviderType.OPENAI]: logging.info( f"Model allow-list not configured for {self.FRIENDLY_NAME} - all models permitted. " f"To restrict access, set {env_var} with comma-separated model names." ) return None def _configure_timeouts(self, **kwargs): """Configure timeout settings based on provider type and custom settings. Custom URLs and local models often need longer timeouts due to: - Network latency on local networks - Extended thinking models taking longer to respond - Local inference being slower than cloud APIs Returns: httpx.Timeout object with appropriate timeout settings """ import httpx # Default timeouts - more generous for custom/local endpoints default_connect = 30.0 # 30 seconds for connection (vs OpenAI's 5s) default_read = 600.0 # 10 minutes for reading (same as OpenAI default) default_write = 600.0 # 10 minutes for writing default_pool = 600.0 # 10 minutes for pool # For custom/local URLs, use even longer timeouts if self.base_url and self._is_localhost_url(): default_connect = 60.0 # 1 minute for local connections default_read = 1800.0 # 30 minutes for local models (extended thinking) default_write = 1800.0 # 30 minutes for local models default_pool = 1800.0 # 30 minutes for local models logging.info(f"Using extended timeouts for local endpoint: {self.base_url}") elif self.base_url: default_connect = 45.0 # 45 seconds for custom remote endpoints default_read = 900.0 # 15 minutes for custom remote endpoints default_write = 900.0 # 15 minutes for custom remote endpoints default_pool = 900.0 # 15 minutes for custom remote endpoints logging.info(f"Using extended timeouts for custom endpoint: {self.base_url}") # Allow override via kwargs or environment variables in future, for now... connect_timeout = kwargs.get("connect_timeout") if connect_timeout is None: connect_timeout_raw = get_env("CUSTOM_CONNECT_TIMEOUT") connect_timeout = float(connect_timeout_raw) if connect_timeout_raw is not None else float(default_connect) read_timeout = kwargs.get("read_timeout") if read_timeout is None: read_timeout_raw = get_env("CUSTOM_READ_TIMEOUT") read_timeout = float(read_timeout_raw) if read_timeout_raw is not None else float(default_read) write_timeout = kwargs.get("write_timeout") if write_timeout is None: write_timeout_raw = get_env("CUSTOM_WRITE_TIMEOUT") write_timeout = float(write_timeout_raw) if write_timeout_raw is not None else float(default_write) pool_timeout = kwargs.get("pool_timeout") if pool_timeout is None: pool_timeout_raw = get_env("CUSTOM_POOL_TIMEOUT") pool_timeout = float(pool_timeout_raw) if pool_timeout_raw is not None else float(default_pool) timeout = httpx.Timeout(connect=connect_timeout, read=read_timeout, write=write_timeout, pool=pool_timeout) logging.debug( f"Configured timeouts - Connect: {connect_timeout}s, Read: {read_timeout}s, " f"Write: {write_timeout}s, Pool: {pool_timeout}s" ) return timeout def _is_localhost_url(self) -> bool: """Check if the base URL points to localhost or local network. Returns: True if URL is localhost or local network, False otherwise """ if not self.base_url: return False try: parsed = urlparse(self.base_url) hostname = parsed.hostname # Check for common localhost patterns if hostname in ["localhost", "127.0.0.1", "::1"]: return True # Check for private network ranges (local network) if hostname: try: ip = ipaddress.ip_address(hostname) return ip.is_private or ip.is_loopback except ValueError: # Not an IP address, might be a hostname pass return False except Exception: return False def _validate_base_url(self) -> None: """Validate base URL for security (SSRF protection). Raises: ValueError: If URL is invalid or potentially unsafe """ if not self.base_url: return try: parsed = urlparse(self.base_url) # Check URL scheme - only allow http/https if parsed.scheme not in ("http", "https"): raise ValueError(f"Invalid URL scheme: {parsed.scheme}. Only http/https allowed.") # Check hostname exists if not parsed.hostname: raise ValueError("URL must include a hostname") # Check port is valid (if specified) port = parsed.port if port is not None and (port < 1 or port > 65535): raise ValueError(f"Invalid port number: {port}. Must be between 1 and 65535.") except Exception as e: if isinstance(e, ValueError): raise raise ValueError(f"Invalid base URL '{self.base_url}': {str(e)}") @property def client(self): """Lazy initialization of OpenAI client with security checks and timeout configuration.""" if self._client is None: import httpx proxy_env_vars = ["HTTP_PROXY", "HTTPS_PROXY", "ALL_PROXY", "http_proxy", "https_proxy", "all_proxy"] with suppress_env_vars(*proxy_env_vars): try: # Create a custom httpx client that explicitly avoids proxy parameters timeout_config = ( self.timeout_config if hasattr(self, "timeout_config") and self.timeout_config else httpx.Timeout(30.0) ) # Create httpx client with minimal config to avoid proxy conflicts # Note: proxies parameter was removed in httpx 0.28.0 # Check for test transport injection if hasattr(self, "_test_transport"): # Use custom transport for testing (HTTP recording/replay) http_client = httpx.Client( transport=self._test_transport, timeout=timeout_config, follow_redirects=True, ) else: # Normal production client http_client = httpx.Client( timeout=timeout_config, follow_redirects=True, ) # Keep client initialization minimal to avoid proxy parameter conflicts client_kwargs = { "api_key": self.api_key, "http_client": http_client, } if self.base_url: client_kwargs["base_url"] = self.base_url if self.organization: client_kwargs["organization"] = self.organization # Add default headers if any if self.DEFAULT_HEADERS: client_kwargs["default_headers"] = self.DEFAULT_HEADERS.copy() logging.debug( "OpenAI client initialized with custom httpx client and timeout: %s", timeout_config, ) # Create OpenAI client with custom httpx client self._client = OpenAI(**client_kwargs) except Exception as e: # If all else fails, try absolute minimal client without custom httpx logging.warning( "Failed to create client with custom httpx, falling back to minimal config: %s", e, ) try: minimal_kwargs = {"api_key": self.api_key} if self.base_url: minimal_kwargs["base_url"] = self.base_url self._client = OpenAI(**minimal_kwargs) except Exception as fallback_error: logging.error("Even minimal OpenAI client creation failed: %s", fallback_error) raise return self._client def _sanitize_for_logging(self, params: dict) -> dict: """Sanitize sensitive data from parameters before logging. Args: params: Dictionary of API parameters Returns: dict: Sanitized copy of parameters safe for logging """ sanitized = copy.deepcopy(params) # Sanitize messages content if "input" in sanitized: for msg in sanitized.get("input", []): if isinstance(msg, dict) and "content" in msg: for content_item in msg.get("content", []): if isinstance(content_item, dict) and "text" in content_item: # Truncate long text and add ellipsis text = content_item["text"] if len(text) > 100: content_item["text"] = text[:100] + "... [truncated]" # Remove any API keys that might be in headers/auth sanitized.pop("api_key", None) sanitized.pop("authorization", None) return sanitized def _safe_extract_output_text(self, response) -> str: """Safely extract output_text from o3-pro response with validation. Args: response: Response object from OpenAI SDK Returns: str: The output text content Raises: ValueError: If output_text is missing, None, or not a string """ logging.debug(f"Response object type: {type(response)}") logging.debug(f"Response attributes: {dir(response)}") if not hasattr(response, "output_text"): raise ValueError(f"o3-pro response missing output_text field. Response type: {type(response).__name__}") content = response.output_text logging.debug(f"Extracted output_text: '{content}' (type: {type(content)})") if content is None: raise ValueError("o3-pro returned None for output_text") if not isinstance(content, str): raise ValueError(f"o3-pro output_text is not a string. Got type: {type(content).__name__}") return content def _generate_with_responses_endpoint( self, model_name: str, messages: list, temperature: float, max_output_tokens: Optional[int] = None, capabilities: Optional[ModelCapabilities] = None, **kwargs, ) -> ModelResponse: """Generate content using the /v1/responses endpoint for reasoning models.""" # Convert messages to the correct format for responses endpoint input_messages = [] for message in messages: role = message.get("role", "") content = message.get("content", "") if role == "system": # For o3-pro, system messages should be handled carefully to avoid policy violations # Instead of prefixing with "System:", we'll include the system content naturally input_messages.append({"role": "user", "content": [{"type": "input_text", "text": content}]}) elif role == "user": input_messages.append({"role": "user", "content": [{"type": "input_text", "text": content}]}) elif role == "assistant": input_messages.append({"role": "assistant", "content": [{"type": "output_text", "text": content}]}) # Prepare completion parameters for responses endpoint # Based on OpenAI documentation, use nested reasoning object for responses endpoint effort = "medium" if capabilities and capabilities.default_reasoning_effort: effort = capabilities.default_reasoning_effort completion_params = { "model": model_name, "input": input_messages, "reasoning": {"effort": effort}, } # Only include store parameter for providers that support it. # OpenRouter's /responses endpoint rejects store:true via Zod validation (Issue #348). # This is an endpoint-level limitation, not model-specific, so we omit for all # OpenRouter /responses calls. If OpenRouter later supports store, revisit this logic. if self.get_provider_type() != ProviderType.OPENROUTER: completion_params["store"] = True else: logging.debug(f"Omitting 'store' parameter for OpenRouter provider (model: {model_name})") # Add max tokens if specified (using max_completion_tokens for responses endpoint) if max_output_tokens: completion_params["max_completion_tokens"] = max_output_tokens # For responses endpoint, we only add parameters that are explicitly supported # Remove unsupported chat completion parameters that may cause API errors # Retry logic with progressive delays max_retries = 4 retry_delays = [1, 3, 5, 8] attempt_counter = {"value": 0} def _attempt() -> ModelResponse: attempt_counter["value"] += 1 import json sanitized_params = self._sanitize_for_logging(completion_params) logging.info( f"o3-pro API request (sanitized): {json.dumps(sanitized_params, indent=2, ensure_ascii=False)}" ) response = self.client.responses.create(**completion_params) content = self._safe_extract_output_text(response) usage = None if hasattr(response, "usage"): usage = self._extract_usage(response) elif hasattr(response, "input_tokens") and hasattr(response, "output_tokens"): input_tokens = getattr(response, "input_tokens", 0) or 0 output_tokens = getattr(response, "output_tokens", 0) or 0 usage = { "input_tokens": input_tokens, "output_tokens": output_tokens, "total_tokens": input_tokens + output_tokens, } return ModelResponse( content=content, usage=usage, model_name=model_name, friendly_name=self.FRIENDLY_NAME, provider=self.get_provider_type(), metadata={ "model": getattr(response, "model", model_name), "id": getattr(response, "id", ""), "created": getattr(response, "created_at", 0), "endpoint": "responses", }, ) try: return self._run_with_retries( operation=_attempt, max_attempts=max_retries, delays=retry_delays, log_prefix="responses endpoint", ) except Exception as exc: attempts = max(attempt_counter["value"], 1) error_msg = f"responses endpoint error after {attempts} attempt{'s' if attempts > 1 else ''}: {exc}" logging.error(error_msg) raise RuntimeError(error_msg) from exc def generate_content( self, prompt: str, model_name: str, system_prompt: Optional[str] = None, temperature: float = 0.3, max_output_tokens: Optional[int] = None, images: Optional[list[str]] = None, **kwargs, ) -> ModelResponse: """Generate content using the OpenAI-compatible API. Args: prompt: User prompt to send to the model model_name: Canonical model name or its alias system_prompt: Optional system prompt for model behavior temperature: Sampling temperature max_output_tokens: Maximum tokens to generate images: Optional list of image paths or data URLs to include with the prompt (for vision models) **kwargs: Additional provider-specific parameters Returns: ModelResponse with generated content and metadata """ # Validate model name against allow-list if not self.validate_model_name(model_name): raise ValueError(f"Model '{model_name}' not in allowed models list. Allowed models: {self.allowed_models}") capabilities: Optional[ModelCapabilities] try: capabilities = self.get_capabilities(model_name) except Exception as exc: logging.debug(f"Falling back to generic capabilities for {model_name}: {exc}") capabilities = None # Get effective temperature for this model from capabilities when available if capabilities: effective_temperature = capabilities.get_effective_temperature(temperature) if effective_temperature is not None and effective_temperature != temperature: logging.debug( f"Adjusting temperature from {temperature} to {effective_temperature} for model {model_name}" ) else: effective_temperature = temperature # Only validate if temperature is not None (meaning the model supports it) if effective_temperature is not None: # Validate parameters with the effective temperature self.validate_parameters(model_name, effective_temperature) # Resolve to canonical model name resolved_model = self._resolve_model_name(model_name) # Prepare messages messages = [] if system_prompt: messages.append({"role": "system", "content": system_prompt}) # Prepare user message with text and potentially images user_content = [] user_content.append({"type": "text", "text": prompt}) # Add images if provided and model supports vision if images and capabilities and capabilities.supports_images: for image_path in images: try: image_content = self._process_image(image_path) if image_content: user_content.append(image_content) except Exception as e: logging.warning(f"Failed to process image {image_path}: {e}") # Continue with other images and text continue elif images and (not capabilities or not capabilities.supports_images): logging.warning(f"Model {resolved_model} does not support images, ignoring {len(images)} image(s)") # Add user message if len(user_content) == 1: # Only text content, use simple string format for compatibility messages.append({"role": "user", "content": prompt}) else: # Text + images, use content array format messages.append({"role": "user", "content": user_content}) # Prepare completion parameters # Always disable streaming for OpenRouter # MCP doesn't use streaming, and this avoids issues with O3 model access completion_params = { "model": resolved_model, "messages": messages, "stream": False, } # Use the effective temperature we calculated earlier supports_sampling = effective_temperature is not None if supports_sampling: completion_params["temperature"] = effective_temperature # Add max tokens if specified and model supports it # O3/O4 models that don't support temperature also don't support max_tokens if max_output_tokens and supports_sampling: completion_params["max_tokens"] = max_output_tokens # Add any additional OpenAI-specific parameters # Use capabilities to filter parameters for reasoning models for key, value in kwargs.items(): if key in ["top_p", "frequency_penalty", "presence_penalty", "seed", "stop", "stream"]: # Reasoning models (those that don't support temperature) also don't support these parameters if not supports_sampling and key in ["top_p", "frequency_penalty", "presence_penalty", "stream"]: continue # Skip unsupported parameters for reasoning models completion_params[key] = value # Check if this model needs the Responses API endpoint # Prefer capability metadata; fall back to static map when capabilities unavailable use_responses_api = False if capabilities is not None: use_responses_api = getattr(capabilities, "use_openai_response_api", False) else: static_capabilities = self.get_all_model_capabilities().get(resolved_model) if static_capabilities is not None: use_responses_api = getattr(static_capabilities, "use_openai_response_api", False) if use_responses_api: # These models require the /v1/responses endpoint for stateful context # If it fails, we should not fall back to chat/completions return self._generate_with_responses_endpoint( model_name=resolved_model, messages=messages, temperature=temperature, max_output_tokens=max_output_tokens, capabilities=capabilities, **kwargs, ) # Retry logic with progressive delays max_retries = 4 # Total of 4 attempts retry_delays = [1, 3, 5, 8] # Progressive delays: 1s, 3s, 5s, 8s attempt_counter = {"value": 0} def _attempt() -> ModelResponse: attempt_counter["value"] += 1 response = self.client.chat.completions.create(**completion_params) content = response.choices[0].message.content usage = self._extract_usage(response) return ModelResponse( content=content, usage=usage, model_name=resolved_model, friendly_name=self.FRIENDLY_NAME, provider=self.get_provider_type(), metadata={ "finish_reason": response.choices[0].finish_reason, "model": response.model, "id": response.id, "created": response.created, }, ) try: return self._run_with_retries( operation=_attempt, max_attempts=max_retries, delays=retry_delays, log_prefix=f"{self.FRIENDLY_NAME} API ({resolved_model})", ) except Exception as exc: attempts = max(attempt_counter["value"], 1) error_msg = ( f"{self.FRIENDLY_NAME} API error for model {resolved_model} after {attempts} attempt" f"{'s' if attempts > 1 else ''}: {exc}" ) logging.error(error_msg) raise RuntimeError(error_msg) from exc def validate_parameters(self, model_name: str, temperature: float, **kwargs) -> None: """Validate model parameters. For proxy providers, this may use generic capabilities. Args: model_name: Canonical model name or its alias temperature: Temperature to validate **kwargs: Additional parameters to validate """ try: capabilities = self.get_capabilities(model_name) # Check if we're using generic capabilities if hasattr(capabilities, "_is_generic"): logging.debug( f"Using generic parameter validation for {model_name}. Actual model constraints may differ." ) # Validate temperature using parent class method super().validate_parameters(model_name, temperature, **kwargs) except Exception as e: # For proxy providers, we might not have accurate capabilities # Log warning but don't fail logging.warning(f"Parameter validation limited for {model_name}: {e}") def _extract_usage(self, response) -> dict[str, int]: """Extract token usage from OpenAI response. Args: response: OpenAI API response object Returns: Dictionary with usage statistics """ usage = {} if hasattr(response, "usage") and response.usage: # Safely extract token counts with None handling usage["input_tokens"] = getattr(response.usage, "prompt_tokens", 0) or 0 usage["output_tokens"] = getattr(response.usage, "completion_tokens", 0) or 0 usage["total_tokens"] = getattr(response.usage, "total_tokens", 0) or 0 return usage def count_tokens(self, text: str, model_name: str) -> int: """Count tokens using OpenAI-compatible tokenizer tables when available.""" resolved_model = self._resolve_model_name(model_name) try: import tiktoken try: encoding = tiktoken.encoding_for_model(resolved_model) except KeyError: encoding = tiktoken.get_encoding("cl100k_base") return len(encoding.encode(text)) except (ImportError, Exception) as exc: logging.debug("tiktoken unavailable for %s: %s", resolved_model, exc) return super().count_tokens(text, model_name) def _is_error_retryable(self, error: Exception) -> bool: """Determine if an error should be retried based on structured error codes. Uses OpenAI API error structure instead of text pattern matching for reliability. Args: error: Exception from OpenAI API call Returns: True if error should be retried, False otherwise """ error_str = str(error).lower() # Check for 429 errors first - these need special handling if "429" in error_str: # Try to extract structured error information error_type = None error_code = None # Parse structured error from OpenAI API response # Format: "Error code: 429 - {'error': {'type': 'tokens', 'code': 'rate_limit_exceeded', ...}}" try: import ast import json import re # Extract JSON part from error string using regex # Look for pattern: {...} (from first { to last }) json_match = re.search(r"\{.*\}", str(error)) if json_match: json_like_str = json_match.group(0) # First try: parse as Python literal (handles single quotes safely) try: error_data = ast.literal_eval(json_like_str) except (ValueError, SyntaxError): # Fallback: try JSON parsing with simple quote replacement # (for cases where it's already valid JSON or simple replacements work) json_str = json_like_str.replace("'", '"') error_data = json.loads(json_str) if "error" in error_data: error_info = error_data["error"] error_type = error_info.get("type") error_code = error_info.get("code") except (json.JSONDecodeError, ValueError, SyntaxError, AttributeError): # Fall back to checking hasattr for OpenAI SDK exception objects if hasattr(error, "response") and hasattr(error.response, "json"): try: response_data = error.response.json() if "error" in response_data: error_info = response_data["error"] error_type = error_info.get("type") error_code = error_info.get("code") except Exception: pass # Determine if 429 is retryable based on structured error codes if error_type == "tokens": # Token-related 429s are typically non-retryable (request too large) logging.debug(f"Non-retryable 429: token-related error (type={error_type}, code={error_code})") return False elif error_code in ["invalid_request_error", "context_length_exceeded"]: # These are permanent failures logging.debug(f"Non-retryable 429: permanent failure (type={error_type}, code={error_code})") return False else: # Other 429s (like requests per minute) are retryable logging.debug(f"Retryable 429: rate limiting (type={error_type}, code={error_code})") return True # For non-429 errors, check if they're retryable retryable_indicators = [ "timeout", "connection", "network", "temporary", "unavailable", "retry", "408", # Request timeout "500", # Internal server error "502", # Bad gateway "503", # Service unavailable "504", # Gateway timeout "ssl", # SSL errors "handshake", # Handshake failures ] return any(indicator in error_str for indicator in retryable_indicators) def _process_image(self, image_path: str) -> Optional[dict]: """Process an image for OpenAI-compatible API.""" try: if image_path.startswith("data:"): # Validate the data URL validate_image(image_path) # Handle data URL: data:image/png;base64,iVBORw0... return {"type": "image_url", "image_url": {"url": image_path}} else: # Use base class validation image_bytes, mime_type = validate_image(image_path) # Read and encode the image import base64 image_data = base64.b64encode(image_bytes).decode() logging.debug(f"Processing image '{image_path}' as MIME type '{mime_type}'") # Create data URL for OpenAI API data_url = f"data:{mime_type};base64,{image_data}" return {"type": "image_url", "image_url": {"url": data_url}} except ValueError as e: logging.warning(str(e)) return None except Exception as e: logging.error(f"Error processing image {image_path}: {e}") return None ================================================ FILE: providers/openrouter.py ================================================ """OpenRouter provider implementation.""" import logging from utils.env import get_env from .openai_compatible import OpenAICompatibleProvider from .registries.openrouter import OpenRouterModelRegistry from .shared import ( ModelCapabilities, ProviderType, RangeTemperatureConstraint, ) class OpenRouterProvider(OpenAICompatibleProvider): """Client for OpenRouter's multi-model aggregation service. Role Surface OpenRouter’s dynamic catalogue through the same interface as native providers so tools can reference OpenRouter models and aliases without special cases. Characteristics * Pulls live model definitions from :class:`OpenRouterModelRegistry` (aliases, provider-specific metadata, capability hints) * Applies alias-aware restriction checks before exposing models to the registry or tooling * Reuses :class:`OpenAICompatibleProvider` infrastructure for request execution so OpenRouter endpoints behave like standard OpenAI-style APIs. """ FRIENDLY_NAME = "OpenRouter" # Custom headers required by OpenRouter DEFAULT_HEADERS = { "HTTP-Referer": get_env("OPENROUTER_REFERER", "https://github.com/BeehiveInnovations/pal-mcp-server") or "https://github.com/BeehiveInnovations/pal-mcp-server", "X-Title": get_env("OPENROUTER_TITLE", "PAL MCP Server") or "PAL MCP Server", } # Model registry for managing configurations and aliases _registry: OpenRouterModelRegistry | None = None def __init__(self, api_key: str, **kwargs): """Initialize OpenRouter provider. Args: api_key: OpenRouter API key **kwargs: Additional configuration """ base_url = "https://openrouter.ai/api/v1" self._alias_cache: dict[str, str] = {} super().__init__(api_key, base_url=base_url, **kwargs) # Initialize model registry if OpenRouterProvider._registry is None: OpenRouterProvider._registry = OpenRouterModelRegistry() # Log loaded models and aliases only on first load models = self._registry.list_models() aliases = self._registry.list_aliases() logging.info(f"OpenRouter loaded {len(models)} models with {len(aliases)} aliases") # ------------------------------------------------------------------ # Capability surface # ------------------------------------------------------------------ def _lookup_capabilities( self, canonical_name: str, requested_name: str | None = None, ) -> ModelCapabilities | None: """Fetch OpenRouter capabilities from the registry or build a generic fallback.""" capabilities = self._registry.get_capabilities(canonical_name) if capabilities: return capabilities base_identifier = canonical_name.split(":", 1)[0] if "/" in base_identifier: logging.debug( "Using generic OpenRouter capabilities for %s (provider/model format detected)", canonical_name ) generic = ModelCapabilities( provider=ProviderType.OPENROUTER, model_name=canonical_name, friendly_name=self.FRIENDLY_NAME, intelligence_score=9, context_window=32_768, max_output_tokens=32_768, supports_extended_thinking=False, supports_system_prompts=True, supports_streaming=True, supports_function_calling=False, temperature_constraint=RangeTemperatureConstraint(0.0, 2.0, 1.0), ) generic._is_generic = True return generic logging.debug( "Rejecting unknown OpenRouter model '%s' (no provider prefix); requires explicit configuration", canonical_name, ) return None # ------------------------------------------------------------------ # Provider identity # ------------------------------------------------------------------ def get_provider_type(self) -> ProviderType: """Identify this provider for restrictions and logging.""" return ProviderType.OPENROUTER # ------------------------------------------------------------------ # Registry helpers # ------------------------------------------------------------------ def list_models( self, *, respect_restrictions: bool = True, include_aliases: bool = True, lowercase: bool = False, unique: bool = False, ) -> list[str]: """Return formatted OpenRouter model names, respecting alias-aware restrictions.""" if not self._registry: return [] from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if respect_restrictions else None allowed_configs: dict[str, ModelCapabilities] = {} for model_name in self._registry.list_models(): config = self._registry.resolve(model_name) if not config: continue # Custom models belong to CustomProvider; skip them here so the two # providers don't race over the same registrations (important for tests # that stub the registry with minimal objects lacking attrs). if config.provider == ProviderType.CUSTOM: continue if restriction_service: allowed = restriction_service.is_allowed(self.get_provider_type(), model_name) if not allowed and config.aliases: for alias in config.aliases: if restriction_service.is_allowed(self.get_provider_type(), alias): allowed = True break if not allowed: continue allowed_configs[model_name] = config if not allowed_configs: return [] # When restrictions are in place, don't include aliases to avoid confusion # Only return the canonical model names that are actually allowed actual_include_aliases = include_aliases and not respect_restrictions return ModelCapabilities.collect_model_names( allowed_configs, include_aliases=actual_include_aliases, lowercase=lowercase, unique=unique, ) # ------------------------------------------------------------------ # Registry helpers # ------------------------------------------------------------------ def _resolve_model_name(self, model_name: str) -> str: """Resolve aliases defined in the OpenRouter registry.""" cache_key = model_name.lower() if cache_key in self._alias_cache: return self._alias_cache[cache_key] config = self._registry.resolve(model_name) if config: if config.model_name != model_name: logging.debug("Resolved model alias '%s' to '%s'", model_name, config.model_name) resolved = config.model_name self._alias_cache[cache_key] = resolved self._alias_cache.setdefault(resolved.lower(), resolved) return resolved logging.debug(f"Model '{model_name}' not found in registry, using as-is") self._alias_cache[cache_key] = model_name return model_name def get_all_model_capabilities(self) -> dict[str, ModelCapabilities]: """Expose registry-backed OpenRouter capabilities.""" if not self._registry: return {} capabilities: dict[str, ModelCapabilities] = {} for model_name in self._registry.list_models(): config = self._registry.resolve(model_name) if not config: continue # See note in list_models: respect the CustomProvider boundary. if config.provider == ProviderType.CUSTOM: continue capabilities[model_name] = config return capabilities ================================================ FILE: providers/registries/__init__.py ================================================ """Registry implementations for provider capability manifests.""" from .azure import AzureModelRegistry from .custom import CustomEndpointModelRegistry from .dial import DialModelRegistry from .gemini import GeminiModelRegistry from .openai import OpenAIModelRegistry from .openrouter import OpenRouterModelRegistry from .xai import XAIModelRegistry __all__ = [ "AzureModelRegistry", "CustomEndpointModelRegistry", "DialModelRegistry", "GeminiModelRegistry", "OpenAIModelRegistry", "OpenRouterModelRegistry", "XAIModelRegistry", ] ================================================ FILE: providers/registries/azure.py ================================================ """Registry loader for Azure OpenAI model configurations.""" from __future__ import annotations import logging from ..shared import ModelCapabilities, ProviderType, TemperatureConstraint from .base import CAPABILITY_FIELD_NAMES, CustomModelRegistryBase logger = logging.getLogger(__name__) class AzureModelRegistry(CustomModelRegistryBase): """Load Azure-specific model metadata from configuration files.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="AZURE_MODELS_CONFIG_PATH", default_filename="azure_models.json", config_path=config_path, ) self.reload() def _extra_keys(self) -> set[str]: return {"deployment", "deployment_name"} def _provider_default(self) -> ProviderType: return ProviderType.AZURE def _default_friendly_name(self, model_name: str) -> str: return f"Azure OpenAI ({model_name})" def _finalise_entry(self, entry: dict) -> tuple[ModelCapabilities, dict]: deployment = entry.pop("deployment", None) or entry.pop("deployment_name", None) if not deployment: raise ValueError(f"Azure model '{entry.get('model_name')}' is missing required 'deployment' field") temp_hint = entry.get("temperature_constraint") if isinstance(temp_hint, str): entry["temperature_constraint"] = TemperatureConstraint.create(temp_hint) filtered = {k: v for k, v in entry.items() if k in CAPABILITY_FIELD_NAMES} filtered.setdefault("provider", ProviderType.AZURE) capability = ModelCapabilities(**filtered) return capability, {"deployment": deployment} ================================================ FILE: providers/registries/base.py ================================================ """Shared infrastructure for JSON-backed model registries.""" from __future__ import annotations import importlib.resources import json import logging from collections.abc import Iterable from dataclasses import fields from pathlib import Path from utils.env import get_env from utils.file_utils import read_json_file from ..shared import ModelCapabilities, ProviderType, TemperatureConstraint logger = logging.getLogger(__name__) CAPABILITY_FIELD_NAMES = {field.name for field in fields(ModelCapabilities)} class CustomModelRegistryBase: """Load and expose capability metadata from a JSON manifest.""" def __init__( self, *, env_var_name: str, default_filename: str, config_path: str | None = None, ) -> None: self._env_var_name = env_var_name self._default_filename = default_filename self._use_resources = False self._resource_package = "conf" self._default_path = Path(__file__).resolve().parents[3] / "conf" / default_filename if config_path: self.config_path = Path(config_path) else: env_path = get_env(env_var_name) if env_path: self.config_path = Path(env_path) else: try: resource = importlib.resources.files(self._resource_package).joinpath(default_filename) if hasattr(resource, "read_text"): self._use_resources = True self.config_path = None else: raise AttributeError("resource accessor not available") except Exception: self.config_path = Path(__file__).resolve().parents[3] / "conf" / default_filename self.alias_map: dict[str, str] = {} self.model_map: dict[str, ModelCapabilities] = {} self._extras: dict[str, dict] = {} def reload(self) -> None: data = self._load_config_data() configs = [config for config in self._parse_models(data) if config is not None] self._build_maps(configs) def list_models(self) -> list[str]: return list(self.model_map.keys()) def list_aliases(self) -> list[str]: return list(self.alias_map.keys()) def resolve(self, name_or_alias: str) -> ModelCapabilities | None: key = name_or_alias.lower() canonical = self.alias_map.get(key) if canonical: return self.model_map.get(canonical) for model_name in self.model_map: if model_name.lower() == key: return self.model_map[model_name] return None def get_capabilities(self, name_or_alias: str) -> ModelCapabilities | None: return self.resolve(name_or_alias) def get_entry(self, model_name: str) -> dict | None: return self._extras.get(model_name) def get_model_config(self, model_name: str) -> ModelCapabilities | None: """Backwards-compatible accessor for registries expecting this helper.""" return self.model_map.get(model_name) or self.resolve(model_name) def iter_entries(self) -> Iterable[tuple[str, ModelCapabilities, dict]]: for model_name, capability in self.model_map.items(): yield model_name, capability, self._extras.get(model_name, {}) # ------------------------------------------------------------------ # Internal helpers # ------------------------------------------------------------------ def _load_config_data(self) -> dict: if self._use_resources: try: resource = importlib.resources.files(self._resource_package).joinpath(self._default_filename) if hasattr(resource, "read_text"): config_text = resource.read_text(encoding="utf-8") else: # pragma: no cover - legacy Python fallback with resource.open("r", encoding="utf-8") as handle: config_text = handle.read() data = json.loads(config_text) except FileNotFoundError: logger.debug("Packaged %s not found", self._default_filename) return {"models": []} except Exception as exc: logger.warning("Failed to read packaged %s: %s", self._default_filename, exc) return {"models": []} return data or {"models": []} if not self.config_path: raise FileNotFoundError("Registry configuration path is not set") if not self.config_path.exists(): logger.debug("Model registry config not found at %s", self.config_path) if self.config_path == self._default_path: fallback = Path.cwd() / "conf" / self._default_filename if fallback != self.config_path and fallback.exists(): logger.debug("Falling back to %s", fallback) self.config_path = fallback else: return {"models": []} else: return {"models": []} data = read_json_file(str(self.config_path)) return data or {"models": []} @property def use_resources(self) -> bool: return self._use_resources def _parse_models(self, data: dict) -> Iterable[ModelCapabilities | None]: for raw in data.get("models", []): if not isinstance(raw, dict): continue yield self._convert_entry(raw) def _convert_entry(self, raw: dict) -> ModelCapabilities | None: entry = dict(raw) model_name = entry.get("model_name") if not model_name: return None aliases = entry.get("aliases") if isinstance(aliases, str): entry["aliases"] = [alias.strip() for alias in aliases.split(",") if alias.strip()] entry.setdefault("friendly_name", self._default_friendly_name(model_name)) temperature_hint = entry.get("temperature_constraint") if isinstance(temperature_hint, str): entry["temperature_constraint"] = TemperatureConstraint.create(temperature_hint) elif temperature_hint is None: entry["temperature_constraint"] = TemperatureConstraint.create("range") if "max_tokens" in entry: raise ValueError( "`max_tokens` is no longer supported. Use `max_output_tokens` in your model configuration." ) unknown_keys = set(entry.keys()) - CAPABILITY_FIELD_NAMES - self._extra_keys() if unknown_keys: raise ValueError("Unsupported fields in model configuration: " + ", ".join(sorted(unknown_keys))) capability, extras = self._finalise_entry(entry) capability.provider = self._provider_default() self._extras[capability.model_name] = extras or {} return capability def _default_friendly_name(self, model_name: str) -> str: return model_name def _extra_keys(self) -> set[str]: return set() def _provider_default(self) -> ProviderType: return ProviderType.OPENROUTER def _finalise_entry(self, entry: dict) -> tuple[ModelCapabilities, dict]: return ModelCapabilities(**{k: v for k, v in entry.items() if k in CAPABILITY_FIELD_NAMES}), {} def _build_maps(self, configs: Iterable[ModelCapabilities]) -> None: alias_map: dict[str, str] = {} model_map: dict[str, ModelCapabilities] = {} for config in configs: if not config: continue model_map[config.model_name] = config model_name_lower = config.model_name.lower() if model_name_lower not in alias_map: alias_map[model_name_lower] = config.model_name for alias in config.aliases: alias_lower = alias.lower() if alias_lower in alias_map and alias_map[alias_lower] != config.model_name: raise ValueError( f"Duplicate alias '{alias}' found for models '{alias_map[alias_lower]}' and '{config.model_name}'" ) alias_map[alias_lower] = config.model_name self.alias_map = alias_map self.model_map = model_map class CapabilityModelRegistry(CustomModelRegistryBase): """Registry that returns :class:`ModelCapabilities` objects with alias support.""" def __init__( self, *, env_var_name: str, default_filename: str, provider: ProviderType, friendly_prefix: str, config_path: str | None = None, ) -> None: self._provider = provider self._friendly_prefix = friendly_prefix super().__init__( env_var_name=env_var_name, default_filename=default_filename, config_path=config_path, ) self.reload() def _provider_default(self) -> ProviderType: return self._provider def _default_friendly_name(self, model_name: str) -> str: return self._friendly_prefix.format(model=model_name) def _finalise_entry(self, entry: dict) -> tuple[ModelCapabilities, dict]: filtered = {k: v for k, v in entry.items() if k in CAPABILITY_FIELD_NAMES} filtered.setdefault("provider", self._provider_default()) capability = ModelCapabilities(**filtered) return capability, {} ================================================ FILE: providers/registries/custom.py ================================================ """Registry loader for custom OpenAI-compatible endpoints.""" from __future__ import annotations from ..shared import ModelCapabilities, ProviderType from .base import CAPABILITY_FIELD_NAMES, CapabilityModelRegistry class CustomEndpointModelRegistry(CapabilityModelRegistry): """Capability registry backed by ``conf/custom_models.json``.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="CUSTOM_MODELS_CONFIG_PATH", default_filename="custom_models.json", provider=ProviderType.CUSTOM, friendly_prefix="Custom ({model})", config_path=config_path, ) def _finalise_entry(self, entry: dict) -> tuple[ModelCapabilities, dict]: filtered = {k: v for k, v in entry.items() if k in CAPABILITY_FIELD_NAMES} filtered.setdefault("provider", ProviderType.CUSTOM) capability = ModelCapabilities(**filtered) return capability, {} ================================================ FILE: providers/registries/dial.py ================================================ """Registry loader for DIAL provider capabilities.""" from __future__ import annotations from ..shared import ProviderType from .base import CapabilityModelRegistry class DialModelRegistry(CapabilityModelRegistry): """Capability registry backed by ``conf/dial_models.json``.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="DIAL_MODELS_CONFIG_PATH", default_filename="dial_models.json", provider=ProviderType.DIAL, friendly_prefix="DIAL ({model})", config_path=config_path, ) ================================================ FILE: providers/registries/gemini.py ================================================ """Registry loader for Gemini model capabilities.""" from __future__ import annotations from ..shared import ProviderType from .base import CapabilityModelRegistry class GeminiModelRegistry(CapabilityModelRegistry): """Capability registry backed by ``conf/gemini_models.json``.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="GEMINI_MODELS_CONFIG_PATH", default_filename="gemini_models.json", provider=ProviderType.GOOGLE, friendly_prefix="Gemini ({model})", config_path=config_path, ) ================================================ FILE: providers/registries/openai.py ================================================ """Registry loader for OpenAI model capabilities.""" from __future__ import annotations from ..shared import ProviderType from .base import CapabilityModelRegistry class OpenAIModelRegistry(CapabilityModelRegistry): """Capability registry backed by ``conf/openai_models.json``.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="OPENAI_MODELS_CONFIG_PATH", default_filename="openai_models.json", provider=ProviderType.OPENAI, friendly_prefix="OpenAI ({model})", config_path=config_path, ) ================================================ FILE: providers/registries/openrouter.py ================================================ """OpenRouter model registry for managing model configurations and aliases.""" from __future__ import annotations from ..shared import ModelCapabilities, ProviderType from .base import CAPABILITY_FIELD_NAMES, CapabilityModelRegistry class OpenRouterModelRegistry(CapabilityModelRegistry): """Capability registry backed by ``conf/openrouter_models.json``.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="OPENROUTER_MODELS_CONFIG_PATH", default_filename="openrouter_models.json", provider=ProviderType.OPENROUTER, friendly_prefix="OpenRouter ({model})", config_path=config_path, ) def _finalise_entry(self, entry: dict) -> tuple[ModelCapabilities, dict]: provider_override = entry.get("provider") if isinstance(provider_override, str): entry_provider = ProviderType(provider_override.lower()) elif isinstance(provider_override, ProviderType): entry_provider = provider_override else: entry_provider = ProviderType.OPENROUTER if entry_provider == ProviderType.CUSTOM: entry.setdefault("friendly_name", f"Custom ({entry['model_name']})") else: entry.setdefault("friendly_name", f"OpenRouter ({entry['model_name']})") filtered = {k: v for k, v in entry.items() if k in CAPABILITY_FIELD_NAMES} filtered.setdefault("provider", entry_provider) capability = ModelCapabilities(**filtered) return capability, {} ================================================ FILE: providers/registries/xai.py ================================================ """Registry loader for X.AI model capabilities.""" from __future__ import annotations from ..shared import ProviderType from .base import CapabilityModelRegistry class XAIModelRegistry(CapabilityModelRegistry): """Capability registry backed by ``conf/xai_models.json``.""" def __init__(self, config_path: str | None = None) -> None: super().__init__( env_var_name="XAI_MODELS_CONFIG_PATH", default_filename="xai_models.json", provider=ProviderType.XAI, friendly_prefix="X.AI ({model})", config_path=config_path, ) ================================================ FILE: providers/registry.py ================================================ """Model provider registry for managing available providers.""" import logging from typing import TYPE_CHECKING, Optional from utils.env import get_env from .base import ModelProvider from .shared import ProviderType if TYPE_CHECKING: from tools.models import ToolModelCategory class ModelProviderRegistry: """Central catalogue of provider implementations used by the MCP server. Role Holds the mapping between :class:`ProviderType` values and concrete :class:`ModelProvider` subclasses/factories. At runtime the registry is responsible for instantiating providers, caching them for reuse, and mediating lookup of providers and model names in provider priority order. Core responsibilities * Resolve API keys and other runtime configuration for each provider * Lazily create provider instances so unused backends incur no cost * Expose convenience methods for enumerating available models and locating which provider can service a requested model name or alias * Honour the project-wide provider priority policy so namespaces (or alias collisions) are resolved deterministically. """ _instance = None # Provider priority order for model selection # Native APIs first, then custom endpoints, then catch-all providers PROVIDER_PRIORITY_ORDER = [ ProviderType.GOOGLE, # Direct Gemini access ProviderType.OPENAI, # Direct OpenAI access ProviderType.AZURE, # Azure-hosted OpenAI deployments ProviderType.XAI, # Direct X.AI GROK access ProviderType.DIAL, # DIAL unified API access ProviderType.CUSTOM, # Local/self-hosted models ProviderType.OPENROUTER, # Catch-all for cloud models ] def __new__(cls): """Singleton pattern for registry.""" if cls._instance is None: logging.debug("REGISTRY: Creating new registry instance") cls._instance = super().__new__(cls) # Initialize instance dictionaries on first creation cls._instance._providers = {} cls._instance._initialized_providers = {} logging.debug(f"REGISTRY: Created instance {cls._instance}") return cls._instance @classmethod def register_provider(cls, provider_type: ProviderType, provider_class: type[ModelProvider]) -> None: """Register a new provider class. Args: provider_type: Type of the provider (e.g., ProviderType.GOOGLE) provider_class: Class that implements ModelProvider interface """ instance = cls() instance._providers[provider_type] = provider_class # Invalidate any cached instance so subsequent lookups use the new registration instance._initialized_providers.pop(provider_type, None) @classmethod def get_provider(cls, provider_type: ProviderType, force_new: bool = False) -> Optional[ModelProvider]: """Get an initialized provider instance. Args: provider_type: Type of provider to get force_new: Force creation of new instance instead of using cached Returns: Initialized ModelProvider instance or None if not available """ instance = cls() # Return cached instance if available and not forcing new if not force_new and provider_type in instance._initialized_providers: return instance._initialized_providers[provider_type] # Check if provider class is registered if provider_type not in instance._providers: return None # Get API key from environment api_key = cls._get_api_key_for_provider(provider_type) # Get provider class or factory function provider_class = instance._providers[provider_type] # For custom providers, handle special initialization requirements if provider_type == ProviderType.CUSTOM: # Check if it's a factory function (callable but not a class) if callable(provider_class) and not isinstance(provider_class, type): # Factory function - call it with api_key parameter provider = provider_class(api_key=api_key) else: # Regular class - need to handle URL requirement custom_url = get_env("CUSTOM_API_URL", "") or "" if not custom_url: if api_key: # Key is set but URL is missing logging.warning("CUSTOM_API_KEY set but CUSTOM_API_URL missing – skipping Custom provider") return None # Use empty string as API key for custom providers that don't need auth (e.g., Ollama) # This allows the provider to be created even without CUSTOM_API_KEY being set api_key = api_key or "" # Initialize custom provider with both API key and base URL provider = provider_class(api_key=api_key, base_url=custom_url) elif provider_type == ProviderType.GOOGLE: # For Gemini, check if custom base URL is configured if not api_key: return None gemini_base_url = get_env("GEMINI_BASE_URL") provider_kwargs = {"api_key": api_key} if gemini_base_url: provider_kwargs["base_url"] = gemini_base_url logging.info(f"Initialized Gemini provider with custom endpoint: {gemini_base_url}") provider = provider_class(**provider_kwargs) elif provider_type == ProviderType.AZURE: if not api_key: return None azure_endpoint = get_env("AZURE_OPENAI_ENDPOINT") if not azure_endpoint: logging.warning("AZURE_OPENAI_ENDPOINT missing – skipping Azure OpenAI provider") return None azure_version = get_env("AZURE_OPENAI_API_VERSION") provider = provider_class( api_key=api_key, azure_endpoint=azure_endpoint, api_version=azure_version, ) else: if not api_key: return None # Initialize non-custom provider with just API key provider = provider_class(api_key=api_key) # Cache the instance instance._initialized_providers[provider_type] = provider return provider @classmethod def get_provider_for_model(cls, model_name: str) -> Optional[ModelProvider]: """Get provider instance for a specific model name. Provider priority order: 1. Native APIs (GOOGLE, OPENAI) - Most direct and efficient 2. CUSTOM - For local/private models with specific endpoints 3. OPENROUTER - Catch-all for cloud models via unified API Args: model_name: Name of the model (e.g., "gemini-2.5-flash", "gpt5") Returns: ModelProvider instance that supports this model """ logging.debug(f"get_provider_for_model called with model_name='{model_name}'") # Check providers in priority order instance = cls() logging.debug(f"Registry instance: {instance}") logging.debug(f"Available providers in registry: {list(instance._providers.keys())}") for provider_type in cls.PROVIDER_PRIORITY_ORDER: if provider_type in instance._providers: logging.debug(f"Found {provider_type} in registry") # Get or create provider instance provider = cls.get_provider(provider_type) if provider and provider.validate_model_name(model_name): logging.debug(f"{provider_type} validates model {model_name}") return provider else: logging.debug(f"{provider_type} does not validate model {model_name}") else: logging.debug(f"{provider_type} not found in registry") logging.debug(f"No provider found for model {model_name}") return None @classmethod def get_available_providers(cls) -> list[ProviderType]: """Get list of registered provider types.""" instance = cls() return list(instance._providers.keys()) @classmethod def get_available_models(cls, respect_restrictions: bool = True) -> dict[str, ProviderType]: """Get mapping of all available models to their providers. Args: respect_restrictions: If True, filter out models not allowed by restrictions Returns: Dict mapping model names to provider types """ # Import here to avoid circular imports from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if respect_restrictions else None models: dict[str, ProviderType] = {} instance = cls() for provider_type in instance._providers: provider = cls.get_provider(provider_type) if not provider: continue try: available = provider.list_models(respect_restrictions=respect_restrictions) except NotImplementedError: logging.warning("Provider %s does not implement list_models", provider_type) continue if restriction_service and restriction_service.has_restrictions(provider_type): restricted_display = cls._collect_restricted_display_names( provider, provider_type, available, restriction_service, ) if restricted_display: for model_name in restricted_display: models[model_name] = provider_type continue for model_name in available: # ===================================================================================== # CRITICAL: Prevent double restriction filtering (Fixed Issue #98) # ===================================================================================== # Previously, both the provider AND registry applied restrictions, causing # double-filtering that resulted in "no models available" errors. # # Logic: If respect_restrictions=True, provider already filtered models, # so registry should NOT filter them again. # TEST COVERAGE: tests/test_provider_routing_bugs.py::TestOpenRouterAliasRestrictions # ===================================================================================== if ( restriction_service and not respect_restrictions # Only filter if provider didn't already filter and not restriction_service.is_allowed(provider_type, model_name) ): logging.debug("Model %s filtered by restrictions", model_name) continue models[model_name] = provider_type return models @classmethod def _collect_restricted_display_names( cls, provider: ModelProvider, provider_type: ProviderType, available: list[str], restriction_service, ) -> list[str] | None: """Derive the human-facing model list when restrictions are active.""" allowed_models = restriction_service.get_allowed_models(provider_type) if not allowed_models: return None allowed_details: list[tuple[str, int]] = [] for model_name in sorted(allowed_models): try: capabilities = provider.get_capabilities(model_name) except (AttributeError, ValueError): continue try: rank = capabilities.get_effective_capability_rank() rank_value = float(rank) except (AttributeError, TypeError, ValueError): rank_value = 0.0 allowed_details.append((model_name, rank_value)) if allowed_details: allowed_details.sort(key=lambda item: (-item[1], item[0])) return [name for name, _ in allowed_details] # Fallback: intersect the allowlist with the provider-advertised names. available_lookup = {name.lower(): name for name in available} display_names: list[str] = [] for model_name in sorted(allowed_models): lowered = model_name.lower() if lowered in available_lookup: display_names.append(available_lookup[lowered]) return display_names @classmethod def get_available_model_names(cls, provider_type: Optional[ProviderType] = None) -> list[str]: """Get list of available model names, optionally filtered by provider. This respects model restrictions automatically. Args: provider_type: Optional provider to filter by Returns: List of available model names """ available_models = cls.get_available_models(respect_restrictions=True) if provider_type: # Filter by specific provider return [name for name, ptype in available_models.items() if ptype == provider_type] else: # Return all available models return list(available_models.keys()) @classmethod def _get_api_key_for_provider(cls, provider_type: ProviderType) -> Optional[str]: """Get API key for a provider from environment variables. Args: provider_type: Provider type to get API key for Returns: API key string or None if not found """ key_mapping = { ProviderType.GOOGLE: "GEMINI_API_KEY", ProviderType.OPENAI: "OPENAI_API_KEY", ProviderType.AZURE: "AZURE_OPENAI_API_KEY", ProviderType.XAI: "XAI_API_KEY", ProviderType.OPENROUTER: "OPENROUTER_API_KEY", ProviderType.CUSTOM: "CUSTOM_API_KEY", # Can be empty for providers that don't need auth ProviderType.DIAL: "DIAL_API_KEY", } env_var = key_mapping.get(provider_type) if not env_var: return None return get_env(env_var) @classmethod def _get_allowed_models_for_provider(cls, provider: ModelProvider, provider_type: ProviderType) -> list[str]: """Get a list of allowed canonical model names for a given provider. Args: provider: The provider instance to get models for provider_type: The provider type for restriction checking Returns: List of model names that are both supported and allowed """ from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() allowed_models = [] # Get the provider's supported models try: # Use list_models to get all supported models (handles both regular and custom providers) supported_models = provider.list_models(respect_restrictions=False) except (NotImplementedError, AttributeError): # Fallback to provider-declared capability maps if list_models not implemented model_map = getattr(provider, "MODEL_CAPABILITIES", None) supported_models = list(model_map.keys()) if isinstance(model_map, dict) else [] # Filter by restrictions for model_name in supported_models: if restriction_service.is_allowed(provider_type, model_name): allowed_models.append(model_name) return allowed_models @classmethod def get_preferred_fallback_model(cls, tool_category: Optional["ToolModelCategory"] = None) -> str: """Get the preferred fallback model based on provider priority and tool category. This method orchestrates model selection by: 1. Getting allowed models for each provider (respecting restrictions) 2. Asking providers for their preference from the allowed list 3. Falling back to first available model if no preference given Args: tool_category: Optional category to influence model selection Returns: Model name string for fallback use """ from tools.models import ToolModelCategory effective_category = tool_category or ToolModelCategory.BALANCED first_available_model = None # Ask each provider for their preference in priority order for provider_type in cls.PROVIDER_PRIORITY_ORDER: provider = cls.get_provider(provider_type) if provider: # 1. Registry filters the models first allowed_models = cls._get_allowed_models_for_provider(provider, provider_type) if not allowed_models: continue # 2. Keep track of the first available model as fallback if not first_available_model: first_available_model = sorted(allowed_models)[0] # 3. Ask provider to pick from allowed list preferred_model = provider.get_preferred_model(effective_category, allowed_models) if preferred_model: logging.debug( f"Provider {provider_type.value} selected '{preferred_model}' for category '{effective_category.value}'" ) return preferred_model # If no provider returned a preference, use first available model if first_available_model: logging.debug(f"No provider preference, using first available: {first_available_model}") return first_available_model # Ultimate fallback if no providers have models logging.warning("No models available from any provider, using default fallback") return "gemini-2.5-flash" @classmethod def get_available_providers_with_keys(cls) -> list[ProviderType]: """Get list of provider types that have valid API keys. Returns: List of ProviderType values for providers with valid API keys """ available = [] instance = cls() for provider_type in instance._providers: if cls.get_provider(provider_type) is not None: available.append(provider_type) return available @classmethod def clear_cache(cls) -> None: """Clear cached provider instances.""" instance = cls() instance._initialized_providers.clear() @classmethod def reset_for_testing(cls) -> None: """Reset the registry to a clean state for testing. This provides a safe, public API for tests to clean up registry state without directly manipulating private attributes. """ cls._instance = None if hasattr(cls, "_providers"): cls._providers = {} @classmethod def unregister_provider(cls, provider_type: ProviderType) -> None: """Unregister a provider (mainly for testing).""" instance = cls() instance._providers.pop(provider_type, None) instance._initialized_providers.pop(provider_type, None) ================================================ FILE: providers/registry_provider_mixin.py ================================================ """Mixin for providers backed by capability registries. This mixin centralises the boilerplate for providers that expose their model capabilities via JSON configuration files. Subclasses only need to set ``REGISTRY_CLASS`` to an appropriate :class:`CapabilityModelRegistry` and the mix-in will take care of: * Populating ``MODEL_CAPABILITIES`` exactly once per process (with optional reload support for tests). * Lazily exposing the registry contents through the standard provider hooks (:meth:`get_all_model_capabilities` and :meth:`get_model_registry`). * Providing defensive logging when a registry cannot be constructed so the provider can degrade gracefully instead of raising during import. Using this helper keeps individual provider implementations focused on their SDK-specific behaviour while ensuring capability loading is consistent across OpenAI, Gemini, X.AI, and other native backends. """ from __future__ import annotations import logging from typing import ClassVar from .registries.base import CapabilityModelRegistry from .shared import ModelCapabilities class RegistryBackedProviderMixin: """Shared helper for providers that load capabilities from JSON registries.""" REGISTRY_CLASS: ClassVar[type[CapabilityModelRegistry] | None] = None _registry: ClassVar[CapabilityModelRegistry | None] = None MODEL_CAPABILITIES: ClassVar[dict[str, ModelCapabilities]] = {} @classmethod def _registry_logger(cls) -> logging.Logger: """Return the logger used for registry lifecycle messages.""" return logging.getLogger(cls.__module__) @classmethod def _ensure_registry(cls, *, force_reload: bool = False) -> None: """Populate ``MODEL_CAPABILITIES`` from the configured registry. Args: force_reload: When ``True`` the registry is re-created even if it was previously loaded. This is primarily used by tests. """ if cls.REGISTRY_CLASS is None: # pragma: no cover - defensive programming raise RuntimeError(f"{cls.__name__} must define REGISTRY_CLASS.") if cls._registry is not None and not force_reload: return try: registry = cls.REGISTRY_CLASS() except Exception as exc: # pragma: no cover - registry failures shouldn't break the provider cls._registry_logger().warning("Unable to load %s registry: %s", cls.__name__, exc) cls._registry = None cls.MODEL_CAPABILITIES = {} return cls._registry = registry cls.MODEL_CAPABILITIES = dict(registry.model_map) @classmethod def reload_registry(cls) -> None: """Force a registry reload (used in tests).""" cls._ensure_registry(force_reload=True) def get_all_model_capabilities(self) -> dict[str, ModelCapabilities]: """Return the registry-backed ``MODEL_CAPABILITIES`` map.""" self._ensure_registry() return super().get_all_model_capabilities() def get_model_registry(self) -> dict[str, ModelCapabilities] | None: """Return a copy of the underlying registry map when available.""" if self._registry is None: return None return dict(self._registry.model_map) ================================================ FILE: providers/shared/__init__.py ================================================ """Shared data structures and helpers for model providers.""" from .model_capabilities import ModelCapabilities from .model_response import ModelResponse from .provider_type import ProviderType from .temperature import ( DiscreteTemperatureConstraint, FixedTemperatureConstraint, RangeTemperatureConstraint, TemperatureConstraint, ) __all__ = [ "ModelCapabilities", "ModelResponse", "ProviderType", "TemperatureConstraint", "FixedTemperatureConstraint", "RangeTemperatureConstraint", "DiscreteTemperatureConstraint", ] ================================================ FILE: providers/shared/model_capabilities.py ================================================ """Dataclass describing the feature set of a model exposed by a provider.""" import math from dataclasses import dataclass, field from typing import Optional from .provider_type import ProviderType from .temperature import RangeTemperatureConstraint, TemperatureConstraint __all__ = ["ModelCapabilities"] @dataclass class ModelCapabilities: """Static description of what a model can do within a provider. Role Acts as the canonical record for everything the server needs to know about a model—its provider, token limits, feature switches, aliases, and temperature rules. Providers populate these objects so tools and higher-level services can rely on a consistent schema. Typical usage * Provider subclasses declare `MODEL_CAPABILITIES` maps containing these objects (for example ``OpenAIModelProvider``) * Helper utilities (e.g. restriction validation, alias expansion) read these objects to build model lists for tooling and policy enforcement * Tool selection logic inspects attributes such as ``supports_extended_thinking`` or ``context_window`` to choose an appropriate model for a task. * The ``allow_code_generation`` flag enables structured code generation in the chat tool for models more capable than the primary CLI. """ provider: ProviderType model_name: str friendly_name: str intelligence_score: int = 10 # Human-curated 1–20 score reflecting general capability description: str = "" aliases: list[str] = field(default_factory=list) # Capacity limits / resource budgets context_window: int = 0 max_output_tokens: int = 0 max_thinking_tokens: int = 0 # Capability flags supports_extended_thinking: bool = False supports_system_prompts: bool = True supports_streaming: bool = True supports_function_calling: bool = False supports_images: bool = False supports_json_mode: bool = False supports_temperature: bool = True use_openai_response_api: bool = False default_reasoning_effort: Optional[str] = None allow_code_generation: bool = ( False # Enables structured code generation in chat tool for substantial implementations ) # Additional attributes max_image_size_mb: float = 0.0 temperature_constraint: TemperatureConstraint = field( default_factory=lambda: RangeTemperatureConstraint(0.0, 2.0, 0.3) ) def get_effective_temperature(self, requested_temperature: float) -> Optional[float]: """Return the temperature that should be sent to the provider. Models that do not support temperature return ``None`` so that callers can omit the parameter entirely. For supported models, the configured constraint clamps the requested value into a provider-safe range. """ if not self.supports_temperature: return None return self.temperature_constraint.get_corrected_value(requested_temperature) def get_effective_capability_rank(self) -> int: """Calculate the runtime capability rank from intelligence + capabilities.""" # Human signal drives the baseline (1–20 → 5–100 after scaling) base_intelligence = self.intelligence_score if self.intelligence_score else 10 base_intelligence = max(1, min(20, base_intelligence)) score = base_intelligence * 5 # Context window bonus with gentle diminishing returns ctx_bonus = 0 ctx = max(self.context_window, 0) if ctx > 0: ctx_bonus = int(min(5, max(0.0, math.log10(ctx) - 3))) score += ctx_bonus # Output token capacity adds a small bonus if self.max_output_tokens >= 65_000: score += 2 elif self.max_output_tokens >= 32_000: score += 1 # Feature-level boosts if self.supports_extended_thinking: score += 3 if self.supports_function_calling: score += 1 if self.supports_json_mode: score += 1 if self.supports_images: score += 1 return max(0, min(100, score)) @staticmethod def collect_aliases(model_configs: dict[str, "ModelCapabilities"]) -> dict[str, list[str]]: """Build a mapping of model name to aliases from capability configs.""" return { base_model: capabilities.aliases for base_model, capabilities in model_configs.items() if capabilities.aliases } @staticmethod def collect_model_names( model_configs: dict[str, "ModelCapabilities"], *, include_aliases: bool = True, lowercase: bool = False, unique: bool = False, ) -> list[str]: """Build an ordered list of model names and aliases. Args: model_configs: Mapping of canonical model names to capabilities. include_aliases: When True, include aliases for each model. lowercase: When True, normalize names to lowercase. unique: When True, ensure each returned name appears once (after formatting). Returns: Ordered list of model names (and optionally aliases) formatted per options. """ formatted_names: list[str] = [] seen: set[str] | None = set() if unique else None def append_name(name: str) -> None: formatted = name.lower() if lowercase else name if seen is not None: if formatted in seen: return seen.add(formatted) formatted_names.append(formatted) # Sort models by capability rank (descending) then by name for deterministic ordering sorted_items = sorted( model_configs.items(), key=lambda item: (-item[1].get_effective_capability_rank(), item[0]), ) for base_model, capabilities in sorted_items: append_name(base_model) if include_aliases and capabilities.aliases: for alias in capabilities.aliases: append_name(alias) return formatted_names ================================================ FILE: providers/shared/model_response.py ================================================ """Dataclass used to normalise provider SDK responses.""" from dataclasses import dataclass, field from typing import Any from .provider_type import ProviderType __all__ = ["ModelResponse"] @dataclass class ModelResponse: """Portable representation of a provider completion.""" content: str usage: dict[str, int] = field(default_factory=dict) model_name: str = "" friendly_name: str = "" provider: ProviderType = ProviderType.GOOGLE metadata: dict[str, Any] = field(default_factory=dict) @property def total_tokens(self) -> int: """Return the total token count if the provider reported usage data.""" return self.usage.get("total_tokens", 0) ================================================ FILE: providers/shared/provider_type.py ================================================ """Enumeration describing which backend owns a given model.""" from enum import Enum __all__ = ["ProviderType"] class ProviderType(Enum): """Canonical identifiers for every supported provider backend.""" GOOGLE = "google" OPENAI = "openai" AZURE = "azure" XAI = "xai" OPENROUTER = "openrouter" CUSTOM = "custom" DIAL = "dial" ================================================ FILE: providers/shared/temperature.py ================================================ """Helper types for validating model temperature parameters.""" from abc import ABC, abstractmethod from typing import Optional __all__ = [ "TemperatureConstraint", "FixedTemperatureConstraint", "RangeTemperatureConstraint", "DiscreteTemperatureConstraint", ] # Common heuristics for determining temperature support when explicit # capabilities are unavailable (e.g., custom/local models). _TEMP_UNSUPPORTED_PATTERNS = { "o1", "o3", "o4", # OpenAI O-series reasoning models "deepseek-reasoner", "deepseek-r1", "r1", # DeepSeek reasoner variants } _TEMP_UNSUPPORTED_KEYWORDS = { "reasoner", # Catch additional DeepSeek-style naming patterns } class TemperatureConstraint(ABC): """Contract for temperature validation used by `ModelCapabilities`. Concrete providers describe their temperature behaviour by creating subclasses that expose three operations: * `validate` – decide whether a requested temperature is acceptable. * `get_corrected_value` – coerce out-of-range values into a safe default. * `get_description` – provide a human readable error message for users. Providers call these hooks before sending traffic to the underlying API so that unsupported temperatures never reach the remote service. """ @abstractmethod def validate(self, temperature: float) -> bool: """Return ``True`` when the temperature may be sent to the backend.""" @abstractmethod def get_corrected_value(self, temperature: float) -> float: """Return a valid substitute for an out-of-range temperature.""" @abstractmethod def get_description(self) -> str: """Describe the acceptable range to include in error messages.""" @abstractmethod def get_default(self) -> float: """Return the default temperature for the model.""" @staticmethod def infer_support(model_name: str) -> tuple[bool, str]: """Heuristically determine whether a model supports temperature.""" model_lower = model_name.lower() for pattern in _TEMP_UNSUPPORTED_PATTERNS: conditions = ( pattern == model_lower, model_lower.startswith(f"{pattern}-"), model_lower.startswith(f"openai/{pattern}"), model_lower.startswith(f"deepseek/{pattern}"), model_lower.endswith(f"-{pattern}"), f"/{pattern}" in model_lower, f"-{pattern}-" in model_lower, ) if any(conditions): return False, f"detected pattern '{pattern}'" for keyword in _TEMP_UNSUPPORTED_KEYWORDS: if keyword in model_lower: return False, f"detected keyword '{keyword}'" return True, "default assumption for models without explicit metadata" @staticmethod def resolve_settings( model_name: str, constraint_hint: Optional[str] = None, ) -> tuple[bool, "TemperatureConstraint", str]: """Derive temperature support and constraint for a model. Args: model_name: Canonical model identifier or alias. constraint_hint: Optional configuration hint (``"fixed"``, ``"range"``, ``"discrete"``). When provided, the hint is honoured directly. Returns: Tuple ``(supports_temperature, constraint, diagnosis)`` describing whether temperature may be tuned, the constraint object that should be attached to :class:`ModelCapabilities`, and the reasoning behind the decision. """ if constraint_hint: constraint = TemperatureConstraint.create(constraint_hint) supports_temperature = constraint_hint != "fixed" reason = f"constraint hint '{constraint_hint}'" return supports_temperature, constraint, reason supports_temperature, reason = TemperatureConstraint.infer_support(model_name) if supports_temperature: constraint: TemperatureConstraint = RangeTemperatureConstraint(0.0, 2.0, 0.7) else: constraint = FixedTemperatureConstraint(1.0) return supports_temperature, constraint, reason @staticmethod def create(constraint_type: str) -> "TemperatureConstraint": """Factory that yields the appropriate constraint for a configuration hint.""" if constraint_type == "fixed": # Fixed temperature models (O3/O4) only support temperature=1.0 return FixedTemperatureConstraint(1.0) if constraint_type == "discrete": # For models with specific allowed values - using common OpenAI values as default return DiscreteTemperatureConstraint([0.0, 0.3, 0.7, 1.0, 1.5, 2.0], 0.3) # Default range constraint (for "range" or None) return RangeTemperatureConstraint(0.0, 2.0, 0.3) class FixedTemperatureConstraint(TemperatureConstraint): """Constraint for models that enforce an exact temperature (for example O3).""" def __init__(self, value: float): self.value = value def validate(self, temperature: float) -> bool: return abs(temperature - self.value) < 1e-6 # Handle floating point precision def get_corrected_value(self, temperature: float) -> float: return self.value def get_description(self) -> str: return f"Only supports temperature={self.value}" def get_default(self) -> float: return self.value class RangeTemperatureConstraint(TemperatureConstraint): """Constraint for providers that expose a continuous min/max temperature range.""" def __init__(self, min_temp: float, max_temp: float, default: Optional[float] = None): self.min_temp = min_temp self.max_temp = max_temp self.default_temp = default or (min_temp + max_temp) / 2 def validate(self, temperature: float) -> bool: return self.min_temp <= temperature <= self.max_temp def get_corrected_value(self, temperature: float) -> float: return max(self.min_temp, min(self.max_temp, temperature)) def get_description(self) -> str: return f"Supports temperature range [{self.min_temp}, {self.max_temp}]" def get_default(self) -> float: return self.default_temp class DiscreteTemperatureConstraint(TemperatureConstraint): """Constraint for models that permit a discrete list of temperature values.""" def __init__(self, allowed_values: list[float], default: Optional[float] = None): self.allowed_values = sorted(allowed_values) self.default_temp = default or allowed_values[len(allowed_values) // 2] def validate(self, temperature: float) -> bool: return any(abs(temperature - val) < 1e-6 for val in self.allowed_values) def get_corrected_value(self, temperature: float) -> float: return min(self.allowed_values, key=lambda x: abs(x - temperature)) def get_description(self) -> str: return f"Supports temperatures: {self.allowed_values}" def get_default(self) -> float: return self.default_temp ================================================ FILE: providers/xai.py ================================================ """X.AI (GROK) model provider implementation.""" import logging from typing import TYPE_CHECKING, ClassVar, Optional if TYPE_CHECKING: from tools.models import ToolModelCategory from .openai_compatible import OpenAICompatibleProvider from .registries.xai import XAIModelRegistry from .registry_provider_mixin import RegistryBackedProviderMixin from .shared import ModelCapabilities, ProviderType logger = logging.getLogger(__name__) class XAIModelProvider(RegistryBackedProviderMixin, OpenAICompatibleProvider): """Integration for X.AI's GROK models exposed over an OpenAI-style API. Publishes capability metadata for the officially supported deployments and maps tool-category preferences to the appropriate GROK model. """ FRIENDLY_NAME = "X.AI" REGISTRY_CLASS = XAIModelRegistry MODEL_CAPABILITIES: ClassVar[dict[str, ModelCapabilities]] = {} # Canonical model identifiers used for category routing. PRIMARY_MODEL = "grok-4-1-fast-reasoning" FALLBACK_MODEL = "grok-4" def __init__(self, api_key: str, **kwargs): """Initialize X.AI provider with API key.""" # Set X.AI base URL kwargs.setdefault("base_url", "https://api.x.ai/v1") self._ensure_registry() super().__init__(api_key, **kwargs) self._invalidate_capability_cache() def get_provider_type(self) -> ProviderType: """Get the provider type.""" return ProviderType.XAI def get_preferred_model(self, category: "ToolModelCategory", allowed_models: list[str]) -> Optional[str]: """Get XAI's preferred model for a given category from allowed models. Args: category: The tool category requiring a model allowed_models: Pre-filtered list of models allowed by restrictions Returns: Preferred model name or None """ from tools.models import ToolModelCategory if not allowed_models: return None if category == ToolModelCategory.EXTENDED_REASONING: # Prefer Grok 4.1 Fast Reasoning for advanced tasks if self.PRIMARY_MODEL in allowed_models: return self.PRIMARY_MODEL if self.FALLBACK_MODEL in allowed_models: return self.FALLBACK_MODEL return allowed_models[0] elif category == ToolModelCategory.FAST_RESPONSE: # Prefer Grok 4.1 Fast Reasoning for speed as well (latest fast SKU). if self.PRIMARY_MODEL in allowed_models: return self.PRIMARY_MODEL if self.FALLBACK_MODEL in allowed_models: return self.FALLBACK_MODEL return allowed_models[0] else: # BALANCED or default # Prefer Grok 4.1 Fast Reasoning for balanced use. if self.PRIMARY_MODEL in allowed_models: return self.PRIMARY_MODEL if self.FALLBACK_MODEL in allowed_models: return self.FALLBACK_MODEL return allowed_models[0] # Load registry data at import time XAIModelProvider._ensure_registry() ================================================ FILE: pyproject.toml ================================================ [project] name = "pal-mcp-server" version = "9.8.2" description = "AI-powered MCP server with multiple model providers" requires-python = ">=3.9" dependencies = [ "mcp>=1.0.0", "google-genai>=1.19.0", "openai>=1.55.2", "pydantic>=2.0.0", "python-dotenv>=1.0.0", ] [tool.setuptools.packages.find] include = ["tools*", "providers*", "systemprompts*", "utils*", "conf*", "clink*"] [tool.setuptools] py-modules = ["server", "config"] [tool.setuptools.package-data] "*" = [ "conf/*.json", "conf/cli_clients/*.json", "systemprompts/clink/*.txt", ] [tool.setuptools.data-files] "conf" = [ "conf/custom_models.json", "conf/openrouter_models.json", "conf/azure_models.json", "conf/openai_models.json", "conf/gemini_models.json", "conf/xai_models.json", "conf/dial_models.json", ] [project.scripts] pal-mcp-server = "server:run" [tool.black] line-length = 120 target-version = ['py39', 'py310', 'py311', 'py312', 'py313'] include = '\.pyi?$' extend-exclude = ''' /( # directories \.eggs | \.git | \.hg | \.mypy_cache | \.tox | \.venv | \.pal_venv | venv | _build | buck-out | build | dist )/ ''' [tool.isort] profile = "black" multi_line_output = 3 include_trailing_comma = true force_grid_wrap = 0 use_parentheses = true ensure_newline_before_comments = true line_length = 120 skip_glob = ["venv/*", ".venv/*", ".pal_venv/*"] [tool.ruff] target-version = "py39" line-length = 120 [tool.ruff.lint] select = [ "E", # pycodestyle errors "W", # pycodestyle warnings "F", # pyflakes "I", # isort "B", # flake8-bugbear "C4", # flake8-comprehensions "UP", # pyupgrade ] ignore = [ "E501", # line too long, handled by black "B008", # do not perform function calls in argument defaults "C901", # too complex "B904", # exception handling with raise from ] [tool.ruff.lint.per-file-ignores] "__init__.py" = ["F401"] "tests/*" = ["B011"] "tests/conftest.py" = ["E402"] # Module level imports not at top of file - needed for test setup [tool.semantic_release] version_toml = ["pyproject.toml:project.version"] branch = "main" version_source = "tag" version_pattern = "v(?P\\d+)\\.(?P\\d+)\\.(?P\\d+)" major_on_zero = false build_command = "python -m pip install --upgrade build && python -m build" dist_path = "dist/" upload_to_vcs_release = true upload_to_repository = false remove_dist = false commit_version_number = true commit_message = "chore(release): {version}\n\nAutomatically generated by python-semantic-release" tag_format = "v{version}" [tool.semantic_release.branches.main] match = "main" prerelease = false [tool.semantic_release.changelog] exclude_commit_patterns = [] [tool.semantic_release.commit_parser_options] allowed_tags = ["build", "chore", "ci", "docs", "feat", "fix", "perf", "style", "refactor", "test"] minor_tags = ["feat"] patch_tags = ["fix", "perf"] [tool.semantic_release.remote.token] env = "GH_TOKEN" [build-system] requires = ["setuptools>=45", "wheel", "setuptools_scm[toml]>=6.2"] build-backend = "setuptools.build_meta" ================================================ FILE: pytest.ini ================================================ [pytest] testpaths = tests python_files = test_*.py python_classes = Test* python_functions = test_* asyncio_mode = auto addopts = -v --strict-markers --tb=short markers = integration: marks tests as integration tests that make real API calls with local-llama (free to run) ================================================ FILE: requirements-dev.txt ================================================ pytest>=7.4.0 pytest-asyncio>=0.21.0 pytest-mock>=3.11.0 black>=23.0.0 ruff>=0.1.0 isort>=5.12.0 python-semantic-release>=10.3.0 build>=1.0.0 ================================================ FILE: requirements.txt ================================================ mcp>=1.0.0 google-genai>=1.19.0 openai>=1.55.2 # Minimum version for httpx 0.28.0 compatibility pydantic>=2.0.0 python-dotenv>=1.0.0 importlib-resources>=5.0.0; python_version<"3.9" # Development dependencies (install with pip install -r requirements-dev.txt) # pytest>=7.4.0 # pytest-asyncio>=0.21.0 # pytest-mock>=3.11.0 ================================================ FILE: run-server.ps1 ================================================ <# .SYNOPSIS Installation, configuration, and launch script for PAL MCP server on Windows. .DESCRIPTION This PowerShell script prepares the environment for the PAL MCP server: - Installs and checks Python 3.10+ (with venv or uv if available) - Installs required Python dependencies - Configures environment files (.env) - Validates presence of required API keys - Cleans Python caches and obsolete Docker artifacts - Offers automatic integration with Claude Desktop, Gemini CLI, VSCode, Cursor, Windsurf, and Trae - Manages configuration file backups (max 3 retained) - Allows real-time log following or server launch .PARAMETER Help Shows script help. .PARAMETER Version Shows PAL MCP server version. .PARAMETER Follow Follows server logs in real time. .PARAMETER Config Shows configuration instructions for Claude and other compatible clients. .PARAMETER ClearCache Removes Python cache files (__pycache__, .pyc). .PARAMETER SkipVenv Skips Python virtual environment creation. .PARAMETER SkipDocker Skips Docker checks and cleanup. .PARAMETER Force Forces recreation of the Python virtual environment. .PARAMETER VerboseOutput Enables more detailed output (currently unused). .PARAMETER Dev Installs development dependencies from requirements-dev.txt if available. .PARAMETER Docker Uses Docker to build and run the MCP server instead of Python virtual environment. .EXAMPLE .\run-server.ps1 Prepares the environment and starts the PAL MCP server. .\run-server.ps1 -Follow Follows server logs in real time. .\run-server.ps1 -Config Shows configuration instructions for clients. .\run-server.ps1 -Dev Prepares the environment with development dependencies and starts the server. .\run-server.ps1 -Docker Builds and runs the server using Docker containers. .\run-server.ps1 -Docker -Follow Builds and runs the server using Docker containers and follows the logs. .\run-server.ps1 -Docker -Force Forces rebuilding of the Docker image and runs the server. .NOTES Project Author : BeehiveInnovations Script Author : GiGiDKR (https://github.com/GiGiDKR) Date : 07-05-2025 Version : See config.py (__version__) References : https://github.com/BeehiveInnovations/pal-mcp-server #> #Requires -Version 5.1 [CmdletBinding()] param( [switch]$Help, [switch]$Version, [switch]$Follow, [switch]$Config, [switch]$ClearCache, [switch]$SkipVenv, [switch]$SkipDocker, [switch]$Force, [switch]$VerboseOutput, [switch]$Dev, [switch]$Docker ) # ============================================================================ # PAL MCP Server Setup Script for Windows # # A Windows-compatible setup script that handles environment setup, # dependency installation, and configuration. # ============================================================================ # Set error action preference $ErrorActionPreference = "Stop" # ---------------------------------------------------------------------------- # Constants and Configuration # ---------------------------------------------------------------------------- $script:VENV_PATH = ".pal_venv" $script:DOCKER_CLEANED_FLAG = ".docker_cleaned" $script:DESKTOP_CONFIG_FLAG = ".desktop_configured" $script:LOG_DIR = "logs" $script:LOG_FILE = "mcp_server.log" $script:LegacyServerNames = @("zen", "zen-mcp", "zen-mcp-server", "zen_mcp", "zen_mcp_server") # ---------------------------------------------------------------------------- # Utility Functions # ---------------------------------------------------------------------------- function Write-Success { param([string]$Message) Write-Host "✓ " -ForegroundColor Green -NoNewline Write-Host $Message } function Write-Error { param([string]$Message) Write-Host "✗ " -ForegroundColor Red -NoNewline Write-Host $Message } function Write-Warning { param([string]$Message) Write-Host "⚠ " -ForegroundColor Yellow -NoNewline Write-Host $Message } function Write-Info { param([string]$Message) Write-Host "ℹ " -ForegroundColor Cyan -NoNewline Write-Host $Message } function Write-Step { param([string]$Message) Write-Host "" Write-Host "=== $Message ===" -ForegroundColor Cyan } # Check if command exists function Test-Command { param([string]$Command) try { $null = Get-Command $Command -ErrorAction Stop return $true } catch { return $false } } # Alternative method to force remove locked directories function Remove-LockedDirectory { param([string]$Path) if (!(Test-Path $Path)) { return $true } try { # Try standard removal first Remove-Item -Recurse -Force $Path -ErrorAction Stop return $true } catch { Write-Warning "Standard removal failed, trying alternative methods..." # Method 1: Use takeown and icacls to force ownership try { Write-Info "Attempting to take ownership of locked files..." takeown /F "$Path" /R /D Y 2>$null | Out-Null icacls "$Path" /grant administrators:F /T 2>$null | Out-Null Remove-Item -Recurse -Force $Path -ErrorAction Stop return $true } catch { Write-Warning "Ownership method failed" } # Method 2: Rename and schedule for deletion on reboot try { $tempName = "$Path.delete_$(Get-Random)" Write-Info "Renaming to: $tempName (will be deleted on next reboot)" Rename-Item $Path $tempName -ErrorAction Stop # Schedule for deletion on reboot using movefile if (Get-Command "schtasks" -ErrorAction SilentlyContinue) { Write-Info "Scheduling for deletion on next reboot..." } Write-Warning "Environment renamed to $tempName and will be deleted on next reboot" return $true } catch { Write-Warning "Rename method failed" } # If all methods fail, return false return $false } } # Remove legacy MCP server entries from a hash/dictionary or PSObject function Remove-LegacyServerKeys { param([object]$Container) $removed = $false if ($null -eq $Container) { return $false } foreach ($legacy in $script:LegacyServerNames) { if ($Container -is [System.Collections.IDictionary]) { if ($Container.Contains($legacy)) { $Container.Remove($legacy) | Out-Null $removed = $true } } elseif ($Container.PSObject -and $Container.PSObject.Properties[$legacy]) { $Container.PSObject.Properties.Remove($legacy) $removed = $true } } return $removed } # Manage configuration file backups with maximum 3 files retention function Manage-ConfigBackups { param( [string]$ConfigFilePath, [int]$MaxBackups = 3 ) if (!(Test-Path $ConfigFilePath)) { Write-Warning "Configuration file not found: $ConfigFilePath" return $null } try { # Create new backup with timestamp $timestamp = Get-Date -Format 'yyyyMMdd_HHmmss' $backupPath = "$ConfigFilePath.backup_$timestamp" Copy-Item $ConfigFilePath $backupPath -ErrorAction Stop # Find all existing backups for this config file $configDir = Split-Path $ConfigFilePath -Parent $configFileName = Split-Path $ConfigFilePath -Leaf $backupPattern = "$configFileName.backup_*" $existingBackups = Get-ChildItem -Path $configDir -Filter $backupPattern -ErrorAction SilentlyContinue | Sort-Object LastWriteTime -Descending # Keep only the most recent MaxBackups files if ($existingBackups.Count -gt $MaxBackups) { $backupsToRemove = $existingBackups | Select-Object -Skip $MaxBackups foreach ($backup in $backupsToRemove) { try { Remove-Item $backup.FullName -Force -ErrorAction Stop Write-Info "Removed old backup: $($backup.Name)" } catch { Write-Warning "Could not remove old backup: $($backup.Name)" } } Write-Success "Backup retention: kept $MaxBackups most recent backups" } Write-Success "Backup created: $(Split-Path $backupPath -Leaf)" return $backupPath } catch { Write-Warning "Failed to create backup: $_" return $null } } # Get version from config.py function Get-Version { try { if (Test-Path "config.py") { $content = Get-Content "config.py" -ErrorAction Stop $versionLine = $content | Where-Object { $_ -match '^__version__ = ' } if ($versionLine) { return ($versionLine -replace '__version__ = "([^"]*)"', '$1') } } return "unknown" } catch { return "unknown" } } # Clear Python cache files function Clear-PythonCache { Write-Info "Clearing Python cache files..." try { # Remove .pyc files Get-ChildItem -Path . -Recurse -Filter "*.pyc" -ErrorAction SilentlyContinue | Remove-Item -Force # Remove __pycache__ directories Get-ChildItem -Path . -Recurse -Name "__pycache__" -Directory -ErrorAction SilentlyContinue | ForEach-Object { Remove-Item -Path $_ -Recurse -Force } Write-Success "Python cache cleared" } catch { Write-Warning "Could not clear all cache files: $_" } } # Get absolute path function Get-AbsolutePath { param([string]$Path) if (Test-Path $Path) { # Use Resolve-Path for full resolution return Resolve-Path $Path } else { # Use unresolved method return $ExecutionContext.SessionState.Path.GetUnresolvedProviderPathFromPSPath($Path) } } # Check Python version function Test-PythonVersion { param([string]$PythonCmd) try { $version = & $PythonCmd --version 2>&1 if ($version -match "Python (\d+)\.(\d+)") { $major = [int]$matches[1] $minor = [int]$matches[2] return ($major -gt 3) -or ($major -eq 3 -and $minor -ge 10) } return $false } catch { return $false } } # Find Python installation function Find-Python { $pythonCandidates = @("python", "python3", "py") foreach ($cmd in $pythonCandidates) { if (Test-Command $cmd) { if (Test-PythonVersion $cmd) { $version = & $cmd --version 2>&1 Write-Success "Found Python: $version" return $cmd } } } # Try Windows Python Launcher with specific versions $pythonVersions = @("3.12", "3.11", "3.10", "3.9") foreach ($version in $pythonVersions) { $cmd = "py -$version" try { $null = Invoke-Expression "$cmd --version" 2>$null Write-Success "Found Python via py launcher: $cmd" return $cmd } catch { continue } } Write-Error "Python 3.10+ not found. Please install Python from https://python.org" return $null } # Clean up old Docker artifacts function Cleanup-Docker { if (Test-Path $DOCKER_CLEANED_FLAG) { return } if (!(Test-Command "docker")) { return } try { $null = docker info 2>$null } catch { return } $foundArtifacts = $false # Define containers to remove $containers = @( "gemini-mcp-server", "gemini-mcp-redis", "pal-mcp-server", "pal-mcp-redis", "pal-mcp-log-monitor" ) # Remove containers foreach ($container in $containers) { try { $exists = docker ps -a --format "{{.Names}}" | Where-Object { $_ -eq $container } if ($exists) { if (!$foundArtifacts) { Write-Info "One-time Docker cleanup..." $foundArtifacts = $true } Write-Info " Removing container: $container" docker stop $container 2>$null | Out-Null docker rm $container 2>$null | Out-Null } } catch { # Ignore errors } } # Remove images $images = @("gemini-mcp-server:latest", "pal-mcp-server:latest") foreach ($image in $images) { try { $exists = docker images --format "{{.Repository}}:{{.Tag}}" | Where-Object { $_ -eq $image } if ($exists) { if (!$foundArtifacts) { Write-Info "One-time Docker cleanup..." $foundArtifacts = $true } Write-Info " Removing image: $image" docker rmi $image 2>$null | Out-Null } } catch { # Ignore errors } } # Remove volumes $volumes = @("redis_data", "mcp_logs") foreach ($volume in $volumes) { try { $exists = docker volume ls --format "{{.Name}}" | Where-Object { $_ -eq $volume } if ($exists) { if (!$foundArtifacts) { Write-Info "One-time Docker cleanup..." $foundArtifacts = $true } Write-Info " Removing volume: $volume" docker volume rm $volume 2>$null | Out-Null } } catch { # Ignore errors } } if ($foundArtifacts) { Write-Success "Docker cleanup complete" } New-Item -Path $DOCKER_CLEANED_FLAG -ItemType File -Force | Out-Null } # Validate API keys function Test-ApiKeys { Write-Step "Validating API Keys" if (!(Test-Path ".env")) { Write-Warning "No .env file found. API keys should be configured." return $false } $envContent = Get-Content ".env" $hasValidKey = $false $keyPatterns = @{ "GEMINI_API_KEY" = "AIza[0-9A-Za-z-_]{35}" "OPENAI_API_KEY" = "sk-[a-zA-Z0-9]{20}T3BlbkFJ[a-zA-Z0-9]{20}" "XAI_API_KEY" = "xai-[a-zA-Z0-9-_]+" "OPENROUTER_API_KEY" = "sk-or-[a-zA-Z0-9-_]+" } foreach ($line in $envContent) { if ($line -match '^([^#][^=]*?)=(.*)$') { $key = $matches[1].Trim() $value = $matches[2].Trim() -replace '^["'']|["'']$', '' if ($keyPatterns.ContainsKey($key) -and $value -ne "your_${key.ToLower()}_here" -and $value.Length -gt 10) { Write-Success "Found valid $key" $hasValidKey = $true } } } if (!$hasValidKey) { Write-Warning "No valid API keys found in .env file" Write-Info "Please edit .env file with your actual API keys" return $false } return $true } # Check if uv is available function Test-Uv { return Test-Command "uv" } # Setup environment using uv-first approach function Initialize-Environment { Write-Step "Setting up Python Environment" # Try uv first for faster package management if (Test-Uv) { Write-Info "Using uv for faster package management..." if (Test-Path $VENV_PATH) { if ($Force) { Write-Warning "Removing existing environment..." Remove-Item -Recurse -Force $VENV_PATH } else { Write-Success "Virtual environment already exists" $pythonPath = "$VENV_PATH\Scripts\python.exe" if (Test-Path $pythonPath) { return Get-AbsolutePath $pythonPath } } } try { Write-Info "Creating virtual environment with uv..." uv venv $VENV_PATH --python 3.12 if ($LASTEXITCODE -eq 0) { Write-Success "Environment created with uv" return Get-AbsolutePath "$VENV_PATH\Scripts\python.exe" } } catch { Write-Warning "uv failed, falling back to venv" } } # Fallback to standard venv $pythonCmd = Find-Python if (!$pythonCmd) { throw "Python 3.10+ not found" } if (Test-Path $VENV_PATH) { if ($Force) { Write-Warning "Removing existing environment..." try { # Stop any Python processes that might be using the venv Get-Process python* -ErrorAction SilentlyContinue | Where-Object { $_.Path -like "*$VENV_PATH*" } | Stop-Process -Force -ErrorAction SilentlyContinue # Wait a moment for processes to terminate Start-Sleep -Seconds 2 # Use the robust removal function if (Remove-LockedDirectory $VENV_PATH) { Write-Success "Existing environment removed" } else { throw "Unable to remove existing environment. Please restart your computer and try again." } } catch { Write-Error "Failed to remove existing environment: $_" Write-Host "" Write-Host "Try these solutions:" -ForegroundColor Yellow Write-Host "1. Close all terminals and VS Code instances" -ForegroundColor White Write-Host "2. Run: Get-Process python* | Stop-Process -Force" -ForegroundColor White Write-Host "3. Manually delete: $VENV_PATH" -ForegroundColor White Write-Host "4. Then run the script again" -ForegroundColor White exit 1 } } else { Write-Success "Virtual environment already exists" return Get-AbsolutePath "$VENV_PATH\Scripts\python.exe" } } Write-Info "Creating virtual environment with $pythonCmd..." if ($pythonCmd.StartsWith("py ")) { Invoke-Expression "$pythonCmd -m venv $VENV_PATH" } else { & $pythonCmd -m venv $VENV_PATH } if ($LASTEXITCODE -ne 0) { throw "Failed to create virtual environment" } Write-Success "Virtual environment created" return Get-AbsolutePath "$VENV_PATH\Scripts\python.exe" } # Setup virtual environment (legacy function for compatibility) function Initialize-VirtualEnvironment { Write-Step "Setting up Python Virtual Environment" if (!$SkipVenv -and (Test-Path $VENV_PATH)) { if ($Force) { Write-Warning "Removing existing virtual environment..." try { # Stop any Python processes that might be using the venv Get-Process python* -ErrorAction SilentlyContinue | Where-Object { $_.Path -like "*$VENV_PATH*" } | Stop-Process -Force -ErrorAction SilentlyContinue # Wait a moment for processes to terminate Start-Sleep -Seconds 2 # Use the robust removal function if (Remove-LockedDirectory $VENV_PATH) { Write-Success "Existing environment removed" } else { throw "Unable to remove existing environment. Please restart your computer and try again." } } catch { Write-Error "Failed to remove existing environment: $_" Write-Host "" Write-Host "Try these solutions:" -ForegroundColor Yellow Write-Host "1. Close all terminals and VS Code instances" -ForegroundColor White Write-Host "2. Run: Get-Process python* | Stop-Process -Force" -ForegroundColor White Write-Host "3. Manually delete: $VENV_PATH" -ForegroundColor White Write-Host "4. Then run the script again" -ForegroundColor White exit 1 } } else { Write-Success "Virtual environment already exists" return } } if ($SkipVenv) { Write-Warning "Skipping virtual environment setup" return } $pythonCmd = Find-Python if (!$pythonCmd) { Write-Error "Python 3.10+ not found. Please install Python from https://python.org" exit 1 } Write-Info "Using Python: $pythonCmd" Write-Info "Creating virtual environment..." try { if ($pythonCmd.StartsWith("py ")) { Invoke-Expression "$pythonCmd -m venv $VENV_PATH" } else { & $pythonCmd -m venv $VENV_PATH } if ($LASTEXITCODE -ne 0) { throw "Failed to create virtual environment" } Write-Success "Virtual environment created" } catch { Write-Error "Failed to create virtual environment: $_" exit 1 } } # Install dependencies function - Simplified uv-first approach function Install-Dependencies { param( [Parameter(Mandatory = $true)] [string]$PythonPath, [switch]$InstallDevDependencies = $false ) Write-Step "Installing Dependencies" # Build requirements files list $requirementsFiles = @("requirements.txt") if ($InstallDevDependencies) { if (Test-Path "requirements-dev.txt") { $requirementsFiles += "requirements-dev.txt" Write-Info "Including development dependencies from requirements-dev.txt" } else { Write-Warning "Development dependencies requested but requirements-dev.txt not found" } } # Try uv first for faster package management $useUv = Test-Uv if ($useUv) { Write-Info "Installing dependencies with uv (fast)..." try { foreach ($file in $requirementsFiles) { Write-Info "Installing from $file with uv..." $uv = (Get-Command uv -ErrorAction Stop).Source $arguments = @('pip', 'install', '-r', $file, '--python', $PythonPath) $proc = Start-Process -FilePath $uv -ArgumentList $arguments -NoNewWindow -Wait -PassThru if ($proc.ExitCode -ne 0) { throw "uv failed to install $file with exit code $($proc.ExitCode)" } } Write-Success "Dependencies installed successfully with uv" return } catch { Write-Warning "uv installation failed: $_. Falling back to pip" $useUv = $false } } # Fallback to pip Write-Info "Installing dependencies with pip..." $pipCmd = Join-Path (Split-Path $PythonPath -Parent) "pip.exe" try { # Upgrade pip first & $pipCmd install --upgrade pip | Out-Null } catch { Write-Warning "Could not upgrade pip, continuing..." } try { foreach ($file in $requirementsFiles) { Write-Info "Installing from $file with pip..." & $pipCmd install -r $file if ($LASTEXITCODE -ne 0) { throw "pip failed to install $file" } } Write-Success "Dependencies installed successfully with pip" } catch { Write-Error "Failed to install dependencies with pip: $_" exit 1 } } # ---------------------------------------------------------------------------- # Docker Functions # ============================================================================ # Test Docker availability and requirements function Test-DockerRequirements { Write-Step "Checking Docker Requirements" if (!(Test-Command "docker")) { Write-Error "Docker not found. Please install Docker Desktop from https://docker.com" return $false } try { $null = docker version 2>$null Write-Success "Docker is installed and running" } catch { Write-Error "Docker is installed but not running. Please start Docker Desktop." return $false } if (!(Test-Command "docker-compose")) { Write-Warning "docker-compose not found. Trying docker compose..." try { $null = docker compose version 2>$null Write-Success "Docker Compose (v2) is available" return $true } catch { Write-Error "Docker Compose not found. Please install Docker Compose." return $false } } else { Write-Success "Docker Compose is available" return $true } } # Build Docker image function Build-DockerImage { param([switch]$Force = $false) Write-Step "Building Docker Image" # Check if image exists try { $imageExists = docker images --format "{{.Repository}}:{{.Tag}}" | Where-Object { $_ -eq "pal-mcp-server:latest" } if ($imageExists -and !$Force) { Write-Success "Docker image already exists. Use -Force to rebuild." return $true } } catch { # Continue if command fails } if ($Force -and $imageExists) { Write-Info "Forcing rebuild of Docker image..." try { docker rmi pal-mcp-server:latest 2>$null } catch { Write-Warning "Could not remove existing image, continuing..." } } Write-Info "Building Docker image from Dockerfile..." try { $buildArgs = @() if ($Dev) { # For development builds, we could add specific build args Write-Info "Building with development support..." } docker build -t pal-mcp-server:latest . if ($LASTEXITCODE -ne 0) { throw "Docker build failed" } Write-Success "Docker image built successfully" return $true } catch { Write-Error "Failed to build Docker image: $_" return $false } } # Prepare Docker environment file function Initialize-DockerEnvironment { Write-Step "Preparing Docker Environment" # Ensure .env file exists if (!(Test-Path ".env")) { Write-Warning "No .env file found. Creating default .env file..." $defaultEnv = @" # API Keys - Replace with your actual keys GEMINI_API_KEY=your_gemini_api_key_here GOOGLE_API_KEY=your_google_api_key_here OPENAI_API_KEY=your_openai_api_key_here ANTHROPIC_API_KEY=your_anthropic_api_key_here XAI_API_KEY=your_xai_api_key_here DIAL_API_KEY=your_dial_api_key_here DIAL_API_HOST=your_dial_api_host_here DIAL_API_VERSION=your_dial_api_version_here OPENROUTER_API_KEY=your_openrouter_api_key_here CUSTOM_API_URL=your_custom_api_url_here CUSTOM_API_KEY=your_custom_api_key_here CUSTOM_MODEL_NAME=your_custom_model_name_here # Server Configuration DEFAULT_MODEL=auto LOG_LEVEL=INFO LOG_MAX_SIZE=10MB LOG_BACKUP_COUNT=5 DEFAULT_THINKING_MODE_THINKDEEP=high # Optional Advanced Settings #DISABLED_TOOLS= #MAX_MCP_OUTPUT_TOKENS= #TZ=UTC "@ $defaultEnv | Out-File -FilePath ".env" -Encoding UTF8 Write-Success "Default .env file created" Write-Warning "Please edit .env file with your actual API keys" } else { Write-Success ".env file exists" } # Create logs directory for volume mount Initialize-Logging return $true } # Start Docker services function Start-DockerServices { param([switch]$Follow = $false) Write-Step "Starting Docker Services" # Check if docker-compose.yml exists if (!(Test-Path "docker-compose.yml")) { Write-Error "docker-compose.yml not found in current directory" return $false } try { # Stop any existing services Write-Info "Stopping any existing services..." if (Test-Command "docker-compose") { docker-compose down 2>$null } else { docker compose down 2>$null } # Start services Write-Info "Starting PAL MCP Server with Docker Compose..." if (Test-Command "docker-compose") { if ($Follow) { docker-compose up --build } else { docker-compose up -d --build } } else { if ($Follow) { docker compose up --build } else { docker compose up -d --build } } if ($LASTEXITCODE -ne 0) { throw "Failed to start Docker services" } if (!$Follow) { Write-Success "Docker services started successfully" Write-Info "Container name: pal-mcp-server" Write-Host "" Write-Host "To view logs: " -NoNewline Write-Host "docker logs -f pal-mcp-server" -ForegroundColor Yellow Write-Host "To stop: " -NoNewline Write-Host "docker-compose down" -ForegroundColor Yellow } return $true } catch { Write-Error "Failed to start Docker services: $_" return $false } } # Get Docker container status function Get-DockerStatus { try { $containerStatus = docker ps --filter "name=pal-mcp-server" --format "{{.Status}}" if ($containerStatus) { Write-Success "Container status: $containerStatus" return $true } else { Write-Warning "Container not running" return $false } } catch { Write-Warning "Could not get container status: $_" return $false } } # ============================================================================ # End Docker Functions # ============================================================================ # Setup logging directory function Initialize-Logging { Write-Step "Setting up Logging" if (!(Test-Path $LOG_DIR)) { New-Item -ItemType Directory -Path $LOG_DIR -Force | Out-Null Write-Success "Logs directory created" } else { Write-Success "Logs directory already exists" } } # Check Docker function Test-Docker { Write-Step "Checking Docker Setup" if ($SkipDocker) { Write-Warning "Skipping Docker checks" return } if (Test-Command "docker") { try { $null = docker version 2>$null Write-Success "Docker is installed and running" if (Test-Command "docker-compose") { Write-Success "Docker Compose is available" } else { Write-Warning "Docker Compose not found. Install Docker Desktop for Windows." } } catch { Write-Warning "Docker is installed but not running. Please start Docker Desktop." } } else { Write-Warning "Docker not found. Install Docker Desktop from https://docker.com" } } # ---------------------------------------------------------------------------- # MCP Client Configuration System # ---------------------------------------------------------------------------- # Centralized MCP client definitions $script:McpClientDefinitions = @( @{ Name = "Claude Desktop" DetectionPath = "$env:APPDATA\Claude\claude_desktop_config.json" DetectionType = "Path" ConfigPath = "$env:APPDATA\Claude\claude_desktop_config.json" ConfigJsonPath = "mcpServers.pal" NeedsConfigDir = $true }, @{ Name = "VSCode" DetectionCommand = "code" DetectionType = "Command" ConfigPath = "$env:APPDATA\Code\User\settings.json" ConfigJsonPath = "mcp.servers.pal" IsVSCode = $true }, @{ Name = "VSCode Insiders" DetectionCommand = "code-insiders" DetectionType = "Command" ConfigPath = "$env:APPDATA\Code - Insiders\User\mcp.json" ConfigJsonPath = "servers.pal" IsVSCodeInsiders = $true }, @{ Name = "Cursor" DetectionCommand = "cursor" DetectionType = "Command" ConfigPath = "$env:USERPROFILE\.cursor\mcp.json" ConfigJsonPath = "mcpServers.pal" }, @{ Name = "Windsurf" DetectionPath = "$env:USERPROFILE\.codeium\windsurf" DetectionType = "Path" ConfigPath = "$env:USERPROFILE\.codeium\windsurf\mcp_config.json" ConfigJsonPath = "mcpServers.pal" }, @{ Name = "Trae" DetectionPath = "$env:APPDATA\Trae" DetectionType = "Path" ConfigPath = "$env:APPDATA\Trae\User\mcp.json" ConfigJsonPath = "mcpServers.pal" } ) # Docker MCP configuration template (legacy, kept for backward compatibility) $script:DockerMcpConfig = @{ command = "docker" args = @("exec", "-i", "pal-mcp-server", "python", "server.py") type = "stdio" } # Generate Docker MCP configuration using docker run (recommended for all clients) function Get-DockerMcpConfigRun { param([string]$ServerPath) $scriptDir = Split-Path $ServerPath -Parent $envFile = Join-Path $scriptDir ".env" return @{ command = "docker" args = @("run", "--rm", "-i", "--env-file", $envFile, "pal-mcp-server:latest", "python", "server.py") type = "stdio" } } # Generate Python MCP configuration function Get-PythonMcpConfig { param([string]$PythonPath, [string]$ServerPath) return @{ command = $PythonPath args = @($ServerPath) type = "stdio" } } # Check if client uses mcp.json format with servers structure function Test-McpJsonFormat { param([hashtable]$Client) $configFileName = Split-Path $Client.ConfigPath -Leaf return $configFileName -eq "mcp.json" } # Check if client uses the new VS Code Insiders format (servers instead of mcpServers) function Test-VSCodeInsidersFormat { param([hashtable]$Client) return $Client.IsVSCodeInsiders -eq $true -and $Client.ConfigJsonPath -eq "servers.pal" } # Analyze existing MCP configuration to determine type (Python or Docker) function Get-ExistingMcpConfigType { param( [Parameter(Mandatory = $true)] [hashtable]$Client, [Parameter(Mandatory = $true)] [string]$ConfigPath ) if (!(Test-Path $ConfigPath)) { return @{ Exists = $false Type = "None" Details = "No configuration found" } } try { $content = Get-Content $ConfigPath -Raw | ConvertFrom-Json -ErrorAction SilentlyContinue if (!$content) { return @{ Exists = $false Type = "None" Details = "Invalid JSON configuration" } } # Navigate to pal configuration $pathParts = $Client.ConfigJsonPath.Split('.') $palKey = $pathParts[-1] $parentPath = $pathParts[0..($pathParts.Length - 2)] $targetObject = $content foreach ($key in $parentPath) { if (!$targetObject.PSObject.Properties[$key]) { return @{ Exists = $false Type = "None" Details = "Configuration structure not found" } } $targetObject = $targetObject.$key } if (!$targetObject.PSObject.Properties[$palKey]) { return @{ Exists = $false Type = "None" Details = "PAL configuration not found" } } $palConfig = $targetObject.$palKey # Analyze configuration type if ($palConfig.command -eq "docker") { $dockerType = "Unknown" $details = "Docker configuration" if ($palConfig.args -and $palConfig.args.Count -gt 0) { if ($palConfig.args[0] -eq "run") { $dockerType = "Docker Run" $details = "Docker run (dedicated container)" } elseif ($palConfig.args[0] -eq "exec") { $dockerType = "Docker Exec" $details = "Docker exec (existing container)" } else { $details = "Docker ($($palConfig.args[0]))" } } return @{ Exists = $true Type = "Docker" SubType = $dockerType Details = $details Command = $palConfig.command Args = $palConfig.args } } elseif ($palConfig.command -and $palConfig.command.EndsWith("python.exe")) { $pythonType = "Python" $details = "Python virtual environment" if ($palConfig.command.Contains(".pal_venv")) { $details = "Python (pal virtual environment)" } elseif ($palConfig.command.Contains("venv")) { $details = "Python (virtual environment)" } else { $details = "Python (system installation)" } return @{ Exists = $true Type = "Python" SubType = $pythonType Details = $details Command = $palConfig.command Args = $palConfig.args } } else { return @{ Exists = $true Type = "Unknown" Details = "Unknown configuration type: $($palConfig.command)" Command = $palConfig.command Args = $palConfig.args } } } catch { return @{ Exists = $false Type = "Error" Details = "Error reading configuration: $_" } } } # Generic MCP client configuration function function Configure-McpClient { param( [Parameter(Mandatory = $true)] [hashtable]$Client, [Parameter(Mandatory = $true)] [bool]$UseDocker, [string]$PythonPath = "", [string]$ServerPath = "" ) Write-Step "Checking $($Client.Name) Integration" # Client detection $detected = $false if ($Client.DetectionType -eq "Command" -and (Test-Command $Client.DetectionCommand)) { $detected = $true } elseif ($Client.DetectionType -eq "Path" -and (Test-Path ($Client.DetectionPath -as [string]))) { $detected = $true } if (!$detected) { Write-Info "$($Client.Name) not detected - skipping integration" return } Write-Info "Found $($Client.Name)" # Handle VSCode special logic for profiles $configPath = $Client.ConfigPath if ($Client.IsVSCode) { $userPath = Split-Path $configPath -Parent if (!(Test-Path $userPath)) { Write-Warning "$($Client.Name) user directory not found. Skipping." return } # Find most recent settings.json (default or profile) $settingsFiles = @() $defaultSettings = $configPath if (Test-Path $defaultSettings) { $settingsFiles += @{ Path = $defaultSettings LastModified = (Get-Item $defaultSettings).LastWriteTime } } $profilesPath = Join-Path $userPath "profiles" if (Test-Path $profilesPath) { Get-ChildItem $profilesPath -Directory | ForEach-Object { $profileSettings = Join-Path $_.FullName "settings.json" if (Test-Path $profileSettings) { $settingsFiles += @{ Path = $profileSettings LastModified = (Get-Item $profileSettings).LastWriteTime } } } } if ($settingsFiles.Count -gt 0) { $configPath = ($settingsFiles | Sort-Object LastModified -Descending | Select-Object -First 1).Path } } # Handle VSCode Insiders special logic for profiles (uses mcp.json) if ($Client.IsVSCodeInsiders) { $userPath = Split-Path $configPath -Parent if (!(Test-Path $userPath)) { Write-Warning "$($Client.Name) user directory not found. Skipping." return } # Find most recent mcp.json (default or profile) $mcpFiles = @() $defaultMcp = $configPath if (Test-Path $defaultMcp) { $mcpFiles += @{ Path = $defaultMcp LastModified = (Get-Item $defaultMcp).LastWriteTime } } $profilesPath = Join-Path $userPath "profiles" if (Test-Path $profilesPath) { Get-ChildItem $profilesPath -Directory | ForEach-Object { $profileMcp = Join-Path $_.FullName "mcp.json" if (Test-Path $profileMcp) { $mcpFiles += @{ Path = $profileMcp LastModified = (Get-Item $profileMcp).LastWriteTime } } } } if ($mcpFiles.Count -gt 0) { $configPath = ($mcpFiles | Sort-Object LastModified -Descending | Select-Object -First 1).Path } } # Check if already configured and analyze existing configuration $existingConfig = Get-ExistingMcpConfigType -Client $Client -ConfigPath $configPath $newConfigType = if ($UseDocker) { "Docker" } else { "Python" } if ($existingConfig.Exists) { Write-Info "Found existing PAL MCP configuration in $($Client.Name)" Write-Info " Current: $($existingConfig.Details)" Write-Info " New: $newConfigType configuration" if ($existingConfig.Type -eq $newConfigType) { Write-Warning "Same configuration type ($($existingConfig.Type)) already exists" $response = Read-Host "`nOverwrite existing $($existingConfig.Type) configuration? (y/N)" } else { Write-Warning "Different configuration type detected" Write-Info " Replacing: $($existingConfig.Type) → $newConfigType" $response = Read-Host "`nReplace $($existingConfig.Type) with $newConfigType configuration? (y/N)" } if ($response -ne 'y' -and $response -ne 'Y') { Write-Info "Keeping existing configuration in $($Client.Name)" return } Write-Info "Proceeding with configuration update..." } else { # User confirmation for new installation $response = Read-Host "`nConfigure PAL MCP for $($Client.Name) (mode: $newConfigType)? (y/N)" if ($response -ne 'y' -and $response -ne 'Y') { Write-Info "Skipping $($Client.Name) integration" return } } try { # Create config directory if needed $configDir = Split-Path $configPath -Parent if (!(Test-Path $configDir)) { New-Item -ItemType Directory -Path $configDir -Force | Out-Null } # Backup existing config if (Test-Path $configPath) { Manage-ConfigBackups -ConfigFilePath $configPath } # Read or create config $config = New-Object PSObject $usesMcpJsonFormat = Test-McpJsonFormat -Client $Client $usesVSCodeInsidersFormat = Test-VSCodeInsidersFormat -Client $Client if (Test-Path $configPath) { $fileContent = Get-Content $configPath -Raw if ($fileContent.Trim()) { $config = $fileContent | ConvertFrom-Json -ErrorAction SilentlyContinue } if ($null -eq $config) { $config = New-Object PSObject } } # Initialize structure for mcp.json format files if they don't exist or are empty if ($usesMcpJsonFormat) { if ($usesVSCodeInsidersFormat) { # For VS Code Insiders format: {"servers": {...}} if (!$config.PSObject.Properties["servers"]) { $config | Add-Member -MemberType NoteProperty -Name "servers" -Value (New-Object PSObject) } } else { # For other clients format: {"mcpServers": {...}} if (!$config.PSObject.Properties["mcpServers"]) { $config | Add-Member -MemberType NoteProperty -Name "mcpServers" -Value (New-Object PSObject) } } } # Initialize MCP structure for VS Code settings.json if it doesn't exist if ($Client.IsVSCode -and $Client.ConfigJsonPath.StartsWith("mcp.")) { if (!$config.PSObject.Properties["mcp"]) { $config | Add-Member -MemberType NoteProperty -Name "mcp" -Value (New-Object PSObject) } if (!$config.mcp.PSObject.Properties["servers"]) { $config.mcp | Add-Member -MemberType NoteProperty -Name "servers" -Value (New-Object PSObject) } } # Generate server config $serverConfig = if ($UseDocker) { # Use docker run for all clients (more reliable than docker exec) Get-DockerMcpConfigRun $ServerPath } else { Get-PythonMcpConfig $PythonPath $ServerPath } # Navigate and set configuration $pathParts = $Client.ConfigJsonPath.Split('.') $palKey = $pathParts[-1] $parentPath = $pathParts[0..($pathParts.Length - 2)] $targetObject = $config foreach ($key in $parentPath) { if (!$targetObject.PSObject.Properties[$key]) { $targetObject | Add-Member -MemberType NoteProperty -Name $key -Value (New-Object PSObject) } $targetObject = $targetObject.$key } # Remove legacy zen entries to avoid duplicate or broken MCP servers $legacyRemoved = Remove-LegacyServerKeys $targetObject if ($legacyRemoved) { Write-Info "Removed legacy MCP entries (zen → pal)" } $targetObject | Add-Member -MemberType NoteProperty -Name $palKey -Value $serverConfig -Force # Write config $config | ConvertTo-Json -Depth 10 | Out-File $configPath -Encoding UTF8 Write-Success "Successfully configured $($Client.Name)" Write-Host " Config: $configPath" -ForegroundColor Gray Write-Host " Restart $($Client.Name) to use the new MCP server" -ForegroundColor Gray } catch { Write-Error "Failed to update $($Client.Name) configuration: $_" } } # Main MCP client configuration orchestrator function Invoke-McpClientConfiguration { param( [Parameter(Mandatory = $true)] [bool]$UseDocker, [string]$PythonPath = "", [string]$ServerPath = "" ) Write-Step "Checking Client Integrations" # Configure GUI clients foreach ($client in $script:McpClientDefinitions) { Configure-McpClient -Client $client -UseDocker $UseDocker -PythonPath $PythonPath -ServerPath $ServerPath } # Handle CLI tools separately (they don't follow JSON config pattern) if (!$UseDocker) { Test-ClaudeCliIntegration $PythonPath $ServerPath Test-GeminiCliIntegration (Split-Path $ServerPath -Parent) Test-QwenCliIntegration $PythonPath $ServerPath } } # Keep existing CLI integration functions function Test-ClaudeCliIntegration { param([string]$PythonPath, [string]$ServerPath) if (!(Test-Command "claude")) { return } Write-Info "Claude CLI detected - checking configuration..." foreach ($legacy in $script:LegacyServerNames) { try { claude mcp remove -s user $legacy 2>$null | Out-Null } catch {} } try { $claudeConfig = claude mcp list 2>$null if ($claudeConfig -match "pal") { Write-Success "Claude CLI already configured for pal server" } else { Write-Info "To add pal server to Claude CLI, run:" Write-Host " claude mcp add -s user pal $PythonPath $ServerPath" -ForegroundColor Cyan } } catch { Write-Info "To configure Claude CLI manually, run:" Write-Host " claude mcp add -s user pal $PythonPath $ServerPath" -ForegroundColor Cyan } } function Test-GeminiCliIntegration { param([string]$ScriptDir) $palWrapper = Join-Path $ScriptDir "pal-mcp-server.cmd" # Check if Gemini settings file exists (Windows path) $geminiConfig = "$env:USERPROFILE\.gemini\settings.json" if (!(Test-Path $geminiConfig)) { return } # Load existing config $config = @{} $configContent = Get-Content $geminiConfig -Raw -ErrorAction SilentlyContinue if ($configContent) { try { $config = $configContent | ConvertFrom-Json -ErrorAction Stop } catch { $config = @{} } } if ($null -eq $config -or $config -isnot [System.Collections.IDictionary]) { $config = @{} } if (-not $config.mcpServers -or $config.mcpServers -isnot [System.Collections.IDictionary]) { $config.mcpServers = [ordered]@{} } $legacyRemoved = Remove-LegacyServerKeys $config.mcpServers $palConfig = $config.mcpServers.pal $needsWrite = $legacyRemoved if ($palConfig) { if ($palConfig.command -ne $palWrapper) { $palConfig.command = $palWrapper $needsWrite = $true } if (!(Test-Path $palWrapper)) { Write-Info "Creating wrapper script for Gemini CLI..." @" @echo off cd /d "%~dp0" if exist ".pal_venv\Scripts\python.exe" ( .pal_venv\Scripts\python.exe server.py %* ) else ( python server.py %* ) "@ | Out-File -FilePath $palWrapper -Encoding ASCII Write-Success "Created pal-mcp-server.cmd wrapper script" } if ($needsWrite) { Manage-ConfigBackups -ConfigFilePath $geminiConfig | Out-Null $config | ConvertTo-Json -Depth 10 | Out-File $geminiConfig -Encoding UTF8 Write-Success "Updated Gemini CLI configuration (cleaned legacy entries)" Write-Host " Config: $geminiConfig" -ForegroundColor Gray Write-Host " Restart Gemini CLI to use PAL MCP Server" -ForegroundColor Gray } return } # Ask user if they want to add PAL to Gemini CLI Write-Host "" $response = Read-Host "Configure PAL for Gemini CLI? (y/N)" if ($response -ne 'y' -and $response -ne 'Y') { Write-Info "Skipping Gemini CLI integration" return } # Ensure wrapper script exists if (!(Test-Path $palWrapper)) { Write-Info "Creating wrapper script for Gemini CLI..." @" @echo off cd /d "%~dp0" if exist ".pal_venv\Scripts\python.exe" ( .pal_venv\Scripts\python.exe server.py %* ) else ( python server.py %* ) "@ | Out-File -FilePath $palWrapper -Encoding ASCII Write-Success "Created pal-mcp-server.cmd wrapper script" } # Update Gemini settings Write-Info "Updating Gemini CLI configuration..." try { # Create backup with retention management $backupPath = Manage-ConfigBackups $geminiConfig # Ensure mcpServers exists if (-not $config.mcpServers -or $config.mcpServers -isnot [System.Collections.IDictionary]) { $config.mcpServers = [ordered]@{} } # Add pal server $palConfig = @{ command = $palWrapper } $config.mcpServers | Add-Member -MemberType NoteProperty -Name "pal" -Value $palConfig -Force # Write updated config $config | ConvertTo-Json -Depth 10 | Out-File $geminiConfig -Encoding UTF8 Write-Success "Successfully configured Gemini CLI" Write-Host " Config: $geminiConfig" -ForegroundColor Gray Write-Host " Restart Gemini CLI to use PAL MCP Server" -ForegroundColor Gray } catch { Write-Error "Failed to update Gemini CLI config: $_" Write-Host "" Write-Host "Manual config location: $geminiConfig" Write-Host "Add this configuration:" Write-Host @" { "mcpServers": { "pal": { "command": "$palWrapper" } } } "@ -ForegroundColor Yellow } } function Show-QwenManualConfig { param( [string]$PythonPath, [string]$ServerPath, [string]$ScriptDir, [string]$ConfigPath, [System.Collections.IDictionary]$EnvironmentMap ) Write-Host "Manual config location: $ConfigPath" -ForegroundColor Yellow Write-Host "Add or update this entry:" -ForegroundColor Yellow if ($EnvironmentMap -and $EnvironmentMap.Count -gt 0) { $pairs = $EnvironmentMap.GetEnumerator() | ForEach-Object { $escaped = ($_.Value -replace '\\', '\\\\' -replace '"', '\\"') ' "{0}": "{1}"' -f $_.Key, $escaped } Write-Host "{" -ForegroundColor Yellow Write-Host " \"mcpServers\": {" -ForegroundColor Yellow Write-Host " \"pal\": {" -ForegroundColor Yellow Write-Host " \"command\": \"$PythonPath\"," -ForegroundColor Yellow Write-Host " \"args\": [\"$ServerPath\"]," -ForegroundColor Yellow Write-Host " \"cwd\": \"$ScriptDir\"," -ForegroundColor Yellow Write-Host " \"env\": {" -ForegroundColor Yellow Write-Host ($pairs -join "`n") -ForegroundColor Yellow Write-Host " }" -ForegroundColor Yellow Write-Host " }" -ForegroundColor Yellow Write-Host " }" -ForegroundColor Yellow Write-Host "}" -ForegroundColor Yellow } else { Write-Host "{" -ForegroundColor Yellow Write-Host " \"mcpServers\": {" -ForegroundColor Yellow Write-Host " \"pal\": {" -ForegroundColor Yellow Write-Host " \"command\": \"$PythonPath\"," -ForegroundColor Yellow Write-Host " \"args\": [\"$ServerPath\"]," -ForegroundColor Yellow Write-Host " \"cwd\": \"$ScriptDir\"" -ForegroundColor Yellow Write-Host " }" -ForegroundColor Yellow Write-Host " }" -ForegroundColor Yellow Write-Host "}" -ForegroundColor Yellow } } function Test-QwenCliIntegration { param([string]$PythonPath, [string]$ServerPath) if (!(Test-Command "qwen")) { return } Write-Info "Qwen CLI detected - checking configuration..." $configPath = Join-Path $env:USERPROFILE ".qwen\settings.json" $configDir = Split-Path $configPath -Parent $scriptDir = Split-Path $ServerPath -Parent $configStatus = "missing" $legacyRemoved = $false $skipPrompt = $false $config = @{} if (Test-Path $configPath) { try { Add-Type -AssemblyName System.Web.Extensions -ErrorAction SilentlyContinue $serializer = New-Object System.Web.Script.Serialization.JavaScriptSerializer $serializer.MaxJsonLength = 67108864 $rawJson = Get-Content $configPath -Raw $config = $serializer.DeserializeObject($rawJson) if (-not ($config -is [System.Collections.IDictionary])) { $config = @{} } if ($config.ContainsKey('mcpServers') -and $config['mcpServers'] -is [System.Collections.IDictionary]) { $servers = $config['mcpServers'] $legacyRemoved = (Remove-LegacyServerKeys $servers) -or $legacyRemoved if ($servers.Contains('pal') -and $servers['pal'] -is [System.Collections.IDictionary]) { $palConfig = $servers['pal'] $commandMatches = ($palConfig['command'] -eq $PythonPath) $argsValue = $palConfig['args'] $argsList = @() if ($argsValue -is [System.Collections.IEnumerable] -and $argsValue -isnot [string]) { $argsList = @($argsValue) } elseif ($null -ne $argsValue) { $argsList = @($argsValue) } $argsMatches = ($argsList.Count -eq 1 -and $argsList[0] -eq $ServerPath) $cwdValue = $null if ($palConfig.Contains('cwd')) { $cwdValue = $palConfig['cwd'] } $cwdMatches = ([string]::IsNullOrEmpty($cwdValue) -or $cwdValue -eq $scriptDir) if ($commandMatches -and $argsMatches -and $cwdMatches) { $configStatus = $legacyRemoved ? "cleanup" : "match" } else { $configStatus = "mismatch" Write-Warning "Existing Qwen CLI configuration differs from the current setup." } } } } catch { $configStatus = "invalid" Write-Warning "Unable to parse Qwen CLI settings at $configPath ($_)." $config = @{} } } $envMap = [ordered]@{} if (Test-Path ".env") { foreach ($line in Get-Content ".env") { $trimmed = $line.Trim() if ([string]::IsNullOrWhiteSpace($trimmed) -or $trimmed.StartsWith('#')) { continue } if ($line -match '^\s*([^=]+)=(.*)$') { $key = $matches[1].Trim() $value = $matches[2] $value = ($value -replace '\s+#.*$', '').Trim() if ($value.StartsWith('"') -and $value.EndsWith('"')) { $value = $value.Substring(1, $value.Length - 2) } if ([string]::IsNullOrWhiteSpace($value)) { $value = [Environment]::GetEnvironmentVariable($key, "Process") } if (![string]::IsNullOrWhiteSpace($value) -and $value -notmatch '^your_.*_here$') { $envMap[$key] = $value } } } } $extraKeys = @( "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "DIAL_API_KEY", "OPENROUTER_API_KEY", "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_API_VERSION", "AZURE_OPENAI_ALLOWED_MODELS", "AZURE_MODELS_CONFIG_PATH", "CUSTOM_API_URL", "CUSTOM_API_KEY", "CUSTOM_MODEL_NAME", "DEFAULT_MODEL", "GOOGLE_ALLOWED_MODELS", "OPENAI_ALLOWED_MODELS", "OPENROUTER_ALLOWED_MODELS", "XAI_ALLOWED_MODELS", "DEFAULT_THINKING_MODE_THINKDEEP", "DISABLED_TOOLS", "CONVERSATION_TIMEOUT_HOURS", "MAX_CONVERSATION_TURNS", "LOG_LEVEL", "PAL_MCP_FORCE_ENV_OVERRIDE" ) foreach ($key in $extraKeys) { if (-not $envMap.Contains($key)) { $value = [Environment]::GetEnvironmentVariable($key, "Process") if (![string]::IsNullOrWhiteSpace($value) -and $value -notmatch '^your_.*_here$') { $envMap[$key] = $value } } } if ($configStatus -eq "match") { Write-Success "Qwen CLI already configured for pal server" return } if ($configStatus -eq "cleanup") { Write-Info "Removing legacy Qwen MCP entries from previous zen configuration..." $skipPrompt = $true } $prompt = "Configure PAL for Qwen CLI? (y/N)" if ($configStatus -eq "cleanup") { $prompt = "Remove legacy Qwen MCP entries and refresh configuration? (Y/n)" } elseif ($configStatus -eq "mismatch" -or $configStatus -eq "invalid") { $prompt = "Update Qwen CLI pal configuration? (y/N)" } if (-not $skipPrompt) { $response = Read-Host $prompt if ($response -ne 'y' -and $response -ne 'Y') { Write-Info "Skipping Qwen CLI integration" Show-QwenManualConfig $PythonPath $ServerPath $scriptDir $configPath $envMap return } } if (!(Test-Path $configDir)) { New-Item -ItemType Directory -Path $configDir -Force | Out-Null } if (Test-Path $configPath -and $configStatus -ne "missing") { Manage-ConfigBackups $configPath | Out-Null } try { if (-not ($config -is [System.Collections.IDictionary])) { $config = @{} } if (-not $config.ContainsKey('mcpServers') -or $config['mcpServers'] -isnot [System.Collections.IDictionary]) { $config['mcpServers'] = @{} } $palConfig = [ordered]@{ command = $PythonPath args = @($ServerPath) cwd = $scriptDir } if ($envMap.Count -gt 0) { $palConfig['env'] = $envMap } $config['mcpServers']['pal'] = $palConfig $json = ($config | ConvertTo-Json -Depth 20) Set-Content -Path $configPath -Value $json -Encoding UTF8 Write-Success "Successfully configured Qwen CLI" Write-Host " Config: $configPath" -ForegroundColor Gray Write-Host " Restart Qwen CLI to use PAL MCP Server" -ForegroundColor Gray } catch { Write-Error "Failed to update Qwen CLI configuration: $_" Show-QwenManualConfig $PythonPath $ServerPath $scriptDir $configPath $envMap } } # ---------------------------------------------------------------------------- # End MCP Client Configuration System # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- # User Interface Functions # ---------------------------------------------------------------------------- # Show script help function Show-Help { Write-Host @" PAL MCP Server - Setup and Launch Script USAGE: .\run-server.ps1 [OPTIONS] OPTIONS: -Help Show this help message -Version Show version information -Follow Follow server logs in real time -Config Show configuration instructions for MCP clients -ClearCache Clear Python cache files and exit -Force Force recreation of Python virtual environment -Dev Install development dependencies from requirements-dev.txt -Docker Use Docker instead of Python virtual environment -SkipVenv Skip Python virtual environment creation -SkipDocker Skip Docker checks and cleanup EXAMPLES: .\run-server.ps1 # Normal startup .\run-server.ps1 -Follow # Start and follow logs .\run-server.ps1 -Config # Show configuration help .\run-server.ps1 -Dev # Include development dependencies .\run-server.ps1 -Docker # Use Docker deployment .\run-server.ps1 -Docker -Follow # Docker with log following For more information, visit: https://github.com/BeehiveInnovations/pal-mcp-server "@ -ForegroundColor White } # Show version information function Show-Version { $version = Get-Version Write-Host "PAL MCP Server version: $version" -ForegroundColor Green Write-Host "PowerShell Setup Script for Windows" -ForegroundColor Cyan Write-Host "Author: GiGiDKR (https://github.com/GiGiDKR)" -ForegroundColor Gray Write-Host "Project: BeehiveInnovations/pal-mcp-server" -ForegroundColor Gray } # Show configuration instructions function Show-ConfigInstructions { param( [string]$PythonPath = "", [string]$ServerPath = "", [switch]$UseDocker = $false ) Write-Step "Configuration Instructions" if ($UseDocker) { Write-Host "Docker Configuration:" -ForegroundColor Yellow Write-Host "The MCP clients have been configured to use Docker containers." -ForegroundColor White Write-Host "Make sure the Docker container is running with: docker-compose up -d" -ForegroundColor Cyan Write-Host "" } else { Write-Host "Python Virtual Environment Configuration:" -ForegroundColor Yellow Write-Host "Python Path: $PythonPath" -ForegroundColor Cyan Write-Host "Server Path: $ServerPath" -ForegroundColor Cyan Write-Host "" } Write-Host "Supported MCP Clients:" -ForegroundColor Green Write-Host "✓ Claude Desktop" -ForegroundColor White Write-Host "✓ Claude CLI" -ForegroundColor White Write-Host "✓ VSCode (with MCP extension)" -ForegroundColor White Write-Host "✓ VSCode Insiders" -ForegroundColor White Write-Host "✓ Cursor" -ForegroundColor White Write-Host "✓ Windsurf" -ForegroundColor White Write-Host "✓ Trae" -ForegroundColor White Write-Host "✓ Gemini CLI" -ForegroundColor White Write-Host "✓ Qwen CLI" -ForegroundColor White Write-Host "" Write-Host "The script automatically detects and configures compatible clients." -ForegroundColor Gray Write-Host "Restart your MCP clients after configuration to use the PAL MCP Server." -ForegroundColor Yellow } # Show setup instructions function Show-SetupInstructions { param( [string]$PythonPath = "", [string]$ServerPath = "", [switch]$UseDocker = $false ) Write-Step "Setup Complete" if ($UseDocker) { Write-Success "PAL MCP Server is configured for Docker deployment" Write-Host "Docker command: docker exec -i pal-mcp-server python server.py" -ForegroundColor Cyan } else { Write-Success "PAL MCP Server is configured for Python virtual environment" Write-Host "Python: $PythonPath" -ForegroundColor Cyan Write-Host "Server: $ServerPath" -ForegroundColor Cyan } Write-Host "" Write-Host "MCP clients will automatically connect to the server." -ForegroundColor Green Write-Host "For manual configuration, use the paths shown above." -ForegroundColor Gray } # Start the server function Start-Server { Write-Step "Starting PAL MCP Server" $pythonPath = "$VENV_PATH\Scripts\python.exe" if (!(Test-Path $pythonPath)) { Write-Error "Python virtual environment not found. Please run setup first." return } $serverPath = "server.py" if (!(Test-Path $serverPath)) { Write-Error "Server script not found: $serverPath" return } try { Write-Info "Launching server..." & $pythonPath $serverPath } catch { Write-Error "Failed to start server: $_" } } # Follow server logs function Follow-Logs { Write-Step "Following Server Logs" $logPath = Join-Path $LOG_DIR $LOG_FILE if (!(Test-Path $logPath)) { Write-Warning "Log file not found: $logPath" Write-Info "Starting server to generate logs..." Start-Server return } try { Write-Info "Following logs at: $logPath" Write-Host "Press Ctrl+C to stop following logs" Write-Host "" Get-Content $logPath -Wait } catch { Write-Error "Failed to follow logs: $_" } } # ---------------------------------------------------------------------------- # Environment File Management # ---------------------------------------------------------------------------- # Initialize .env file if it doesn't exist function Initialize-EnvFile { Write-Step "Setting up Environment File" if (!(Test-Path ".env")) { Write-Info "Creating default .env file..." @" # API Keys - Replace with your actual keys GEMINI_API_KEY=your_gemini_api_key_here GOOGLE_API_KEY=your_google_api_key_here OPENAI_API_KEY=your_openai_api_key_here ANTHROPIC_API_KEY=your_anthropic_api_key_here XAI_API_KEY=your_xai_api_key_here DIAL_API_KEY=your_dial_api_key_here DIAL_API_HOST=your_dial_api_host_here DIAL_API_VERSION=your_dial_api_version_here OPENROUTER_API_KEY=your_openrouter_api_key_here CUSTOM_API_URL=your_custom_api_url_here CUSTOM_API_KEY=your_custom_api_key_here CUSTOM_MODEL_NAME=your_custom_model_name_here # Server Configuration DEFAULT_MODEL=auto LOG_LEVEL=INFO LOG_MAX_SIZE=10MB LOG_BACKUP_COUNT=5 DEFAULT_THINKING_MODE_THINKDEEP=high # Optional Advanced Settings #DISABLED_TOOLS= #MAX_MCP_OUTPUT_TOKENS= #TZ=UTC "@ | Out-File -FilePath ".env" -Encoding UTF8 Write-Success "Default .env file created" Write-Warning "Please edit .env file with your actual API keys" } else { Write-Success ".env file already exists" } } # Import environment variables from .env file function Import-EnvFile { if (!(Test-Path ".env")) { Write-Warning "No .env file found" return } try { $envContent = Get-Content ".env" -ErrorAction Stop foreach ($line in $envContent) { if ($line -match '^([^#][^=]*?)=(.*)$') { $key = $matches[1].Trim() $value = $matches[2].Trim() -replace '^["'']|["'']$', '' # Set environment variable for the current session [Environment]::SetEnvironmentVariable($key, $value, "Process") } } Write-Success "Environment variables loaded from .env file" } catch { Write-Warning "Could not load .env file: $_" } } # ---------------------------------------------------------------------------- # Workflow Functions # ---------------------------------------------------------------------------- # Docker deployment workflow function Invoke-DockerWorkflow { Write-Step "Starting Docker Workflow" Write-Host "PAL MCP Server" -ForegroundColor Green Write-Host "=================" -ForegroundColor Cyan $version = Get-Version Write-Host "Version: $version" Write-Host "Mode: Docker Container" -ForegroundColor Yellow Write-Host "" # Docker setup and validation if (!(Test-DockerRequirements)) { exit 1 } if (!(Initialize-DockerEnvironment)) { exit 1 } Import-EnvFile Test-ApiKeys if (!(Build-DockerImage -Force:$Force)) { exit 1 } # Configure MCP clients for Docker Invoke-McpClientConfiguration -UseDocker $true Show-SetupInstructions -UseDocker # Start Docker services Write-Step "Starting PAL MCP Server" if ($Follow) { Write-Info "Starting server and following logs..." Start-DockerServices -Follow exit 0 } if (!(Start-DockerServices)) { exit 1 } Write-Host "" Write-Success "PAL MCP Server is running in Docker!" Write-Host "" Write-Host "Next steps:" -ForegroundColor Cyan Write-Host "1. Restart your MCP clients (Claude Desktop, etc.)" -ForegroundColor White Write-Host "2. The server is now ready to use" -ForegroundColor White Write-Host "" Write-Host "Useful commands:" -ForegroundColor Cyan Write-Host " View logs: " -NoNewline -ForegroundColor White Write-Host "docker logs -f pal-mcp-server" -ForegroundColor Yellow Write-Host " Stop server: " -NoNewline -ForegroundColor White Write-Host "docker-compose down" -ForegroundColor Yellow Write-Host " Restart server: " -NoNewline -ForegroundColor White Write-Host "docker-compose restart" -ForegroundColor Yellow } # Python virtual environment deployment workflow function Invoke-PythonWorkflow { Write-Step "Starting Python Virtual Environment Workflow" Write-Host "PAL MCP Server" -ForegroundColor Green Write-Host "=================" -ForegroundColor Cyan $version = Get-Version Write-Host "Version: $version" Write-Host "" if (!(Test-Path $VENV_PATH)) { Write-Info "Setting up Python environment for first time..." } # Python environment setup Cleanup-Docker Clear-PythonCache Initialize-EnvFile Import-EnvFile Test-ApiKeys try { $pythonPath = Initialize-Environment } catch { Write-Error "Failed to setup Python environment: $_" exit 1 } try { Install-Dependencies $pythonPath -InstallDevDependencies:$Dev } catch { Write-Error "Failed to install dependencies: $_" exit 1 } $serverPath = Get-AbsolutePath "server.py" # Configure MCP clients for Python Invoke-McpClientConfiguration -UseDocker $false -PythonPath $pythonPath -ServerPath $serverPath Show-SetupInstructions $pythonPath $serverPath Initialize-Logging Write-Host "" Write-Host "Logs will be written to: $(Get-AbsolutePath $LOG_DIR)\$LOG_FILE" Write-Host "" if ($Follow) { Follow-Logs } else { Write-Host "To follow logs: .\run-server.ps1 -Follow" -ForegroundColor Yellow Write-Host "To show config: .\run-server.ps1 -Config" -ForegroundColor Yellow Write-Host "To update: git pull, then run .\run-server.ps1 again" -ForegroundColor Yellow Write-Host "" Write-Host "Happy coding! 🎉" -ForegroundColor Green $response = Read-Host "`nStart the server now? (y/N)" if ($response -eq 'y' -or $response -eq 'Y') { Start-Server } } } # ---------------------------------------------------------------------------- # End Workflow Functions # ---------------------------------------------------------------------------- # ---------------------------------------------------------------------------- # Main Execution # ---------------------------------------------------------------------------- # Main execution function function Start-MainProcess { # Parse command line arguments if ($Help) { Show-Help exit 0 } if ($Version) { Show-Version exit 0 } if ($ClearCache) { Clear-PythonCache Write-Success "Cache cleared successfully" Write-Host "" Write-Host "You can now run '.\run-server.ps1' normally" exit 0 } if ($Config) { # Setup minimal environment to get paths for config display Write-Info "Setting up environment for configuration display..." Write-Host "" try { if ($Docker) { # Docker configuration mode if (!(Test-DockerRequirements)) { exit 1 } Initialize-DockerEnvironment Show-ConfigInstructions "" "" -UseDocker } else { # Python virtual environment configuration mode $pythonPath = Initialize-Environment $serverPath = Get-AbsolutePath "server.py" Show-ConfigInstructions $pythonPath $serverPath } } catch { Write-Error "Failed to setup environment for configuration: $_" exit 1 } exit 0 } # ============================================================================ # Docker Workflow # ============================================================================ if ($Docker) { Invoke-DockerWorkflow exit 0 } # ============================================================================ # Python Virtual Environment Workflow (Default) # ============================================================================ Invoke-PythonWorkflow exit 0 } # ============================================================================ # Main Script Execution # ============================================================================ # Execute main process Start-MainProcess ================================================ FILE: run-server.sh ================================================ #!/bin/bash set -euo pipefail # ============================================================================ # PAL MCP Server Setup Script # # A platform-agnostic setup script that works on macOS, Linux, and WSL. # Handles environment setup, dependency installation, and configuration. # ============================================================================ # Initialize pyenv if available (do this early) if [[ -d "$HOME/.pyenv" ]]; then export PYENV_ROOT="$HOME/.pyenv" export PATH="$PYENV_ROOT/bin:$PATH" if command -v pyenv &> /dev/null; then eval "$(pyenv init --path)" 2>/dev/null || true eval "$(pyenv init -)" 2>/dev/null || true fi fi # ---------------------------------------------------------------------------- # Constants and Configuration # ---------------------------------------------------------------------------- # Colors for output (ANSI codes work on all platforms) readonly GREEN='\033[0;32m' readonly YELLOW='\033[1;33m' readonly RED='\033[0;31m' readonly NC='\033[0m' # No Color # Configuration readonly VENV_PATH=".pal_venv" readonly DOCKER_CLEANED_FLAG=".docker_cleaned" readonly DESKTOP_CONFIG_FLAG=".desktop_configured" readonly LOG_DIR="logs" readonly LOG_FILE="mcp_server.log" readonly LEGACY_MCP_NAMES=("zen" "zen-mcp" "zen-mcp-server" "zen_mcp" "zen_mcp_server") # Determine portable arguments for sed -i (GNU vs BSD) declare -a SED_INPLACE_ARGS if sed --version >/dev/null 2>&1; then SED_INPLACE_ARGS=(-i) else SED_INPLACE_ARGS=(-i "") fi # ---------------------------------------------------------------------------- # Utility Functions # ---------------------------------------------------------------------------- # Print colored output print_success() { echo -e "${GREEN}✓${NC} $1" >&2 } print_error() { echo -e "${RED}✗${NC} $1" >&2 } print_warning() { echo -e "${YELLOW}!${NC} $1" >&2 } print_info() { echo -e "${YELLOW}$1${NC}" >&2 } # Get the script's directory (works on all platforms) get_script_dir() { cd "$(dirname "$0")" && pwd } # Extract version from config.py get_version() { grep -E '^__version__ = ' config.py 2>/dev/null | sed 's/__version__ = "\(.*\)"/\1/' || echo "unknown" } # Clear Python cache files to prevent import issues clear_python_cache() { print_info "Clearing Python cache files..." find . -name "*.pyc" -delete 2>/dev/null || true find . -name "__pycache__" -type d -exec rm -rf {} + 2>/dev/null || true print_success "Python cache cleared" } # ---------------------------------------------------------------------------- # Platform Detection Functions # ---------------------------------------------------------------------------- # Get cross-platform Python executable path from venv get_venv_python_path() { local venv_path="$1" # Convert to absolute path for consistent behavior across shell environments local abs_venv_path abs_venv_path=$(cd "$(dirname "$venv_path")" && pwd)/$(basename "$venv_path") # Check for both Unix and Windows Python executable paths if [[ -f "$abs_venv_path/bin/python" ]]; then echo "$abs_venv_path/bin/python" elif [[ -f "$abs_venv_path/Scripts/python.exe" ]]; then echo "$abs_venv_path/Scripts/python.exe" else return 1 # No Python executable found fi } # Detect the operating system detect_os() { case "$OSTYPE" in darwin*) echo "macos" ;; linux*) if grep -qi microsoft /proc/version 2>/dev/null; then echo "wsl" else echo "linux" fi ;; msys*|cygwin*|win32) echo "windows" ;; *) echo "unknown" ;; esac } # Get Claude config path based on platform get_claude_config_path() { local os_type=$(detect_os) case "$os_type" in macos) echo "$HOME/Library/Application Support/Claude/claude_desktop_config.json" ;; linux) echo "$HOME/.config/Claude/claude_desktop_config.json" ;; wsl) local win_appdata if command -v wslvar &> /dev/null; then win_appdata=$(wslvar APPDATA 2>/dev/null) fi if [[ -n "${win_appdata:-}" ]]; then echo "$(wslpath "$win_appdata")/Claude/claude_desktop_config.json" else print_warning "Could not determine Windows user path automatically. Please ensure APPDATA is set correctly or provide the full path manually." echo "/mnt/c/Users/$USER/AppData/Roaming/Claude/claude_desktop_config.json" fi ;; windows) echo "$APPDATA/Claude/claude_desktop_config.json" ;; *) echo "" ;; esac } # ---------------------------------------------------------------------------- # Docker Cleanup Functions # ---------------------------------------------------------------------------- # Clean up old Docker artifacts cleanup_docker() { # Skip if already cleaned or Docker not available [[ -f "$DOCKER_CLEANED_FLAG" ]] && return 0 if ! command -v docker &> /dev/null || ! docker info &> /dev/null 2>&1; then return 0 fi local found_artifacts=false # Define containers to remove local containers=( "gemini-mcp-server" "gemini-mcp-redis" "zen-mcp-server" "zen-mcp-redis" "zen-mcp-log-monitor" ) # Remove containers for container in "${containers[@]}"; do if docker ps -a --format "{{.Names}}" | grep -q "^${container}$" 2>/dev/null; then if [[ "$found_artifacts" == false ]]; then echo "One-time Docker cleanup..." found_artifacts=true fi echo " Removing container: $container" docker stop "$container" >/dev/null 2>&1 || true docker rm "$container" >/dev/null 2>&1 || true fi done # Remove images local images=("gemini-mcp-server:latest" "zen-mcp-server:latest") for image in "${images[@]}"; do if docker images --format "{{.Repository}}:{{.Tag}}" | grep -q "^${image}$" 2>/dev/null; then if [[ "$found_artifacts" == false ]]; then echo "One-time Docker cleanup..." found_artifacts=true fi echo " Removing image: $image" docker rmi "$image" >/dev/null 2>&1 || true fi done # Remove volumes local volumes=("redis_data" "mcp_logs") for volume in "${volumes[@]}"; do if docker volume ls --format "{{.Name}}" | grep -q "^${volume}$" 2>/dev/null; then if [[ "$found_artifacts" == false ]]; then echo "One-time Docker cleanup..." found_artifacts=true fi echo " Removing volume: $volume" docker volume rm "$volume" >/dev/null 2>&1 || true fi done if [[ "$found_artifacts" == true ]]; then print_success "Docker cleanup complete" fi touch "$DOCKER_CLEANED_FLAG" } # ---------------------------------------------------------------------------- # Python Environment Functions # ---------------------------------------------------------------------------- # Find suitable Python command find_python() { # Pyenv should already be initialized at script start, but check if .python-version exists if [[ -f ".python-version" ]] && command -v pyenv &> /dev/null; then # Ensure pyenv respects the local .python-version pyenv local &>/dev/null || true fi # Prefer Python 3.12 for best compatibility local python_cmds=("python3.12" "python3.13" "python3.11" "python3.10" "python3" "python" "py") for cmd in "${python_cmds[@]}"; do if command -v "$cmd" &> /dev/null; then local version=$($cmd --version 2>&1) if [[ $version =~ Python\ 3\.([0-9]+)\.([0-9]+) ]]; then local major_version=${BASH_REMATCH[1]} local minor_version=${BASH_REMATCH[2]} # Check minimum version (3.10) for better library compatibility if [[ $major_version -ge 10 ]]; then # Verify the command actually exists (important for pyenv) if command -v "$cmd" &> /dev/null; then echo "$cmd" print_success "Found Python: $version" # Recommend Python 3.12 if [[ $major_version -ne 12 ]]; then print_info "Note: Python 3.12 is recommended for best compatibility." fi return 0 fi fi fi fi done # No suitable Python found - check if we can use pyenv local os_type=$(detect_os) # Check for pyenv on Unix-like systems (macOS/Linux) if [[ "$os_type" == "macos" || "$os_type" == "linux" || "$os_type" == "wsl" ]]; then if command -v pyenv &> /dev/null; then # pyenv exists, check if Python 3.12 is installed if ! pyenv versions 2>/dev/null | grep -E "3\.(1[2-9]|[2-9][0-9])" >/dev/null; then echo "" echo "Python 3.10+ is required. Pyenv can install Python 3.12 locally for this project." read -p "Install Python 3.12 using pyenv? (Y/n): " -n 1 -r echo "" if [[ ! $REPLY =~ ^[Nn]$ ]]; then if install_python_with_pyenv; then # Try finding Python again if python_cmd=$(find_python); then echo "$python_cmd" return 0 fi fi fi else # Python 3.12+ is installed in pyenv but may not be active # Check if .python-version exists if [[ ! -f ".python-version" ]] || ! grep -qE "3\.(1[2-9]|[2-9][0-9])" .python-version 2>/dev/null; then echo "" print_info "Python 3.12 is installed via pyenv but not set for this project." read -p "Set Python 3.12.0 for this project? (Y/n): " -n 1 -r echo "" if [[ ! $REPLY =~ ^[Nn]$ ]]; then # Find the first suitable Python version local py_version=$(pyenv versions --bare | grep -E "^3\.(1[2-9]|[2-9][0-9])" | head -1) if [[ -n "$py_version" ]]; then pyenv local "$py_version" print_success "Set Python $py_version for this project" # Re-initialize pyenv to pick up the change eval "$(pyenv init --path)" 2>/dev/null || true eval "$(pyenv init -)" 2>/dev/null || true # Try finding Python again if python_cmd=$(find_python); then echo "$python_cmd" return 0 fi fi fi fi fi else # No pyenv installed - show instructions echo "" >&2 print_error "Python 3.10+ not found. The 'mcp' package requires Python 3.10+." echo "" >&2 if [[ "$os_type" == "macos" ]]; then echo "To install Python locally for this project:" >&2 echo "" >&2 echo "1. Install pyenv (manages Python versions per project):" >&2 echo " brew install pyenv" >&2 echo "" >&2 echo "2. Add to ~/.zshrc:" >&2 echo ' export PYENV_ROOT="$HOME/.pyenv"' >&2 echo ' export PATH="$PYENV_ROOT/bin:$PATH"' >&2 echo ' eval "$(pyenv init -)"' >&2 echo "" >&2 echo "3. Restart terminal, then run:" >&2 echo " pyenv install 3.12.0" >&2 echo " cd $(pwd)" >&2 echo " pyenv local 3.12.0" >&2 echo " ./run-server.sh" >&2 else # Linux/WSL echo "To install Python locally for this project:" >&2 echo "" >&2 echo "1. Install pyenv:" >&2 echo " curl https://pyenv.run | bash" >&2 echo "" >&2 echo "2. Add to ~/.bashrc:" >&2 echo ' export PYENV_ROOT="$HOME/.pyenv"' >&2 echo ' export PATH="$PYENV_ROOT/bin:$PATH"' >&2 echo ' eval "$(pyenv init -)"' >&2 echo "" >&2 echo "3. Restart terminal, then run:" >&2 echo " pyenv install 3.12.0" >&2 echo " cd $(pwd)" >&2 echo " pyenv local 3.12.0" >&2 echo " ./run-server.sh" >&2 fi fi else # Other systems (shouldn't happen with bash script) print_error "Python 3.10+ not found. Please install Python 3.10 or newer." fi return 1 } # Install Python with pyenv (when pyenv is already installed) install_python_with_pyenv() { # Ensure pyenv is initialized export PYENV_ROOT="${PYENV_ROOT:-$HOME/.pyenv}" export PATH="$PYENV_ROOT/bin:$PATH" eval "$(pyenv init -)" 2>/dev/null || true print_info "Installing Python 3.12 (this may take a few minutes)..." if pyenv install -s 3.12.0; then print_success "Python 3.12 installed" # Set local Python version for this project pyenv local 3.12.0 print_success "Python 3.12 set for this project" # Show shell configuration instructions echo "" print_info "To make pyenv work in new terminals, add to your shell config:" local shell_config="~/.zshrc" if [[ "$SHELL" == *"bash"* ]]; then shell_config="~/.bashrc" fi echo ' export PYENV_ROOT="$HOME/.pyenv"' echo ' command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"' echo ' eval "$(pyenv init -)"' echo "" # Re-initialize pyenv to use the newly installed Python eval "$(pyenv init --path)" 2>/dev/null || true eval "$(pyenv init -)" 2>/dev/null || true return 0 else print_error "Failed to install Python 3.12" return 1 fi } # Detect Linux distribution detect_linux_distro() { if [[ -f /etc/os-release ]]; then . /etc/os-release echo "${ID:-unknown}" elif [[ -f /etc/debian_version ]]; then echo "debian" elif [[ -f /etc/redhat-release ]]; then echo "rhel" elif [[ -f /etc/arch-release ]]; then echo "arch" else echo "unknown" fi } # Get package manager and install command for the distro get_install_command() { local distro="$1" local python_version="${2:-}" # Extract major.minor version if provided local version_suffix="" if [[ -n "$python_version" ]] && [[ "$python_version" =~ ([0-9]+\.[0-9]+) ]]; then version_suffix="${BASH_REMATCH[1]}" fi case "$distro" in ubuntu|debian|raspbian|pop|linuxmint|elementary) if [[ -n "$version_suffix" ]]; then # Try version-specific packages first, then fall back to generic echo "sudo apt update && (sudo apt install -y python${version_suffix}-venv python${version_suffix}-dev || sudo apt install -y python3-venv python3-pip)" else echo "sudo apt update && sudo apt install -y python3-venv python3-pip" fi ;; fedora) echo "sudo dnf install -y python3-venv python3-pip" ;; rhel|centos|rocky|almalinux|oracle) echo "sudo dnf install -y python3-venv python3-pip || sudo yum install -y python3-venv python3-pip" ;; arch|manjaro|endeavouros) echo "sudo pacman -Syu --noconfirm python-pip python-virtualenv" ;; opensuse|suse) echo "sudo zypper install -y python3-venv python3-pip" ;; alpine) echo "sudo apk add --no-cache python3-dev py3-pip py3-virtualenv" ;; *) echo "" ;; esac } # Check if we can use sudo can_use_sudo() { # Check if sudo exists and user can use it if command -v sudo &> /dev/null; then # Test sudo with a harmless command if sudo -n true 2>/dev/null; then return 0 elif [[ -t 0 ]]; then # Terminal is interactive, test if sudo works with password if sudo true 2>/dev/null; then return 0 fi fi fi return 1 } # Try to install system packages automatically try_install_system_packages() { local python_cmd="${1:-python3}" local os_type=$(detect_os) # Skip on macOS as it works fine if [[ "$os_type" == "macos" ]]; then return 1 fi # Only try on Linux systems if [[ "$os_type" != "linux" && "$os_type" != "wsl" ]]; then return 1 fi # Get Python version local python_version="" if command -v "$python_cmd" &> /dev/null; then python_version=$($python_cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "") fi local distro=$(detect_linux_distro) local install_cmd=$(get_install_command "$distro" "$python_version") if [[ -z "$install_cmd" ]]; then return 1 fi print_info "Attempting to install required Python packages..." # Check if we can use sudo if can_use_sudo; then print_info "Installing system packages (this may ask for your password)..." if bash -c "$install_cmd" >/dev/null 2>&1; then # Replaced eval to prevent command injection print_success "System packages installed successfully" return 0 else print_warning "Failed to install system packages automatically" fi fi return 1 } # Bootstrap pip in virtual environment bootstrap_pip() { local venv_python="$1" local python_cmd="$2" print_info "Bootstrapping pip in virtual environment..." # Try ensurepip first if $venv_python -m ensurepip --default-pip >/dev/null 2>&1; then print_success "Successfully bootstrapped pip using ensurepip" return 0 fi # Try to download get-pip.py print_info "Downloading pip installer..." local get_pip_url="https://bootstrap.pypa.io/get-pip.py" local temp_pip=$(mktemp) local download_success=false # Try curl first if command -v curl &> /dev/null; then if curl -sSL "$get_pip_url" -o "$temp_pip" 2>/dev/null; then download_success=true fi fi # Try wget if curl failed if [[ "$download_success" == false ]] && command -v wget &> /dev/null; then if wget -qO "$temp_pip" "$get_pip_url" 2>/dev/null; then download_success=true fi fi # Try python urllib as last resort if [[ "$download_success" == false ]]; then print_info "Using Python to download pip installer..." if $python_cmd -c "import urllib.request; urllib.request.urlretrieve('$get_pip_url', '$temp_pip')" 2>/dev/null; then download_success=true fi fi if [[ "$download_success" == true ]] && [[ -f "$temp_pip" ]] && [[ -s "$temp_pip" ]]; then print_info "Installing pip..." if $venv_python "$temp_pip" --no-warn-script-location >/dev/null 2>&1; then rm -f "$temp_pip" print_success "Successfully installed pip" return 0 fi fi rm -f "$temp_pip" 2>/dev/null return 1 } # Setup environment using uv-first approach setup_environment() { local venv_python="" # Try uv-first approach if command -v uv &> /dev/null; then print_info "Setting up environment with uv..." # Only remove existing venv if it wasn't created by uv (to ensure clean uv setup) if [[ -d "$VENV_PATH" ]] && [[ ! -f "$VENV_PATH/uv_created" ]]; then print_info "Removing existing environment for clean uv setup..." rm -rf "$VENV_PATH" fi # Try Python 3.12 first (preferred) local uv_output if uv_output=$(uv venv --python 3.12 "$VENV_PATH" 2>&1); then # Use helper function for cross-platform path detection if venv_python=$(get_venv_python_path "$VENV_PATH"); then touch "$VENV_PATH/uv_created" # Mark as uv-created print_success "Created environment with uv using Python 3.12" # Ensure pip is installed in uv environment if ! $venv_python -m pip --version &>/dev/null 2>&1; then print_info "Installing pip in uv environment..." # uv doesn't install pip by default, use bootstrap method if bootstrap_pip "$venv_python" "python3"; then print_success "pip installed in uv environment" else print_warning "Failed to install pip in uv environment" fi fi else print_warning "uv succeeded but Python executable not found in venv" fi # Fall back to any available Python through uv elif uv_output=$(uv venv "$VENV_PATH" 2>&1); then # Use helper function for cross-platform path detection if venv_python=$(get_venv_python_path "$VENV_PATH"); then touch "$VENV_PATH/uv_created" # Mark as uv-created local python_version=$($venv_python --version 2>&1) print_success "Created environment with uv using $python_version" # Ensure pip is installed in uv environment if ! $venv_python -m pip --version &>/dev/null 2>&1; then print_info "Installing pip in uv environment..." # uv doesn't install pip by default, use bootstrap method if bootstrap_pip "$venv_python" "python3"; then print_success "pip installed in uv environment" else print_warning "Failed to install pip in uv environment" fi fi else print_warning "uv succeeded but Python executable not found in venv" fi else print_warning "uv environment creation failed, falling back to system Python detection" print_warning "uv output: $uv_output" fi else print_info "uv not found, using system Python detection" fi # If uv failed or not available, fallback to system Python detection if [[ -z "$venv_python" ]]; then print_info "Setting up environment with system Python..." local python_cmd python_cmd=$(find_python) || return 1 # Use existing venv creation logic venv_python=$(setup_venv "$python_cmd") if [[ $? -ne 0 ]]; then return 1 fi else # venv_python was already set by uv creation above, just convert to absolute path if [[ -n "$venv_python" ]]; then # Convert to absolute path for MCP registration local abs_venv_python if cd "$(dirname "$venv_python")" 2>/dev/null; then abs_venv_python=$(pwd)/$(basename "$venv_python") venv_python="$abs_venv_python" else print_error "Failed to resolve absolute path for venv_python" return 1 fi fi fi echo "$venv_python" return 0 } # Setup virtual environment setup_venv() { local python_cmd="$1" local venv_python="" local venv_pip="" # Create venv if it doesn't exist if [[ ! -d "$VENV_PATH" ]]; then print_info "Creating isolated environment..." # Capture error output for better diagnostics local venv_error if venv_error=$($python_cmd -m venv "$VENV_PATH" 2>&1); then print_success "Created isolated environment" else # Check for common Linux issues and try fallbacks local os_type=$(detect_os) if [[ "$os_type" == "linux" || "$os_type" == "wsl" ]]; then if echo "$venv_error" | grep -E -q "No module named venv|venv.*not found|ensurepip is not|python3.*-venv"; then # Try to install system packages automatically first if try_install_system_packages "$python_cmd"; then print_info "Retrying virtual environment creation..." if venv_error=$($python_cmd -m venv "$VENV_PATH" 2>&1); then print_success "Created isolated environment" else # Continue to fallback methods below print_warning "Still unable to create venv, trying fallback methods..." fi fi # If venv still doesn't exist, try fallback methods if [[ ! -d "$VENV_PATH" ]]; then # Try virtualenv as fallback if command -v virtualenv &> /dev/null; then print_info "Attempting to create environment with virtualenv..." if virtualenv -p "$python_cmd" "$VENV_PATH" &>/dev/null 2>&1; then print_success "Created environment using virtualenv fallback" fi fi # Try python -m virtualenv if directory wasn't created if [[ ! -d "$VENV_PATH" ]]; then if $python_cmd -m virtualenv "$VENV_PATH" &>/dev/null 2>&1; then print_success "Created environment using python -m virtualenv fallback" fi fi # Last resort: try to install virtualenv via pip and use it if [[ ! -d "$VENV_PATH" ]] && command -v pip3 &> /dev/null; then print_info "Installing virtualenv via pip..." if pip3 install --user virtualenv &>/dev/null 2>&1; then local user_bin="$HOME/.local/bin" if [[ -f "$user_bin/virtualenv" ]]; then if "$user_bin/virtualenv" -p "$python_cmd" "$VENV_PATH" &>/dev/null 2>&1; then print_success "Created environment using pip-installed virtualenv" fi fi fi fi fi # Check if any method succeeded if [[ ! -d "$VENV_PATH" ]]; then print_error "Unable to create virtual environment" echo "" echo "Your system is missing Python development packages." echo "" local distro=$(detect_linux_distro) local python_version=$($python_cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "") local install_cmd=$(get_install_command "$distro" "$python_version") if [[ -n "$install_cmd" ]]; then echo "Please run this command to install them:" echo " $install_cmd" else echo "Please install Python venv support for your system:" echo " Ubuntu/Debian: sudo apt install python3-venv python3-pip" echo " RHEL/CentOS: sudo dnf install python3-venv python3-pip" echo " Arch: sudo pacman -S python-pip python-virtualenv" fi echo "" echo "Then run this script again." exit 1 fi elif echo "$venv_error" | grep -q "Permission denied"; then print_error "Permission denied creating virtual environment" echo "" echo "Try running in a different directory:" echo " cd ~ && git clone && cd pal-mcp-server && ./run-server.sh" echo "" exit 1 else print_error "Failed to create virtual environment" echo "Error: $venv_error" exit 1 fi else # For non-Linux systems, show the error and exit print_error "Failed to create virtual environment" echo "Error: $venv_error" exit 1 fi fi fi # Get venv Python path based on platform local os_type=$(detect_os) case "$os_type" in windows) venv_python="$VENV_PATH/Scripts/python.exe" venv_pip="$VENV_PATH/Scripts/pip.exe" ;; *) venv_python="$VENV_PATH/bin/python" venv_pip="$VENV_PATH/bin/pip" ;; esac # Check if venv Python exists if [[ ! -f "$venv_python" ]]; then print_error "Virtual environment Python not found" exit 1 fi # Always check if pip exists in the virtual environment (regardless of how it was created) if [[ ! -f "$venv_pip" ]] && ! $venv_python -m pip --version &>/dev/null 2>&1; then print_warning "pip not found in virtual environment, installing..." # On Linux, try to install system packages if pip is missing local os_type=$(detect_os) if [[ "$os_type" == "linux" || "$os_type" == "wsl" ]]; then if try_install_system_packages "$python_cmd"; then # Check if pip is now available after system package install if $venv_python -m pip --version &>/dev/null 2>&1; then print_success "pip is now available" else # Still need to bootstrap pip bootstrap_pip "$venv_python" "$python_cmd" || true fi else # Try to bootstrap pip without system packages bootstrap_pip "$venv_python" "$python_cmd" || true fi else # For non-Linux systems, just try to bootstrap pip bootstrap_pip "$venv_python" "$python_cmd" || true fi # Final check after all attempts if ! $venv_python -m pip --version &>/dev/null 2>&1; then print_error "Failed to install pip in virtual environment" echo "" echo "Your Python installation appears to be incomplete." echo "" local distro=$(detect_linux_distro) local python_version=$($python_cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "") local install_cmd=$(get_install_command "$distro" "$python_version") if [[ -n "$install_cmd" ]]; then echo "Please run this command to install Python packages:" echo " $install_cmd" else echo "Please install Python pip support for your system." fi echo "" echo "Then delete the virtual environment and run this script again:" echo " rm -rf $VENV_PATH" echo " ./run-server.sh" echo "" exit 1 fi fi # Verify pip is working if ! $venv_python -m pip --version &>/dev/null 2>&1; then print_error "pip is not working correctly in the virtual environment" echo "" echo "Try deleting the virtual environment and running again:" echo " rm -rf $VENV_PATH" echo " ./run-server.sh" echo "" exit 1 fi if [[ -n "${VIRTUAL_ENV:-}" ]]; then print_success "Using activated virtual environment with pip" else print_success "Virtual environment ready with pip" fi # Convert to absolute path for MCP registration local abs_venv_python=$(cd "$(dirname "$venv_python")" && pwd)/$(basename "$venv_python") echo "$abs_venv_python" return 0 } # Check if package is installed check_package() { local python_cmd="$1" local module_name="$2" "$python_cmd" -c "import importlib, sys; importlib.import_module(sys.argv[1])" "$module_name" &>/dev/null } # Install dependencies install_dependencies() { local python_cmd="$1" local deps_needed=false # First verify pip is available with retry logic and bootstrap fallback local pip_available=false local max_attempts=3 for ((attempt=1; attempt<=max_attempts; attempt++)); do if "$python_cmd" -m pip --version &>/dev/null; then pip_available=true break else if (( attempt < max_attempts )); then print_warning "Attempt $attempt/$max_attempts: pip not available, retrying in 1 second..." sleep 1 fi fi done # If pip is still not available after retries, try to bootstrap it if [[ "$pip_available" == false ]]; then print_warning "pip is not available in the Python environment after $max_attempts attempts" # Enhanced diagnostic information for debugging print_info "Diagnostic information:" print_info " Python executable: $python_cmd" print_info " Python executable exists: $(if [[ -f "$python_cmd" ]]; then echo "Yes"; else echo "No"; fi)" print_info " Python executable permissions: $(ls -la "$python_cmd" 2>/dev/null || echo "Cannot check")" print_info " Virtual environment path: $VENV_PATH" print_info " Virtual environment exists: $(if [[ -d "$VENV_PATH" ]]; then echo "Yes"; else echo "No"; fi)" print_info "Attempting to bootstrap pip..." # Extract the base python command for bootstrap (fallback to python3) local base_python_cmd="python3" if command -v python &> /dev/null; then base_python_cmd="python" fi # Try to bootstrap pip if bootstrap_pip "$python_cmd" "$base_python_cmd"; then print_success "Successfully bootstrapped pip" # Verify pip is now available if $python_cmd -m pip --version &>/dev/null 2>&1; then pip_available=true else print_error "pip still not available after bootstrap attempt" fi else print_error "Failed to bootstrap pip" fi fi # Final check - if pip is still not available, exit with error if [[ "$pip_available" == false ]]; then print_error "pip is not available in the Python environment" echo "" echo "This indicates an incomplete Python installation or a problem with the virtual environment." echo "" echo "Final diagnostic information:" echo " Python executable: $python_cmd" echo " Python version: $($python_cmd --version 2>&1 || echo "Cannot determine")" echo " pip module check: $($python_cmd -c "import pip; print('Available')" 2>&1 || echo "Not available")" echo "" echo "Troubleshooting steps:" echo "1. Delete the virtual environment: rm -rf $VENV_PATH" echo "2. Run this script again: ./run-server.sh" echo "3. If the problem persists, check your Python installation" echo "4. For Git Bash on Windows, try running from a regular Command Prompt or PowerShell" echo "" return 1 fi # Check required packages local packages=("mcp" "google.genai" "openai" "pydantic" "dotenv") for package in "${packages[@]}"; do if ! check_package "$python_cmd" "$package"; then deps_needed=true break fi done if [[ "$deps_needed" == false ]]; then print_success "Dependencies already installed" return 0 fi echo "" print_info "Setting up PAL MCP Server..." echo "Installing required components:" echo " • MCP protocol library" echo " • AI model connectors" echo " • Data validation tools" echo " • Environment configuration" echo "" # Determine installation method and execute directly to handle paths with spaces local install_output local exit_code=0 echo -n "Downloading packages..." if command -v uv &> /dev/null && [[ -f "$VENV_PATH/uv_created" ]]; then print_info "Using uv for faster package installation..." install_output=$(uv pip install -q -r requirements.txt --python "$python_cmd" 2>&1) || exit_code=$? elif [[ -n "${VIRTUAL_ENV:-}" ]] || [[ "$python_cmd" == *"$VENV_PATH"* ]]; then install_output=$("$python_cmd" -m pip install -q -r requirements.txt 2>&1) || exit_code=$? else install_output=$("$python_cmd" -m pip install -q --user -r requirements.txt 2>&1) || exit_code=$? fi if [[ $exit_code -ne 0 ]]; then echo -e "\r${RED}✗ Setup failed${NC} " echo "" echo "Installation error:" echo "$install_output" | head -20 echo "" # Check for common issues if echo "$install_output" | grep -q "No module named pip"; then print_error "pip module not found" echo "" echo "Your Python installation is incomplete. Please install pip:" local distro=$(detect_linux_distro) local python_version=$($python_cmd --version 2>&1 | grep -oE '[0-9]+\.[0-9]+\.[0-9]+' || echo "") local install_cmd=$(get_install_command "$distro" "$python_version") if [[ -n "$install_cmd" ]]; then echo "" echo "For your system ($distro), run:" echo " $install_cmd" else echo "" echo " Ubuntu/Debian: sudo apt install python3-pip" echo " RHEL/CentOS: sudo dnf install python3-pip" echo " Arch: sudo pacman -S python-pip" fi elif echo "$install_output" | grep -q "Permission denied"; then print_error "Permission denied during installation" echo "" echo "Try using a virtual environment or install with --user flag:" echo " $python_cmd -m pip install --user -r requirements.txt" else echo "Try running manually:" if [[ "$use_uv" == true ]]; then echo " uv pip install -r requirements.txt --python $python_cmd" echo "Or fallback to pip:" fi echo " $python_cmd -m pip install -r requirements.txt" echo "" echo "Or install individual packages:" echo " $python_cmd -m pip install mcp google-genai openai pydantic python-dotenv" fi return 1 else echo -e "\r${GREEN}✓ Setup complete!${NC} " # Verify critical imports work if ! check_package "$python_cmd" "dotenv"; then print_warning "python-dotenv not imported correctly, installing explicitly..." if $python_cmd -m pip install python-dotenv &>/dev/null 2>&1; then print_success "python-dotenv installed successfully" else print_error "Failed to install python-dotenv" return 1 fi fi return 0 fi } # ---------------------------------------------------------------------------- # Environment Configuration Functions # ---------------------------------------------------------------------------- # Setup .env file setup_env_file() { if [[ -f .env ]]; then print_success ".env file already exists" migrate_env_file return 0 fi if [[ ! -f .env.example ]]; then print_error ".env.example not found!" return 1 fi cp .env.example .env print_success "Created .env from .env.example" # Update API keys from environment if present local api_keys=( "GEMINI_API_KEY:your_gemini_api_key_here" "OPENAI_API_KEY:your_openai_api_key_here" "XAI_API_KEY:your_xai_api_key_here" "DIAL_API_KEY:your_dial_api_key_here" "OPENROUTER_API_KEY:your_openrouter_api_key_here" ) for key_pair in "${api_keys[@]}"; do local key_name="${key_pair%%:*}" local placeholder="${key_pair##*:}" local key_value="${!key_name:-}" if [[ -n "$key_value" ]]; then sed "${SED_INPLACE_ARGS[@]}" "s/$placeholder/$key_value/" .env print_success "Updated .env with $key_name from environment" fi done return 0 } # Migrate .env file from Docker to standalone format migrate_env_file() { # Check if migration is needed if ! grep -q "host\.docker\.internal" .env 2>/dev/null; then return 0 fi print_warning "Migrating .env from Docker to standalone format..." # Create backup cp .env .env.backup_$(date +%Y%m%d_%H%M%S) # Replace host.docker.internal with localhost sed "${SED_INPLACE_ARGS[@]}" 's/host\.docker\.internal/localhost/g' .env print_success "Migrated Docker URLs to localhost in .env" echo " (Backup saved as .env.backup_*)" } # Check API keys and warn if missing (non-blocking) check_api_keys() { local has_key=false local api_keys=( "GEMINI_API_KEY:your_gemini_api_key_here" "OPENAI_API_KEY:your_openai_api_key_here" "XAI_API_KEY:your_xai_api_key_here" "DIAL_API_KEY:your_dial_api_key_here" "OPENROUTER_API_KEY:your_openrouter_api_key_here" ) for key_pair in "${api_keys[@]}"; do local key_name="${key_pair%%:*}" local placeholder="${key_pair##*:}" local key_value="${!key_name:-}" if [[ -n "$key_value" ]] && [[ "$key_value" != "$placeholder" ]]; then print_success "$key_name configured" has_key=true fi done # Check custom API URL if [[ -n "${CUSTOM_API_URL:-}" ]]; then print_success "CUSTOM_API_URL configured: $CUSTOM_API_URL" has_key=true fi if [[ "$has_key" == false ]]; then print_warning "No API keys found in .env!" echo "" echo "The Python development environment will be set up, but you won't be able to use the MCP server until you add API keys." echo "" echo "To add API keys, edit .env and add at least one:" echo " GEMINI_API_KEY=your-actual-key" echo " OPENAI_API_KEY=your-actual-key" echo " XAI_API_KEY=your-actual-key" echo " DIAL_API_KEY=your-actual-key" echo " OPENROUTER_API_KEY=your-actual-key" echo "" print_info "You can continue with development setup and add API keys later." echo "" fi return 0 # Always return success to continue setup } # ---------------------------------------------------------------------------- # Environment Variable Parsing Function # ---------------------------------------------------------------------------- # Parse .env file and extract all valid environment variables parse_env_variables() { local env_vars="" if [[ -f .env ]]; then # Read .env file and extract non-empty, non-comment variables while IFS= read -r line; do # Skip comments, empty lines, and lines starting with # if [[ -n "$line" && ! "$line" =~ ^[[:space:]]*# && "$line" =~ ^[[:space:]]*([^=]+)=(.*)$ ]]; then local key="${BASH_REMATCH[1]}" local value="${BASH_REMATCH[2]}" # Clean up key (remove leading/trailing whitespace) key=$(echo "$key" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//') # Skip if value is empty or just whitespace if [[ -n "$value" && ! "$value" =~ ^[[:space:]]*$ ]]; then # Clean up value (remove leading/trailing whitespace and quotes) value=$(echo "$value" | sed 's/^[[:space:]]*//;s/[[:space:]]*$//' | sed 's/^"//;s/"$//') # Remove inline comments (everything after # that's not in quotes) value=$(echo "$value" | sed 's/[[:space:]]*#.*$//') # Skip if value is a placeholder or empty after comment removal if [[ ! "$value" =~ ^your_.*_here$ && "$value" != "your_" && -n "$value" && ! "$value" =~ ^[[:space:]]*$ ]]; then env_vars+="$key=$value"$'\n' fi fi fi done < .env fi # If no .env file or no valid vars, fall back to environment variables if [[ -z "$env_vars" ]]; then local api_keys=( "GEMINI_API_KEY" "OPENAI_API_KEY" "XAI_API_KEY" "DIAL_API_KEY" "OPENROUTER_API_KEY" "CUSTOM_API_URL" "CUSTOM_API_KEY" "CUSTOM_MODEL_NAME" "DISABLED_TOOLS" "DEFAULT_MODEL" "LOG_LEVEL" "DEFAULT_THINKING_MODE_THINKDEEP" "CONVERSATION_TIMEOUT_HOURS" "MAX_CONVERSATION_TURNS" ) for key_name in "${api_keys[@]}"; do local key_value="${!key_name:-}" if [[ -n "$key_value" && ! "$key_value" =~ ^your_.*_here$ ]]; then env_vars+="$key_name=$key_value"$'\n' fi done fi echo "$env_vars" } # ---------------------------------------------------------------------------- # Claude Integration Functions # ---------------------------------------------------------------------------- # Check if MCP is added to Claude CLI and verify it's correct check_claude_cli_integration() { local python_cmd="$1" local server_path="$2" # Check for native installed Claude CLI (not in PATH by default) # Native installs: # - curl https://claude.ai/install.sh | bash -> ~/.local/bin/claude # - brew install --cask claude-code -> /opt/homebrew/bin/claude (Apple Silicon) or /usr/local/bin/claude (Intel) if ! command -v claude &> /dev/null; then local claude_paths=( "$HOME/.local/bin" "/opt/homebrew/bin" "/usr/local/bin" ) for dir in "${claude_paths[@]}"; do if [[ -x "$dir/claude" ]]; then print_info "Found native installed Claude CLI at $dir/claude" export PATH="$dir:$PATH" print_success "Added $dir to PATH" break fi done fi if ! command -v claude &> /dev/null; then echo "" print_warning "Claude CLI not found" echo "" read -p "Would you like to add PAL to Claude Code? (Y/n): " -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then print_info "Skipping Claude Code integration" return 0 fi echo "" echo "Please install Claude Code first:" echo " Visit: https://docs.anthropic.com/en/docs/claude-code/cli-usage" echo "" echo "Then run this script again to register MCP." return 1 fi # Remove legacy zen registrations to avoid duplicate errors after rename for legacy_name in "${LEGACY_MCP_NAMES[@]}"; do claude mcp remove "$legacy_name" -s user >/dev/null 2>&1 || true done # Check if pal is registered local mcp_list=$(claude mcp list 2>/dev/null) if echo "$mcp_list" | grep -q "pal"; then # Check if it's using the old Docker command if echo "$mcp_list" | grep -E "zen.*docker|zen.*compose" &>/dev/null; then print_warning "Found old Docker-based Zen registration, updating..." claude mcp remove zen -s user 2>/dev/null || true # Re-add with correct Python command and environment variables local env_vars=$(parse_env_variables) local env_args="" # Convert environment variables to -e arguments if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then env_args+=" -e ${BASH_REMATCH[1]}=\"${BASH_REMATCH[2]}\"" fi done <<< "$env_vars" fi local claude_cmd="claude mcp add pal -s user$env_args -- \"$python_cmd\" \"$server_path\"" if eval "$claude_cmd" 2>/dev/null; then print_success "Updated PAL to become a standalone script with environment variables" return 0 else echo "" echo "Failed to update MCP registration. Please run manually:" echo " claude mcp remove pal -s user" echo " $claude_cmd" return 1 fi else # Verify the registered path matches current setup local expected_cmd="$python_cmd $server_path" if echo "$mcp_list" | grep -F "$server_path" &>/dev/null; then return 0 else print_warning "PAL registered with different path, updating..." claude mcp remove pal -s user 2>/dev/null || true # Re-add with current path and environment variables local env_vars=$(parse_env_variables) local env_args="" # Convert environment variables to -e arguments if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then env_args+=" -e ${BASH_REMATCH[1]}=\"${BASH_REMATCH[2]}\"" fi done <<< "$env_vars" fi local claude_cmd="claude mcp add pal -s user$env_args -- \"$python_cmd\" \"$server_path\"" if eval "$claude_cmd" 2>/dev/null; then print_success "Updated PAL with current path and environment variables" return 0 else echo "" echo "Failed to update MCP registration. Please run manually:" echo " claude mcp remove pal -s user" echo " $claude_cmd" return 1 fi fi fi else # Not registered at all, ask user if they want to add it echo "" read -p "Add PAL to Claude Code? (Y/n): " -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then local env_vars=$(parse_env_variables) local env_args="" # Convert environment variables to -e arguments for manual command if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then env_args+=" -e ${BASH_REMATCH[1]}=\"${BASH_REMATCH[2]}\"" fi done <<< "$env_vars" fi print_info "To add manually later, run:" echo " claude mcp add pal -s user$env_args -- $python_cmd $server_path" return 0 fi print_info "Registering PAL with Claude Code..." # Add with environment variables local env_vars=$(parse_env_variables) local env_args="" # Convert environment variables to -e arguments if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then env_args+=" -e ${BASH_REMATCH[1]}=\"${BASH_REMATCH[2]}\"" fi done <<< "$env_vars" fi local claude_cmd="claude mcp add pal -s user$env_args -- \"$python_cmd\" \"$server_path\"" if eval "$claude_cmd" 2>/dev/null; then print_success "Successfully added PAL to Claude Code with environment variables" return 0 else echo "" echo "Failed to add automatically. To add manually, run:" echo " $claude_cmd" return 1 fi fi } # Check and update Claude Desktop configuration check_claude_desktop_integration() { local python_cmd="$1" local server_path="$2" # Skip if already configured (check flag) if [[ -f "$DESKTOP_CONFIG_FLAG" ]]; then return 0 fi local config_path=$(get_claude_config_path) if [[ -z "$config_path" ]]; then print_warning "Unable to determine Claude Desktop config path for this platform" return 0 fi # Legacy MCP server names to clean out from previous releases local legacy_names_csv legacy_names_csv=$(IFS=,; echo "${LEGACY_MCP_NAMES[*]}") echo "" read -p "Configure PAL for Claude Desktop? (Y/n): " -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then print_info "Skipping Claude Desktop integration" touch "$DESKTOP_CONFIG_FLAG" # Don't ask again return 0 fi # Create config directory if it doesn't exist local config_dir=$(dirname "$config_path") mkdir -p "$config_dir" 2>/dev/null || true # Handle existing config if [[ -f "$config_path" ]]; then print_info "Updating existing Claude Desktop config..." # Check for old Docker config and remove it if grep -q "docker.*compose.*pal\|pal.*docker" "$config_path" 2>/dev/null; then print_warning "Removing old Docker-based MCP configuration..." # Create backup cp "$config_path" "${config_path}.backup_$(date +%Y%m%d_%H%M%S)" # Remove old pal config using a more robust approach local temp_file=$(mktemp) python3 -c " import json import sys try: with open('$config_path', 'r') as f: config = json.load(f) # Remove pal from mcpServers if it exists if 'mcpServers' in config and 'pal' in config['mcpServers']: del config['mcpServers']['pal'] print('Removed old pal MCP configuration') with open('$temp_file', 'w') as f: json.dump(config, f, indent=2) except Exception as e: print(f'Error processing config: {e}', file=sys.stderr) sys.exit(1) " && mv "$temp_file" "$config_path" fi # Add new config with environment variables local env_vars=$(parse_env_variables) local temp_file=$(mktemp) local env_file=$(mktemp) # Write environment variables to a temporary file for Python to read if [[ -n "$env_vars" ]]; then echo "$env_vars" > "$env_file" fi PAL_LEGACY_NAMES="$legacy_names_csv" python3 -c " import json import os import sys legacy_keys = [k for k in os.environ.get('PAL_LEGACY_NAMES', '').split(',') if k] try: with open('$config_path', 'r') as f: config = json.load(f) except Exception: config = {} if not isinstance(config, dict): config = {} # Ensure mcpServers exists if 'mcpServers' not in config or not isinstance(config.get('mcpServers'), dict): config['mcpServers'] = {} # Remove legacy entries from any known server blocks for container in ('mcpServers', 'servers'): servers = config.get(container) if isinstance(servers, dict): for key in legacy_keys: servers.pop(key, None) # Add pal server pal_config = { 'command': '$python_cmd', 'args': ['$server_path'] } # Add environment variables if they exist env_dict = {} try: with open('$env_file', 'r') as f: for line in f: line = line.strip() if '=' in line and line: key, value = line.split('=', 1) env_dict[key] = value except Exception: pass if env_dict: pal_config['env'] = env_dict config['mcpServers']['pal'] = pal_config with open('$temp_file', 'w') as f: json.dump(config, f, indent=2) " && mv "$temp_file" "$config_path" # Clean up temporary env file rm -f "$env_file" 2>/dev/null || true else print_info "Creating new Claude Desktop config..." # Create new config with environment variables local env_vars=$(parse_env_variables) local temp_file=$(mktemp) local env_file=$(mktemp) # Write environment variables to a temporary file for Python to read if [[ -n "$env_vars" ]]; then echo "$env_vars" > "$env_file" fi python3 -c " import json import sys config = {'mcpServers': {}} # Add pal server pal_config = { 'command': '$python_cmd', 'args': ['$server_path'] } # Add environment variables if they exist env_dict = {} try: with open('$env_file', 'r') as f: for line in f: line = line.strip() if '=' in line and line: key, value = line.split('=', 1) env_dict[key] = value except: pass if env_dict: pal_config['env'] = env_dict config['mcpServers']['pal'] = pal_config with open('$temp_file', 'w') as f: json.dump(config, f, indent=2) " && mv "$temp_file" "$config_path" # Clean up temporary env file rm -f "$env_file" 2>/dev/null || true fi if [[ $? -eq 0 ]]; then print_success "Successfully configured Claude Desktop" echo " Config: $config_path" echo " Restart Claude Desktop to use the new MCP server" touch "$DESKTOP_CONFIG_FLAG" else print_error "Failed to update Claude Desktop config" echo "Manual config location: $config_path" echo "Add this configuration:" # Generate example with actual environment variables for error case example_env="" env_vars=$(parse_env_variables) if [[ -n "$env_vars" ]]; then local first_entry=true while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then local key="${BASH_REMATCH[1]}" local value="your_$(echo "${key}" | tr '[:upper:]' '[:lower:]')" if [[ "$first_entry" == true ]]; then first_entry=false example_env=" \"$key\": \"$value\"" else example_env+=",\n \"$key\": \"$value\"" fi fi done <<< "$env_vars" fi cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"]$(if [[ -n "$example_env" ]]; then echo ","; fi)$(if [[ -n "$example_env" ]]; then echo " \"env\": { $(echo -e "$example_env") }"; fi) } } } EOF fi } # Check and update Gemini CLI configuration check_gemini_cli_integration() { local script_dir="$1" local pal_wrapper="$script_dir/pal-mcp-server" # Check if Gemini settings file exists local gemini_config="$HOME/.gemini/settings.json" if [[ ! -f "$gemini_config" ]]; then # Gemini CLI not installed or not configured return 0 fi # Clean up legacy zen entries and detect existing pal configuration local legacy_names_csv legacy_names_csv=$(IFS=,; echo "${LEGACY_MCP_NAMES[*]}") local gemini_status gemini_status=$( PAL_LEGACY_NAMES="$legacy_names_csv" PAL_WRAPPER="$pal_wrapper" PAL_GEMINI_CONFIG="$gemini_config" python3 - <<'PY' 2>/dev/null import json import os import pathlib import sys config_path = pathlib.Path(os.environ["PAL_GEMINI_CONFIG"]) legacy = [n for n in os.environ.get("PAL_LEGACY_NAMES", "").split(",") if n] wrapper = os.environ["PAL_WRAPPER"] changed = False has_pal = False try: data = json.loads(config_path.read_text()) except Exception: data = {} if not isinstance(data, dict): data = {} servers = data.get("mcpServers") if not isinstance(servers, dict): servers = {} data["mcpServers"] = servers for key in legacy: if servers.pop(key, None) is not None: changed = True pal_cfg = servers.get("pal") if isinstance(pal_cfg, dict): has_pal = True if pal_cfg.get("command") != wrapper: pal_cfg["command"] = wrapper servers["pal"] = pal_cfg changed = True if changed: config_path.parent.mkdir(parents=True, exist_ok=True) config_path.write_text(json.dumps(data, indent=2)) status = ("CHANGED" if changed else "UNCHANGED") + ":" + ("HAS_PAL" if has_pal else "NO_PAL") sys.stdout.write(status) sys.exit(0) PY ) || true local gemini_changed=false local gemini_has_pal=false [[ "$gemini_status" == CHANGED:* ]] && gemini_changed=true [[ "$gemini_status" == *:HAS_PAL ]] && gemini_has_pal=true if [[ "$gemini_has_pal" == true ]]; then if [[ "$gemini_changed" == true ]]; then print_success "Removed legacy Gemini MCP entries" fi return 0 fi # Ask user if they want to add PAL to Gemini CLI echo "" read -p "Configure PAL for Gemini CLI? (Y/n): " -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then print_info "Skipping Gemini CLI integration" return 0 fi # Ensure wrapper script exists if [[ ! -f "$pal_wrapper" ]]; then print_info "Creating wrapper script for Gemini CLI..." cat > "$pal_wrapper" << 'EOF' #!/bin/bash # Wrapper script for Gemini CLI compatibility DIR="$(cd "$(dirname "$0")" && pwd)" cd "$DIR" exec .pal_venv/bin/python server.py "$@" EOF chmod +x "$pal_wrapper" print_success "Created pal-mcp-server wrapper script" fi # Update Gemini settings print_info "Updating Gemini CLI configuration..." # Create backup cp "$gemini_config" "${gemini_config}.backup_$(date +%Y%m%d_%H%M%S)" # Add pal configuration using Python for proper JSON handling local temp_file=$(mktemp) python3 -c " import json import sys try: with open('$gemini_config', 'r') as f: config = json.load(f) # Ensure mcpServers exists if 'mcpServers' not in config: config['mcpServers'] = {} # Add pal server config['mcpServers']['pal'] = { 'command': '$pal_wrapper' } with open('$temp_file', 'w') as f: json.dump(config, f, indent=2) except Exception as e: print(f'Error processing config: {e}', file=sys.stderr) sys.exit(1) " && mv "$temp_file" "$gemini_config" if [[ $? -eq 0 ]]; then print_success "Successfully configured Gemini CLI" echo " Config: $gemini_config" echo " Restart Gemini CLI to use PAL MCP Server" else print_error "Failed to update Gemini CLI config" echo "Manual config location: $gemini_config" echo "Add this configuration:" cat << EOF { "mcpServers": { "pal": { "command": "$pal_wrapper" } } } EOF fi } # Check and update Codex CLI configuration check_codex_cli_integration() { if ! command -v codex &> /dev/null; then return 0 fi local codex_config="$HOME/.codex/config.toml" local legacy_names_csv legacy_names_csv=$(IFS=,; echo "${LEGACY_MCP_NAMES[*]}") if [[ -f "$codex_config" ]]; then local codex_cleanup_status codex_cleanup_status=$( PAL_LEGACY_NAMES="$legacy_names_csv" PAL_CODEX_CONFIG="$codex_config" python3 - <<'PY' 2>/dev/null import os import pathlib import re import sys config_path = pathlib.Path(os.environ["PAL_CODEX_CONFIG"]) legacy = [n for n in os.environ.get("PAL_LEGACY_NAMES", "").split(",") if n] if not config_path.exists(): sys.exit(0) lines = config_path.read_text().splitlines() output = [] skip = False removed = False section_re = re.compile(r"\s*\[([^\]]+)\]") for line in lines: match = section_re.match(line) if match: header = match.group(1).strip() parts = header.split(".") is_legacy = False if len(parts) >= 2 and parts[0] == "mcp_servers": section_key = ".".join(parts[1:]) for name in legacy: if section_key == name or section_key.startswith(name + "."): is_legacy = True break skip = is_legacy if is_legacy: removed = True continue if not skip: output.append(line) if removed: config_path.write_text("\n".join(output).rstrip() + ("\n" if output else "")) sys.stdout.write("REMOVED") else: sys.stdout.write("UNCHANGED") sys.exit(0) PY ) || true if [[ "$codex_cleanup_status" == "REMOVED" ]]; then print_success "Removed legacy Codex MCP entries" fi fi local codex_has_pal=false if [[ -f "$codex_config" ]] && grep -q '\[mcp_servers\.pal\]' "$codex_config" 2>/dev/null; then codex_has_pal=true fi if [[ "$codex_has_pal" == false ]]; then echo "" read -p "Configure PAL for Codex CLI? (Y/n): " -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then print_info "Skipping Codex CLI integration" return 0 fi print_info "Updating Codex CLI configuration..." mkdir -p "$(dirname "$codex_config")" 2>/dev/null || true if [[ -f "$codex_config" ]]; then cp "$codex_config" "${codex_config}.backup_$(date +%Y%m%d_%H%M%S)" fi local env_vars=$(parse_env_variables) { echo "" echo "[mcp_servers.pal]" echo "command = \"bash\"" echo "args = [\"-c\", \"for p in \$(which uvx 2>/dev/null) \$HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \\\"\$p\\\" ] && exec \\\"\$p\\\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1\"]" echo "tool_timeout_sec = 1200" echo "" echo "[mcp_servers.pal.env]" echo "PATH = \"/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$HOME/bin\"" if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then local key="${BASH_REMATCH[1]}" local value="${BASH_REMATCH[2]}" local escaped_value escaped_value=$(echo "$value" | sed -e 's/\\/\\\\/g' -e 's/"/\\"/g') echo "$key = \"$escaped_value\"" fi done <<< "$env_vars" fi } >> "$codex_config" if [[ $? -ne 0 ]]; then print_error "Failed to update Codex CLI config" echo "Manual config location: $codex_config" echo "Add this configuration:" cat <<'CODExEOF' [mcp_servers.pal] command = "sh" args = ["-c", "exec \$(which uvx 2>/dev/null || echo uvx) --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server"] tool_timeout_sec = 1200 [mcp_servers.pal.env] PATH = "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$HOME/bin" [features] web_search_request = true CODExEOF if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then local key="${BASH_REMATCH[1]}" echo "${key} = \"your_$(echo "${key}" | tr '[:upper:]' '[:lower:]')\"" fi done <<< "$env_vars" else echo "GEMINI_API_KEY = \"your_gemini_api_key_here\"" fi return 0 fi print_success "Successfully configured Codex CLI" echo " Config: $codex_config" echo " Restart Codex CLI to use PAL MCP Server" codex_has_pal=true else print_info "Codex CLI already configured; refreshing Codex settings..." fi if [[ "$codex_has_pal" == true ]]; then if ! grep -Eq '^\s*web_search_request\s*=' "$codex_config" 2>/dev/null; then echo "" print_info "Web search requests let Codex pull fresh documentation for PAL's API lookup tooling." read -p "Enable Codex CLI web search requests? (Y/n): " -n 1 -r echo "" if [[ ! $REPLY =~ ^[Nn]$ ]]; then if grep -Eq '^\s*\[features\]' "$codex_config" 2>/dev/null; then if ! python3 - "$codex_config" <<'PY' import sys from pathlib import Path cfg_path = Path(sys.argv[1]) content = cfg_path.read_text().splitlines() output = [] in_features = False added = False for line in content: stripped = line.strip() if stripped.startswith("[") and stripped.endswith("]"): if in_features and not added: output.append("web_search_request = true") added = True in_features = stripped == "[features]" output.append(line) continue if in_features and stripped.startswith("web_search_request"): added = True output.append(line) if in_features and not added: output.append("web_search_request = true") cfg_path.write_text("\n".join(output) + "\n") PY then print_error "Failed to enable Codex web search request feature. Add 'web_search_request = true' under [features] in $codex_config manually." else print_success "Enabled Codex web search request feature" fi else { echo "" echo "[features]" echo "web_search_request = true" } >> "$codex_config" && print_success "Enabled Codex web search request feature" || \ print_error "Failed to enable Codex web search request feature. Add 'web_search_request = true' under [features] in $codex_config manually." fi else print_info "Skipping Codex web search request feature" fi fi if grep -Eq '^\s*\[tools\]' "$codex_config" 2>/dev/null && \ grep -Eq '^\s*web_search\s*=' "$codex_config" 2>/dev/null; then local removal_status if removal_status=$(python3 - "$codex_config" <<'PY' | tr -d '\n' import sys from pathlib import Path cfg_path = Path(sys.argv[1]) lines = cfg_path.read_text().splitlines() output = [] in_tools = False removed = False for line in lines: stripped = line.strip() if stripped.startswith('[') and stripped.endswith(']'): in_tools = stripped == '[tools]' output.append(line) continue if in_tools and stripped.startswith('web_search'): removed = True continue output.append(line) if removed: cfg_path.write_text("\n".join(output) + "\n") print('REMOVED', end='') else: print('UNCHANGED', end='') PY ); then if [[ "$removal_status" == "REMOVED" ]]; then print_success "Removed deprecated Codex [tools].web_search entry" fi else print_warning "Failed to clean up deprecated Codex [tools].web_search entry; remove manually from $codex_config" fi fi fi } # Print manual Qwen CLI configuration guidance print_qwen_manual_instructions() { local python_cmd="$1" local server_path="$2" local script_dir="$3" local config_path="$4" local env_lines="$5" local env_array=() if [[ -n "$env_lines" ]]; then while IFS= read -r line; do [[ -z "$line" ]] && continue env_array+=("$line") done <<< "$env_lines" fi echo "Manual config location: $config_path" echo "Add or update this entry:" local env_block="" if [[ ${#env_array[@]} -gt 0 ]]; then env_block=$' "env": {\n' local first=true for env_entry in "${env_array[@]}"; do local key="${env_entry%%=*}" local value="${env_entry#*=}" value=${value//\\/\\\\} value=${value//"/\\"} if [[ "$first" == true ]]; then first=false env_block+=" \"$key\": \"$value\"" else env_block+=$',\n ' env_block+="\"$key\": \"$value\"" fi done env_block+=$'\n }' fi if [[ -n "$env_block" ]]; then cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"], "cwd": "$script_dir", $env_block } } } EOF else cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"], "cwd": "$script_dir" } } } EOF fi } # Check and update Qwen Code CLI configuration check_qwen_cli_integration() { local python_cmd="$1" local server_path="$2" if ! command -v qwen &> /dev/null; then return 0 fi local qwen_config="$HOME/.qwen/settings.json" local script_dir script_dir=$(dirname "$server_path") local env_vars env_vars=$(parse_env_variables) local env_array=() if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then env_array+=("${BASH_REMATCH[1]}=${BASH_REMATCH[2]}") fi done <<< "$env_vars" fi local env_lines="" if [[ ${#env_array[@]} -gt 0 ]]; then env_lines=$(printf '%s\n' "${env_array[@]}") fi local legacy_names_csv legacy_names_csv=$(IFS=,; echo "${LEGACY_MCP_NAMES[*]}") if [[ -f "$qwen_config" ]]; then PAL_QWEN_LEGACY="$legacy_names_csv" PAL_QWEN_CONFIG="$qwen_config" python3 - <<'PYCLEANCONF' 2>/dev/null || true import json import os import pathlib import sys config_path = pathlib.Path(os.environ.get("PAL_QWEN_CONFIG", "")) legacy = [n for n in os.environ.get("PAL_QWEN_LEGACY", "").split(",") if n] if not config_path.exists(): sys.exit(0) try: data = json.loads(config_path.read_text(encoding="utf-8")) except Exception: sys.exit(0) if not isinstance(data, dict): sys.exit(0) servers = data.get("mcpServers") if isinstance(servers, dict): removed = False for key in legacy: if servers.pop(key, None) is not None: removed = True if removed: config_path.write_text(json.dumps(data, indent=2)) sys.exit(0) PYCLEANCONF fi local config_status=3 if [[ -f "$qwen_config" ]]; then if python3 - "$qwen_config" "$python_cmd" "$server_path" "$script_dir" <<'PYCONF' import json import sys config_path, expected_cmd, expected_arg, expected_cwd = sys.argv[1:5] try: with open(config_path, 'r', encoding='utf-8') as f: data = json.load(f) except FileNotFoundError: sys.exit(1) except Exception: sys.exit(5) servers = data.get('mcpServers') if not isinstance(servers, dict): sys.exit(3) config = servers.get('pal') if not isinstance(config, dict): sys.exit(3) cmd = config.get('command') args = config.get('args') or [] cwd = config.get('cwd') cwd_matches = cwd in (None, "", expected_cwd) if cmd == expected_cmd and len(args) == 1 and args[0] == expected_arg and cwd_matches: sys.exit(0) sys.exit(4) PYCONF then config_status=0 else config_status=$? if [[ $config_status -eq 1 ]]; then config_status=3 fi fi fi if [[ $config_status -eq 0 ]]; then return 0 fi echo "" if [[ $config_status -eq 4 ]]; then print_warning "Found existing Qwen CLI pal configuration with different settings." elif [[ $config_status -eq 5 ]]; then print_warning "Unable to parse Qwen CLI settings; replacing with a fresh entry may help." fi local prompt="Configure PAL for Qwen CLI? (Y/n): " if [[ $config_status -eq 4 || $config_status -eq 5 ]]; then prompt="Update Qwen CLI pal configuration? (Y/n): " fi read -p "$prompt" -n 1 -r echo "" if [[ $REPLY =~ ^[Nn]$ ]]; then print_info "Skipping Qwen CLI integration" print_qwen_manual_instructions "$python_cmd" "$server_path" "$script_dir" "$qwen_config" "$env_lines" return 0 fi mkdir -p "$(dirname "$qwen_config")" 2>/dev/null || true if [[ -f "$qwen_config" && $config_status -ne 3 ]]; then cp "$qwen_config" "${qwen_config}.backup_$(date +%Y%m%d_%H%M%S)" 2>/dev/null || true fi local update_output local update_status=0 update_output=$(PAL_QWEN_ENV="$env_lines" PAL_QWEN_CMD="$python_cmd" PAL_QWEN_ARG="$server_path" PAL_QWEN_CWD="$script_dir" python3 - "$qwen_config" <<'PYUPDATE' import json import os import pathlib import sys config_path = pathlib.Path(sys.argv[1]) cmd = os.environ['PAL_QWEN_CMD'] arg = os.environ['PAL_QWEN_ARG'] cwd = os.environ['PAL_QWEN_CWD'] env_lines = os.environ.get('PAL_QWEN_ENV', '').splitlines() env_map = {} for line in env_lines: if not line.strip(): continue if '=' in line: key, value = line.split('=', 1) env_map[key] = value if config_path.exists(): try: with config_path.open('r', encoding='utf-8') as f: data = json.load(f) except Exception: data = {} else: data = {} if not isinstance(data, dict): data = {} servers = data.get('mcpServers') if not isinstance(servers, dict): servers = {} data['mcpServers'] = servers pal_config = { 'command': cmd, 'args': [arg], 'cwd': cwd, } if env_map: pal_config['env'] = env_map servers['pal'] = pal_config config_path.parent.mkdir(parents=True, exist_ok=True) tmp_path = config_path.with_suffix(config_path.suffix + '.tmp') with tmp_path.open('w', encoding='utf-8') as f: json.dump(data, f, indent=2) f.write('\n') tmp_path.replace(config_path) PYUPDATE ) || update_status=$? if [[ $update_status -eq 0 ]]; then print_success "Successfully configured Qwen CLI" echo " Config: $qwen_config" echo " Restart Qwen CLI to use PAL MCP Server" else print_error "Failed to update Qwen CLI config" if [[ -n "$update_output" ]]; then echo "$update_output" fi print_qwen_manual_instructions "$python_cmd" "$server_path" "$script_dir" "$qwen_config" "$env_lines" fi } # Display configuration instructions display_config_instructions() { local python_cmd="$1" local server_path="$2" # Get script directory for Gemini CLI config local script_dir=$(dirname "$server_path") echo "" local config_header="PAL MCP SERVER CONFIGURATION" echo "===== $config_header =====" printf '%*s\n' "$((${#config_header} + 12))" | tr ' ' '=' echo "" echo "To use PAL MCP Server with your CLI clients:" echo "" print_info "1. For Claude Code (CLI):" # Show command with environment variables local env_vars=$(parse_env_variables) local env_args="" if [[ -n "$env_vars" ]]; then while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then env_args+=" -e ${BASH_REMATCH[1]}=\"${BASH_REMATCH[2]}\"" fi done <<< "$env_vars" fi echo -e " ${GREEN}claude mcp add pal -s user$env_args -- $python_cmd $server_path${NC}" echo "" print_info "2. For Claude Desktop:" echo " Add this configuration to your Claude Desktop config file:" echo "" # Generate example with actual environment variables that exist example_env="" env_vars=$(parse_env_variables) if [[ -n "$env_vars" ]]; then local first_entry=true while IFS= read -r line; do if [[ -n "$line" && "$line" =~ ^([^=]+)=(.*)$ ]]; then local key="${BASH_REMATCH[1]}" local value="your_$(echo "${key}" | tr '[:upper:]' '[:lower:]')" if [[ "$first_entry" == true ]]; then first_entry=false example_env=" \"$key\": \"$value\"" else example_env+=",\n \"$key\": \"$value\"" fi fi done <<< "$env_vars" fi if [[ -n "$example_env" ]]; then cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"], "cwd": "$script_dir", "env": { $(echo -e "$example_env") } } } } EOF else cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"], "cwd": "$script_dir" } } } EOF fi # Show platform-specific config location local config_path=$(get_claude_config_path) if [[ -n "$config_path" ]]; then echo "" print_info " Config file location:" echo -e " ${YELLOW}$config_path${NC}" fi echo "" print_info "3. Restart Claude Desktop after updating the config file" echo "" print_info "For Gemini CLI:" echo " Add this configuration to ~/.gemini/settings.json:" echo "" cat << EOF { "mcpServers": { "pal": { "command": "$script_dir/pal-mcp-server" } } } EOF echo "" print_info "For Qwen Code CLI:" echo " Add this configuration to ~/.qwen/settings.json:" echo "" if [[ -n "$example_env" ]]; then cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"], "cwd": "$script_dir", "env": { $(echo -e "$example_env") } } } } EOF else cat << EOF { "mcpServers": { "pal": { "command": "$python_cmd", "args": ["$server_path"], "cwd": "$script_dir" } } } EOF fi echo "" print_info "For Codex CLI:" echo " Add this configuration to ~/.codex/config.toml:" echo "" cat << EOF [mcp_servers.pal] command = "bash" args = ["-c", "for p in \$(which uvx 2>/dev/null) \$HOME/.local/bin/uvx /opt/homebrew/bin/uvx /usr/local/bin/uvx uvx; do [ -x \\\"\$p\\\" ] && exec \\\"\$p\\\" --from git+https://github.com/BeehiveInnovations/pal-mcp-server.git pal-mcp-server; done; echo 'uvx not found' >&2; exit 1"] [mcp_servers.pal.env] PATH = "/usr/local/bin:/usr/bin:/bin:/opt/homebrew/bin:\$HOME/.local/bin:\$HOME/.cargo/bin:\$HOME/bin" GEMINI_API_KEY = "your_gemini_api_key_here" EOF echo "" } # Display setup instructions display_setup_instructions() { local python_cmd="$1" local server_path="$2" echo "" local setup_header="SETUP COMPLETE" echo "===== $setup_header =====" printf '%*s\n' "$((${#setup_header} + 12))" | tr ' ' '=' echo "" print_success "PAL is ready to use!" # Display enabled/disabled tools if DISABLED_TOOLS is configured if [[ -n "${DISABLED_TOOLS:-}" ]]; then echo "" print_info "Tool Configuration:" # Dynamically discover all available tools from the tools directory # Excludes: __pycache__, shared modules, models.py, listmodels.py, version.py local all_tools=() for tool_file in tools/*.py; do if [[ -f "$tool_file" ]]; then local tool_name=$(basename "$tool_file" .py) # Skip non-tool files if [[ "$tool_name" != "models" && "$tool_name" != "listmodels" && "$tool_name" != "version" && "$tool_name" != "__init__" ]]; then all_tools+=("$tool_name") fi fi done # Convert DISABLED_TOOLS to array IFS=',' read -ra disabled_array <<< "$DISABLED_TOOLS" # Trim whitespace from disabled tools local disabled_tools=() for tool in "${disabled_array[@]}"; do disabled_tools+=("$(echo "$tool" | xargs)") done # Determine enabled tools local enabled_tools=() for tool in "${all_tools[@]}"; do local is_disabled=false for disabled in "${disabled_tools[@]}"; do if [[ "$tool" == "$disabled" ]]; then is_disabled=true break fi done if [[ "$is_disabled" == false ]]; then enabled_tools+=("$tool") fi done # Display enabled tools echo "" echo -e " ${GREEN}Enabled Tools (${#enabled_tools[@]}):${NC}" local enabled_list="" for tool in "${enabled_tools[@]}"; do if [[ -n "$enabled_list" ]]; then enabled_list+=", " fi enabled_list+="$tool" done echo " $enabled_list" # Display disabled tools echo "" echo -e " ${YELLOW}Disabled Tools (${#disabled_tools[@]}):${NC}" local disabled_list="" for tool in "${disabled_tools[@]}"; do if [[ -n "$disabled_list" ]]; then disabled_list+=", " fi disabled_list+="$tool" done echo " $disabled_list" echo "" echo " To enable more tools, edit the DISABLED_TOOLS variable in .env" fi } # ---------------------------------------------------------------------------- # Log Management Functions # ---------------------------------------------------------------------------- # Show help message show_help() { local version=$(get_version) local header="🤖 PAL MCP Server v$version" echo "$header" printf '%*s\n' "${#header}" | tr ' ' '=' echo "" echo "Usage: $0 [OPTIONS]" echo "" echo "Options:" echo " -h, --help Show this help message" echo " -v, --version Show version information" echo " -f, --follow Follow server logs in real-time" echo " -c, --config Show configuration instructions for Claude clients" echo " --clear-cache Clear Python cache and exit (helpful for import issues)" echo "" echo "Examples:" echo " $0 Setup and start the MCP server" echo " $0 -f Setup and follow logs" echo " $0 -c Show configuration instructions" echo " $0 --version Show version only" echo " $0 --clear-cache Clear Python cache (fixes import issues)" echo "" echo "For more information, visit:" echo " https://github.com/BeehiveInnovations/pal-mcp-server" } # Show version only show_version() { local version=$(get_version) echo "$version" } # Follow logs follow_logs() { local log_path="$LOG_DIR/$LOG_FILE" echo "Following server logs (Ctrl+C to stop)..." echo "" # Create logs directory and file if they don't exist mkdir -p "$LOG_DIR" touch "$log_path" # Follow the log file tail -f "$log_path" } # ---------------------------------------------------------------------------- # Main Function # ---------------------------------------------------------------------------- main() { # Parse command line arguments local arg="${1:-}" case "$arg" in -h|--help) show_help exit 0 ;; -v|--version) show_version exit 0 ;; -c|--config) # Setup minimal environment to get paths for config display echo "Setting up environment for configuration display..." echo "" local python_cmd python_cmd=$(setup_environment) || exit 1 local script_dir=$(get_script_dir) local server_path="$script_dir/server.py" display_config_instructions "$python_cmd" "$server_path" exit 0 ;; -f|--follow) # Continue with normal setup then follow logs ;; --clear-cache) # Clear cache and exit clear_python_cache print_success "Cache cleared successfully" echo "" echo "You can now run './run-server.sh' normally" exit 0 ;; "") # Normal setup without following logs ;; *) print_error "Unknown option: $arg" echo "" >&2 show_help exit 1 ;; esac # Display header local main_header="🤖 PAL MCP Server" echo "$main_header" printf '%*s\n' "${#main_header}" | tr ' ' '=' # Get and display version local version=$(get_version) echo "Version: $version" echo "" # Check if venv exists if [[ ! -d "$VENV_PATH" ]]; then echo "Setting up Python environment for first time..." fi # Step 1: Docker cleanup cleanup_docker # Step 1.5: Clear Python cache to prevent import issues clear_python_cache # Step 2: Setup environment file setup_env_file || exit 1 # Step 3: Source .env file if [[ -f .env ]]; then set -a source .env set +a fi # Step 4: Check API keys (non-blocking - just warn if missing) check_api_keys # Step 5: Setup Python environment (uv-first approach) local python_cmd python_cmd=$(setup_environment) || exit 1 # Step 6: Install dependencies install_dependencies "$python_cmd" || exit 1 # Step 7: Get absolute server path local script_dir=$(get_script_dir) local server_path="$script_dir/server.py" # Step 8: Display setup instructions display_setup_instructions "$python_cmd" "$server_path" # Step 9: Check Claude integrations check_claude_cli_integration "$python_cmd" "$server_path" check_claude_desktop_integration "$python_cmd" "$server_path" # Step 10: Check Gemini CLI integration check_gemini_cli_integration "$script_dir" # Step 11: Check Codex CLI integration check_codex_cli_integration # Step 12: Check Qwen CLI integration check_qwen_cli_integration "$python_cmd" "$server_path" # Step 13: Display log information echo "" echo "Logs will be written to: $script_dir/$LOG_DIR/$LOG_FILE" echo "" # Step 14: Handle command line arguments if [[ "$arg" == "-f" ]] || [[ "$arg" == "--follow" ]]; then follow_logs else echo "To follow logs: ./run-server.sh -f" echo "To show config: ./run-server.sh -c" echo "To update: git pull, then run ./run-server.sh again" echo "" echo "Happy coding! 🎉" fi } # ---------------------------------------------------------------------------- # Script Entry Point # ---------------------------------------------------------------------------- if [[ "${BASH_SOURCE[0]}" == "$0" ]]; then main "$@" fi ================================================ FILE: run_integration_tests.ps1 ================================================ <# .SYNOPSIS Integration test runner script for the PAL MCP server on Windows. .DESCRIPTION This PowerShell script prepares and runs integration tests for the PAL MCP server: - Sets up the test environment - Installs required dependencies - Runs automated integration tests - Displays test results and related logs - Allows output customization via parameters (e.g., display color) .PARAMETER Color Sets the display color for console messages (default: White). .EXAMPLE .\run_integration_tests.ps1 Prepares the environment and runs all integration tests. .\run_integration_tests.ps1 -Color Cyan Runs the tests with messages displayed in cyan. .NOTES Project Author : BeehiveInnovations Script Author : GiGiDKR (https://github.com/GiGiDKR) Date : 07-05-2025 Version : See config.py (__version__) References : https://github.com/BeehiveInnovations/pal-mcp-server #> #Requires -Version 5.1 [CmdletBinding()] param( [switch]$WithSimulator, [switch]$VerboseOutput ) # Set error action preference $ErrorActionPreference = "Stop" # Colors for output function Write-ColorText { param( [Parameter(Mandatory)] [string]$Text, [string]$Color = "White", [switch]$NoNewline ) if ($NoNewline) { Write-Host $Text -ForegroundColor $Color -NoNewline } else { Write-Host $Text -ForegroundColor $Color } } function Write-Emoji { param( [Parameter(Mandatory)] [string]$Emoji, [Parameter(Mandatory)] [string]$Text, [string]$Color = "White" ) Write-Host "$Emoji " -NoNewline Write-ColorText $Text -Color $Color } Write-Emoji "🧪" "Running Integration Tests for PAL MCP Server" -Color Cyan Write-ColorText "==============================================" -Color Cyan Write-ColorText "These tests use real API calls with your configured keys" Write-Host "" # Check for virtual environment $venvPath = ".pal_venv" $activateScript = if ($IsWindows -or $env:OS -eq "Windows_NT") { "$venvPath\Scripts\Activate.ps1" } else { "$venvPath/bin/activate" } if (Test-Path $venvPath) { Write-Emoji "✅" "Virtual environment found" -Color Green # Activate virtual environment (for PowerShell on Windows) if ($IsWindows -or $env:OS -eq "Windows_NT") { if (Test-Path "$venvPath\Scripts\Activate.ps1") { & "$venvPath\Scripts\Activate.ps1" } elseif (Test-Path "$venvPath\Scripts\activate.bat") { # Use Python directly from venv $env:PATH = "$PWD\$venvPath\Scripts;$env:PATH" } } } else { Write-Emoji "❌" "No virtual environment found!" -Color Red Write-ColorText "Please run: .\run-server.ps1 first" -Color Yellow exit 1 } # Check for .env file if (!(Test-Path ".env")) { Write-Emoji "⚠️" "Warning: No .env file found. Integration tests may fail without API keys." -Color Yellow Write-Host "" } Write-Emoji "🔑" "Checking API key availability:" -Color Cyan Write-ColorText "---------------------------------" -Color Cyan # Function to check if API key is configured function Test-ApiKey { param( [string]$KeyName ) # Check environment variable $envValue = [Environment]::GetEnvironmentVariable($KeyName) if (![string]::IsNullOrWhiteSpace($envValue)) { return $true } # Check .env file if (Test-Path ".env") { $envContent = Get-Content ".env" -ErrorAction SilentlyContinue $found = $envContent | Where-Object { $_ -match "^$KeyName\s*=" -and $_ -notmatch "^$KeyName\s*=\s*$" } return $found.Count -gt 0 } return $false } # Check API keys $apiKeys = @( "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "CUSTOM_API_URL" ) foreach ($key in $apiKeys) { if (Test-ApiKey $key) { if ($key -eq "CUSTOM_API_URL") { Write-Emoji "✅" "$key configured (local models)" -Color Green } else { Write-Emoji "✅" "$key configured" -Color Green } } else { Write-Emoji "❌" "$key not found" -Color Red } } Write-Host "" # Load environment variables from .env if it exists if (Test-Path ".env") { Get-Content ".env" | ForEach-Object { if ($_ -match '^([^#][^=]*?)=(.*)$') { $name = $matches[1].Trim() $value = $matches[2].Trim() # Remove quotes if present $value = $value -replace '^["'']|["'']$', '' [Environment]::SetEnvironmentVariable($name, $value, "Process") } } } # Run integration tests Write-Emoji "🏃" "Running integration tests..." -Color Cyan Write-ColorText "------------------------------" -Color Cyan try { # Build pytest command $pytestArgs = @("tests/", "-v", "-m", "integration", "--tb=short") if ($VerboseOutput) { $pytestArgs += "--verbose" } # Run pytest python -m pytest @pytestArgs if ($LASTEXITCODE -ne 0) { throw "Integration tests failed" } Write-Host "" Write-Emoji "✅" "Integration tests completed!" -Color Green } catch { Write-Host "" Write-Emoji "❌" "Integration tests failed!" -Color Red Write-ColorText "Error: $_" -Color Red exit 1 } # Run simulator tests if requested if ($WithSimulator) { Write-Host "" Write-Emoji "🤖" "Running simulator tests..." -Color Cyan Write-ColorText "----------------------------" -Color Cyan try { if ($VerboseOutput) { python communication_simulator_test.py --verbose } else { python communication_simulator_test.py } if ($LASTEXITCODE -ne 0) { Write-Host "" Write-Emoji "❌" "Simulator tests failed!" -Color Red Write-ColorText "This may be due to a known issue in communication_simulator_test.py" -Color Yellow Write-ColorText "Integration tests completed successfully - you can proceed." -Color Green } else { Write-Host "" Write-Emoji "✅" "Simulator tests completed!" -Color Green } } catch { Write-Host "" Write-Emoji "❌" "Simulator tests failed!" -Color Red Write-ColorText "Error: $_" -Color Red Write-ColorText "This may be due to a known issue in communication_simulator_test.py" -Color Yellow Write-ColorText "Integration tests completed successfully - you can proceed." -Color Green } } Write-Host "" Write-Emoji "💡" "Tips:" -Color Yellow Write-ColorText "- Run '.\run_integration_tests.ps1' for integration tests only" -Color White Write-ColorText "- Run '.\run_integration_tests.ps1 -WithSimulator' to also run simulator tests" -Color White Write-ColorText "- Run '.\code_quality_checks.ps1' for unit tests and linting" -Color White Write-ColorText "- Check logs in logs\mcp_server.log if tests fail" -Color White ================================================ FILE: run_integration_tests.sh ================================================ #!/bin/bash # PAL MCP Server - Run Integration Tests # This script runs integration tests that require API keys # Run this locally on your Mac to ensure everything works end-to-end set -e # Exit on any error echo "🧪 Running Integration Tests for PAL MCP Server" echo "==============================================" echo "These tests use real API calls with your configured keys" echo "" # Activate virtual environment if [[ -f ".pal_venv/bin/activate" ]]; then source .pal_venv/bin/activate echo "✅ Using virtual environment" else echo "❌ No virtual environment found!" echo "Please run: ./run-server.sh first" exit 1 fi # Check for .env file if [[ ! -f ".env" ]]; then echo "⚠️ Warning: No .env file found. Integration tests may fail without API keys." echo "" fi echo "🔑 Checking API key availability:" echo "---------------------------------" # Check which API keys are available if [[ -n "$GEMINI_API_KEY" ]] || grep -q "GEMINI_API_KEY=" .env 2>/dev/null; then echo "✅ GEMINI_API_KEY configured" else echo "❌ GEMINI_API_KEY not found" fi if [[ -n "$OPENAI_API_KEY" ]] || grep -q "OPENAI_API_KEY=" .env 2>/dev/null; then echo "✅ OPENAI_API_KEY configured" else echo "❌ OPENAI_API_KEY not found" fi if [[ -n "$XAI_API_KEY" ]] || grep -q "XAI_API_KEY=" .env 2>/dev/null; then echo "✅ XAI_API_KEY configured" else echo "❌ XAI_API_KEY not found" fi if [[ -n "$OPENROUTER_API_KEY" ]] || grep -q "OPENROUTER_API_KEY=" .env 2>/dev/null; then echo "✅ OPENROUTER_API_KEY configured" else echo "❌ OPENROUTER_API_KEY not found" fi if [[ -n "$CUSTOM_API_URL" ]] || grep -q "CUSTOM_API_URL=" .env 2>/dev/null; then echo "✅ CUSTOM_API_URL configured (local models)" else echo "❌ CUSTOM_API_URL not found" fi echo "" # Run integration tests echo "🏃 Running integration tests..." echo "------------------------------" # Run only integration tests (marked with @pytest.mark.integration) python -m pytest tests/ -v -m "integration" --tb=short echo "" echo "✅ Integration tests completed!" echo "" # Also run simulator tests if requested if [[ "$1" == "--with-simulator" ]]; then echo "🤖 Running simulator tests..." echo "----------------------------" python communication_simulator_test.py --verbose echo "" echo "✅ Simulator tests completed!" fi echo "💡 Tips:" echo "- Run './run_integration_tests.sh' for integration tests only" echo "- Run './run_integration_tests.sh --with-simulator' to also run simulator tests" echo "- Run './code_quality_checks.sh' for unit tests and linting" echo "- Check logs in logs/mcp_server.log if tests fail" ================================================ FILE: scripts/sync_version.py ================================================ #!/usr/bin/env python3 """ Sync version from pyproject.toml to config.py This script is called by GitHub Actions after semantic-release updates the version """ import re from datetime import datetime import toml def update_config_version(): # Read version from pyproject.toml with open("pyproject.toml") as f: data = toml.load(f) version = data["project"]["version"] # Read current config.py with open("config.py") as f: content = f.read() # Update version content = re.sub(r'__version__ = "[^"]*"', f'__version__ = "{version}"', content) # Update date to current date today = datetime.now().strftime("%Y-%m-%d") content = re.sub(r'__updated__ = "[^"]*"', f'__updated__ = "{today}"', content) # Write back with open("config.py", "w") as f: f.write(content) print(f"Updated config.py to version {version}") if __name__ == "__main__": update_config_version() ================================================ FILE: server.py ================================================ """ PAL MCP Server - Main server implementation This module implements the core MCP (Model Context Protocol) server that provides AI-powered tools for code analysis, review, and assistance using multiple AI models. The server follows the MCP specification to expose various AI tools as callable functions that can be used by MCP clients (like Claude). Each tool provides specialized functionality such as code review, debugging, deep thinking, and general chat capabilities. Key Components: - MCP Server: Handles protocol communication and tool discovery - Tool Registry: Maps tool names to their implementations - Request Handler: Processes incoming tool calls and returns formatted responses - Configuration: Manages API keys and model settings The server runs on stdio (standard input/output) and communicates using JSON-RPC messages as defined by the MCP protocol. """ import asyncio import atexit import logging import os import sys import time from logging.handlers import RotatingFileHandler from pathlib import Path from typing import Any, Optional from mcp.server import Server # noqa: E402 from mcp.server.models import InitializationOptions # noqa: E402 from mcp.server.stdio import stdio_server # noqa: E402 from mcp.types import ( # noqa: E402 GetPromptResult, Prompt, PromptMessage, PromptsCapability, ServerCapabilities, TextContent, Tool, ToolAnnotations, ToolsCapability, ) from config import ( # noqa: E402 DEFAULT_MODEL, __version__, ) from tools import ( # noqa: E402 AnalyzeTool, ChallengeTool, ChatTool, CLinkTool, CodeReviewTool, ConsensusTool, DebugIssueTool, DocgenTool, ListModelsTool, LookupTool, PlannerTool, PrecommitTool, RefactorTool, SecauditTool, TestGenTool, ThinkDeepTool, TracerTool, VersionTool, ) from tools.models import ToolOutput # noqa: E402 from tools.shared.exceptions import ToolExecutionError # noqa: E402 from utils.env import env_override_enabled, get_env # noqa: E402 # Configure logging for server operations # Can be controlled via LOG_LEVEL environment variable (DEBUG, INFO, WARNING, ERROR) log_level = (get_env("LOG_LEVEL", "DEBUG") or "DEBUG").upper() # Create timezone-aware formatter class LocalTimeFormatter(logging.Formatter): def formatTime(self, record, datefmt=None): """Override to use local timezone instead of UTC""" ct = self.converter(record.created) if datefmt: s = time.strftime(datefmt, ct) else: t = time.strftime("%Y-%m-%d %H:%M:%S", ct) s = f"{t},{record.msecs:03.0f}" return s # Configure both console and file logging log_format = "%(asctime)s - %(name)s - %(levelname)s - %(message)s" # Clear any existing handlers first root_logger = logging.getLogger() root_logger.handlers.clear() # Create and configure stderr handler explicitly stderr_handler = logging.StreamHandler(sys.stderr) stderr_handler.setLevel(getattr(logging, log_level, logging.INFO)) stderr_handler.setFormatter(LocalTimeFormatter(log_format)) root_logger.addHandler(stderr_handler) # Note: MCP stdio_server interferes with stderr during tool execution # All logs are properly written to logs/mcp_server.log for monitoring # Set root logger level root_logger.setLevel(getattr(logging, log_level, logging.INFO)) # Add rotating file handler for local log monitoring try: # Create logs directory in project root log_dir = Path(__file__).parent / "logs" log_dir.mkdir(exist_ok=True) # Main server log with size-based rotation (20MB max per file) # This ensures logs don't grow indefinitely and are properly managed file_handler = RotatingFileHandler( log_dir / "mcp_server.log", maxBytes=20 * 1024 * 1024, # 20MB max file size backupCount=5, # Keep 10 rotated files (100MB total) encoding="utf-8", ) file_handler.setLevel(getattr(logging, log_level, logging.INFO)) file_handler.setFormatter(LocalTimeFormatter(log_format)) logging.getLogger().addHandler(file_handler) # Create a special logger for MCP activity tracking with size-based rotation mcp_logger = logging.getLogger("mcp_activity") mcp_file_handler = RotatingFileHandler( log_dir / "mcp_activity.log", maxBytes=10 * 1024 * 1024, # 20MB max file size backupCount=2, # Keep 5 rotated files (20MB total) encoding="utf-8", ) mcp_file_handler.setLevel(logging.INFO) mcp_file_handler.setFormatter(LocalTimeFormatter("%(asctime)s - %(message)s")) mcp_logger.addHandler(mcp_file_handler) mcp_logger.setLevel(logging.INFO) # Ensure MCP activity also goes to stderr mcp_logger.propagate = True # Log setup info directly to root logger since logger isn't defined yet logging.info(f"Logging to: {log_dir / 'mcp_server.log'}") logging.info(f"Process PID: {os.getpid()}") except Exception as e: print(f"Warning: Could not set up file logging: {e}", file=sys.stderr) logger = logging.getLogger(__name__) # Log PAL_MCP_FORCE_ENV_OVERRIDE configuration for transparency if env_override_enabled(): logger.info("PAL_MCP_FORCE_ENV_OVERRIDE enabled - .env file values will override system environment variables") logger.debug("Environment override prevents conflicts between different AI tools passing cached API keys") else: logger.debug("PAL_MCP_FORCE_ENV_OVERRIDE disabled - system environment variables take precedence") # Create the MCP server instance with a unique name identifier # This name is used by MCP clients to identify and connect to this specific server server: Server = Server("pal-server") # Constants for tool filtering ESSENTIAL_TOOLS = {"version", "listmodels"} def parse_disabled_tools_env() -> set[str]: """ Parse the DISABLED_TOOLS environment variable into a set of tool names. Returns: Set of lowercase tool names to disable, empty set if none specified """ disabled_tools_env = (get_env("DISABLED_TOOLS", "") or "").strip() if not disabled_tools_env: return set() return {t.strip().lower() for t in disabled_tools_env.split(",") if t.strip()} def validate_disabled_tools(disabled_tools: set[str], all_tools: dict[str, Any]) -> None: """ Validate the disabled tools list and log appropriate warnings. Args: disabled_tools: Set of tool names requested to be disabled all_tools: Dictionary of all available tool instances """ essential_disabled = disabled_tools & ESSENTIAL_TOOLS if essential_disabled: logger.warning(f"Cannot disable essential tools: {sorted(essential_disabled)}") unknown_tools = disabled_tools - set(all_tools.keys()) if unknown_tools: logger.warning(f"Unknown tools in DISABLED_TOOLS: {sorted(unknown_tools)}") def apply_tool_filter(all_tools: dict[str, Any], disabled_tools: set[str]) -> dict[str, Any]: """ Apply the disabled tools filter to create the final tools dictionary. Args: all_tools: Dictionary of all available tool instances disabled_tools: Set of tool names to disable Returns: Dictionary containing only enabled tools """ enabled_tools = {} for tool_name, tool_instance in all_tools.items(): if tool_name in ESSENTIAL_TOOLS or tool_name not in disabled_tools: enabled_tools[tool_name] = tool_instance else: logger.debug(f"Tool '{tool_name}' disabled via DISABLED_TOOLS") return enabled_tools def log_tool_configuration(disabled_tools: set[str], enabled_tools: dict[str, Any]) -> None: """ Log the final tool configuration for visibility. Args: disabled_tools: Set of tool names that were requested to be disabled enabled_tools: Dictionary of tools that remain enabled """ if not disabled_tools: logger.info("All tools enabled (DISABLED_TOOLS not set)") return actual_disabled = disabled_tools - ESSENTIAL_TOOLS if actual_disabled: logger.debug(f"Disabled tools: {sorted(actual_disabled)}") logger.info(f"Active tools: {sorted(enabled_tools.keys())}") def filter_disabled_tools(all_tools: dict[str, Any]) -> dict[str, Any]: """ Filter tools based on DISABLED_TOOLS environment variable. Args: all_tools: Dictionary of all available tool instances Returns: dict: Filtered dictionary containing only enabled tools """ disabled_tools = parse_disabled_tools_env() if not disabled_tools: log_tool_configuration(disabled_tools, all_tools) return all_tools validate_disabled_tools(disabled_tools, all_tools) enabled_tools = apply_tool_filter(all_tools, disabled_tools) log_tool_configuration(disabled_tools, enabled_tools) return enabled_tools # Initialize the tool registry with all available AI-powered tools # Each tool provides specialized functionality for different development tasks # Tools are instantiated once and reused across requests (stateless design) TOOLS = { "chat": ChatTool(), # Interactive development chat and brainstorming "clink": CLinkTool(), # Bridge requests to configured AI CLIs "thinkdeep": ThinkDeepTool(), # Step-by-step deep thinking workflow with expert analysis "planner": PlannerTool(), # Interactive sequential planner using workflow architecture "consensus": ConsensusTool(), # Step-by-step consensus workflow with multi-model analysis "codereview": CodeReviewTool(), # Comprehensive step-by-step code review workflow with expert analysis "precommit": PrecommitTool(), # Step-by-step pre-commit validation workflow "debug": DebugIssueTool(), # Root cause analysis and debugging assistance "secaudit": SecauditTool(), # Comprehensive security audit with OWASP Top 10 and compliance coverage "docgen": DocgenTool(), # Step-by-step documentation generation with complexity analysis "analyze": AnalyzeTool(), # General-purpose file and code analysis "refactor": RefactorTool(), # Step-by-step refactoring analysis workflow with expert validation "tracer": TracerTool(), # Static call path prediction and control flow analysis "testgen": TestGenTool(), # Step-by-step test generation workflow with expert validation "challenge": ChallengeTool(), # Critical challenge prompt wrapper to avoid automatic agreement "apilookup": LookupTool(), # Quick web/API lookup instructions "listmodels": ListModelsTool(), # List all available AI models by provider "version": VersionTool(), # Display server version and system information } TOOLS = filter_disabled_tools(TOOLS) # Rich prompt templates for all tools PROMPT_TEMPLATES = { "chat": { "name": "chat", "description": "Chat and brainstorm ideas", "template": "Chat with {model} about this", }, "clink": { "name": "clink", "description": "Forward a request to a configured AI CLI (e.g., Gemini)", "template": "Use clink with cli_name= to run this prompt", }, "thinkdeep": { "name": "thinkdeeper", "description": "Step-by-step deep thinking workflow with expert analysis", "template": "Start comprehensive deep thinking workflow with {model} using {thinking_mode} thinking mode", }, "planner": { "name": "planner", "description": "Break down complex ideas, problems, or projects into multiple manageable steps", "template": "Create a detailed plan with {model}", }, "consensus": { "name": "consensus", "description": "Step-by-step consensus workflow with multi-model analysis", "template": "Start comprehensive consensus workflow with {model}", }, "codereview": { "name": "review", "description": "Perform a comprehensive code review", "template": "Perform a comprehensive code review with {model}", }, "precommit": { "name": "precommit", "description": "Step-by-step pre-commit validation workflow", "template": "Start comprehensive pre-commit validation workflow with {model}", }, "debug": { "name": "debug", "description": "Debug an issue or error", "template": "Help debug this issue with {model}", }, "secaudit": { "name": "secaudit", "description": "Comprehensive security audit with OWASP Top 10 coverage", "template": "Perform comprehensive security audit with {model}", }, "docgen": { "name": "docgen", "description": "Generate comprehensive code documentation with complexity analysis", "template": "Generate comprehensive documentation with {model}", }, "analyze": { "name": "analyze", "description": "Analyze files and code structure", "template": "Analyze these files with {model}", }, "refactor": { "name": "refactor", "description": "Refactor and improve code structure", "template": "Refactor this code with {model}", }, "tracer": { "name": "tracer", "description": "Trace code execution paths", "template": "Generate tracer analysis with {model}", }, "testgen": { "name": "testgen", "description": "Generate comprehensive tests", "template": "Generate comprehensive tests with {model}", }, "challenge": { "name": "challenge", "description": "Challenge a statement critically without automatic agreement", "template": "Challenge this statement critically", }, "apilookup": { "name": "apilookup", "description": "Look up the latest API or SDK information", "template": "Lookup latest API docs for {model}", }, "listmodels": { "name": "listmodels", "description": "List available AI models", "template": "List all available models", }, "version": { "name": "version", "description": "Show server version and system information", "template": "Show PAL MCP Server version", }, } def configure_providers(): """ Configure and validate AI providers based on available API keys. This function checks for API keys and registers the appropriate providers. At least one valid API key (Gemini or OpenAI) is required. Raises: ValueError: If no valid API keys are found or conflicting configurations detected """ # Log environment variable status for debugging logger.debug("Checking environment variables for API keys...") api_keys_to_check = ["OPENAI_API_KEY", "OPENROUTER_API_KEY", "GEMINI_API_KEY", "XAI_API_KEY", "CUSTOM_API_URL"] for key in api_keys_to_check: value = get_env(key) logger.debug(f" {key}: {'[PRESENT]' if value else '[MISSING]'}") from providers import ModelProviderRegistry from providers.azure_openai import AzureOpenAIProvider from providers.custom import CustomProvider from providers.dial import DIALModelProvider from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.openrouter import OpenRouterProvider from providers.shared import ProviderType from providers.xai import XAIModelProvider from utils.model_restrictions import get_restriction_service valid_providers = [] has_native_apis = False has_openrouter = False has_custom = False # Check for Gemini API key gemini_key = get_env("GEMINI_API_KEY") if gemini_key and gemini_key != "your_gemini_api_key_here": valid_providers.append("Gemini") has_native_apis = True logger.info("Gemini API key found - Gemini models available") # Check for OpenAI API key openai_key = get_env("OPENAI_API_KEY") logger.debug(f"OpenAI key check: key={'[PRESENT]' if openai_key else '[MISSING]'}") if openai_key and openai_key != "your_openai_api_key_here": valid_providers.append("OpenAI") has_native_apis = True logger.info("OpenAI API key found") else: if not openai_key: logger.debug("OpenAI API key not found in environment") else: logger.debug("OpenAI API key is placeholder value") # Check for Azure OpenAI configuration azure_key = get_env("AZURE_OPENAI_API_KEY") azure_endpoint = get_env("AZURE_OPENAI_ENDPOINT") azure_models_available = False if azure_key and azure_key != "your_azure_openai_key_here" and azure_endpoint: try: from providers.registries.azure import AzureModelRegistry azure_registry = AzureModelRegistry() if azure_registry.list_models(): valid_providers.append("Azure OpenAI") has_native_apis = True azure_models_available = True logger.info("Azure OpenAI configuration detected") else: logger.warning( "Azure OpenAI models configuration is empty. Populate conf/azure_models.json or set AZURE_MODELS_CONFIG_PATH." ) except Exception as exc: logger.warning(f"Failed to load Azure OpenAI models: {exc}") # Check for X.AI API key xai_key = get_env("XAI_API_KEY") if xai_key and xai_key != "your_xai_api_key_here": valid_providers.append("X.AI (GROK)") has_native_apis = True logger.info("X.AI API key found - GROK models available") # Check for DIAL API key dial_key = get_env("DIAL_API_KEY") if dial_key and dial_key != "your_dial_api_key_here": valid_providers.append("DIAL") has_native_apis = True logger.info("DIAL API key found - DIAL models available") # Check for OpenRouter API key openrouter_key = get_env("OPENROUTER_API_KEY") logger.debug(f"OpenRouter key check: key={'[PRESENT]' if openrouter_key else '[MISSING]'}") if openrouter_key and openrouter_key != "your_openrouter_api_key_here": valid_providers.append("OpenRouter") has_openrouter = True logger.info("OpenRouter API key found - Multiple models available via OpenRouter") else: if not openrouter_key: logger.debug("OpenRouter API key not found in environment") else: logger.debug("OpenRouter API key is placeholder value") # Check for custom API endpoint (Ollama, vLLM, etc.) custom_url = get_env("CUSTOM_API_URL") if custom_url: # IMPORTANT: Always read CUSTOM_API_KEY even if empty # - Some providers (vLLM, LM Studio, enterprise APIs) require authentication # - Others (Ollama) work without authentication (empty key) # - DO NOT remove this variable - it's needed for provider factory function custom_key = get_env("CUSTOM_API_KEY", "") or "" # Default to empty (Ollama doesn't need auth) custom_model = get_env("CUSTOM_MODEL_NAME", "llama3.2") or "llama3.2" valid_providers.append(f"Custom API ({custom_url})") has_custom = True logger.info(f"Custom API endpoint found: {custom_url} with model {custom_model}") if custom_key: logger.debug("Custom API key provided for authentication") else: logger.debug("No custom API key provided (using unauthenticated access)") # Register providers in priority order: # 1. Native APIs first (most direct and efficient) registered_providers = [] if has_native_apis: if gemini_key and gemini_key != "your_gemini_api_key_here": ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) registered_providers.append(ProviderType.GOOGLE.value) logger.debug(f"Registered provider: {ProviderType.GOOGLE.value}") if openai_key and openai_key != "your_openai_api_key_here": ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) registered_providers.append(ProviderType.OPENAI.value) logger.debug(f"Registered provider: {ProviderType.OPENAI.value}") if azure_models_available: ModelProviderRegistry.register_provider(ProviderType.AZURE, AzureOpenAIProvider) registered_providers.append(ProviderType.AZURE.value) logger.debug(f"Registered provider: {ProviderType.AZURE.value}") if xai_key and xai_key != "your_xai_api_key_here": ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) registered_providers.append(ProviderType.XAI.value) logger.debug(f"Registered provider: {ProviderType.XAI.value}") if dial_key and dial_key != "your_dial_api_key_here": ModelProviderRegistry.register_provider(ProviderType.DIAL, DIALModelProvider) registered_providers.append(ProviderType.DIAL.value) logger.debug(f"Registered provider: {ProviderType.DIAL.value}") # 2. Custom provider second (for local/private models) if has_custom: # Factory function that creates CustomProvider with proper parameters def custom_provider_factory(api_key=None): # api_key is CUSTOM_API_KEY (can be empty for Ollama), base_url from CUSTOM_API_URL base_url = get_env("CUSTOM_API_URL", "") or "" return CustomProvider(api_key=api_key or "", base_url=base_url) # Use provided API key or empty string ModelProviderRegistry.register_provider(ProviderType.CUSTOM, custom_provider_factory) registered_providers.append(ProviderType.CUSTOM.value) logger.debug(f"Registered provider: {ProviderType.CUSTOM.value}") # 3. OpenRouter last (catch-all for everything else) if has_openrouter: ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) registered_providers.append(ProviderType.OPENROUTER.value) logger.debug(f"Registered provider: {ProviderType.OPENROUTER.value}") # Log all registered providers if registered_providers: logger.info(f"Registered providers: {', '.join(registered_providers)}") # Require at least one valid provider if not valid_providers: raise ValueError( "At least one API configuration is required. Please set either:\n" "- GEMINI_API_KEY for Gemini models\n" "- OPENAI_API_KEY for OpenAI models\n" "- XAI_API_KEY for X.AI GROK models\n" "- DIAL_API_KEY for DIAL models\n" "- OPENROUTER_API_KEY for OpenRouter (multiple models)\n" "- CUSTOM_API_URL for local models (Ollama, vLLM, etc.)" ) logger.info(f"Available providers: {', '.join(valid_providers)}") # Log provider priority priority_info = [] if has_native_apis: priority_info.append("Native APIs (Gemini, OpenAI)") if has_custom: priority_info.append("Custom endpoints") if has_openrouter: priority_info.append("OpenRouter (catch-all)") if len(priority_info) > 1: logger.info(f"Provider priority: {' → '.join(priority_info)}") # Register cleanup function for providers def cleanup_providers(): """Clean up all registered providers on shutdown.""" try: registry = ModelProviderRegistry() if hasattr(registry, "_initialized_providers"): # Iterate over provider instances (values), not (type, instance) tuples for provider in list(registry._initialized_providers.values()): try: if provider and hasattr(provider, "close"): provider.close() except Exception: # Logger might be closed during shutdown pass except Exception: # Silently ignore any errors during cleanup pass atexit.register(cleanup_providers) # Check and log model restrictions restriction_service = get_restriction_service() restrictions = restriction_service.get_restriction_summary() if restrictions: logger.info("Model restrictions configured:") for provider_name, allowed_models in restrictions.items(): if isinstance(allowed_models, list): logger.info(f" {provider_name}: {', '.join(allowed_models)}") else: logger.info(f" {provider_name}: {allowed_models}") # Validate restrictions against known models provider_instances = {} provider_types_to_validate = [ProviderType.GOOGLE, ProviderType.OPENAI, ProviderType.XAI, ProviderType.DIAL] for provider_type in provider_types_to_validate: provider = ModelProviderRegistry.get_provider(provider_type) if provider: provider_instances[provider_type] = provider if provider_instances: restriction_service.validate_against_known_models(provider_instances) else: logger.info("No model restrictions configured - all models allowed") # Check if auto mode has any models available after restrictions from config import IS_AUTO_MODE if IS_AUTO_MODE: available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) if not available_models: logger.error( "Auto mode is enabled but no models are available after applying restrictions. " "Please check your OPENAI_ALLOWED_MODELS and GOOGLE_ALLOWED_MODELS settings." ) raise ValueError( "No models available for auto mode due to restrictions. " "Please adjust your allowed model settings or disable auto mode." ) @server.list_tools() async def handle_list_tools() -> list[Tool]: """ List all available tools with their descriptions and input schemas. This handler is called by MCP clients during initialization to discover what tools are available. Each tool provides: - name: Unique identifier for the tool - description: Detailed explanation of what the tool does - inputSchema: JSON Schema defining the expected parameters Returns: List of Tool objects representing all available tools """ logger.debug("MCP client requested tool list") # Try to log client info if available (this happens early in the handshake) try: from utils.client_info import format_client_info, get_client_info_from_context client_info = get_client_info_from_context(server) if client_info: formatted = format_client_info(client_info) logger.info(f"MCP Client Connected: {formatted}") # Log to activity file as well try: mcp_activity_logger = logging.getLogger("mcp_activity") friendly_name = client_info.get("friendly_name", "CLI Agent") raw_name = client_info.get("name", "Unknown") version = client_info.get("version", "Unknown") mcp_activity_logger.info(f"MCP_CLIENT_INFO: {friendly_name} (raw={raw_name} v{version})") except Exception: pass except Exception as e: logger.debug(f"Could not log client info during list_tools: {e}") tools = [] # Add all registered AI-powered tools from the TOOLS registry for tool in TOOLS.values(): # Get optional annotations from the tool annotations = tool.get_annotations() tool_annotations = ToolAnnotations(**annotations) if annotations else None tools.append( Tool( name=tool.name, description=tool.description, inputSchema=tool.get_input_schema(), annotations=tool_annotations, ) ) # Log cache efficiency info openrouter_key_for_cache = get_env("OPENROUTER_API_KEY") if openrouter_key_for_cache and openrouter_key_for_cache != "your_openrouter_api_key_here": logger.debug("OpenRouter registry cache used efficiently across all tool schemas") logger.debug(f"Returning {len(tools)} tools to MCP client") return tools @server.call_tool() async def handle_call_tool(name: str, arguments: dict[str, Any]) -> list[TextContent]: """ Handle incoming tool execution requests from MCP clients. This is the main request dispatcher that routes tool calls to their appropriate handlers. It supports both AI-powered tools (from TOOLS registry) and utility tools (implemented as static functions). CONVERSATION LIFECYCLE MANAGEMENT: This function serves as the central orchestrator for multi-turn AI-to-AI conversations: 1. THREAD RESUMPTION: When continuation_id is present, it reconstructs complete conversation context from in-memory storage including conversation history and file references 2. CROSS-TOOL CONTINUATION: Enables seamless handoffs between different tools (analyze → codereview → debug) while preserving full conversation context and file references 3. CONTEXT INJECTION: Reconstructed conversation history is embedded into tool prompts using the dual prioritization strategy: - Files: Newest-first prioritization (recent file versions take precedence) - Turns: Newest-first collection for token efficiency, chronological presentation for LLM 4. FOLLOW-UP GENERATION: After tool execution, generates continuation offers for ongoing AI-to-AI collaboration with natural language instructions STATELESS TO STATEFUL BRIDGE: The MCP protocol is inherently stateless, but this function bridges the gap by: - Loading persistent conversation state from in-memory storage - Reconstructing full multi-turn context for tool execution - Enabling tools to access previous exchanges and file references - Supporting conversation chains across different tool types Args: name: The name of the tool to execute (e.g., "analyze", "chat", "codereview") arguments: Dictionary of arguments to pass to the tool, potentially including: - continuation_id: UUID for conversation thread resumption - files: File paths for analysis (subject to deduplication) - prompt: User request or follow-up question - model: Specific AI model to use (optional) Returns: List of TextContent objects containing: - Tool's primary response with analysis/results - Continuation offers for follow-up conversations (when applicable) - Structured JSON responses with status and content Raises: ValueError: If continuation_id is invalid or conversation thread not found Exception: For tool-specific errors or execution failures Example Conversation Flow: 1. The CLI calls analyze tool with files → creates new thread 2. Thread ID returned in continuation offer 3. The CLI continues with codereview tool + continuation_id → full context preserved 4. Multiple tools can collaborate using same thread ID """ logger.info(f"MCP tool call: {name}") logger.debug(f"MCP tool arguments: {list(arguments.keys())}") # Log to activity file for monitoring try: mcp_activity_logger = logging.getLogger("mcp_activity") mcp_activity_logger.info(f"TOOL_CALL: {name} with {len(arguments)} arguments") except Exception: pass # Handle thread context reconstruction if continuation_id is present if "continuation_id" in arguments and arguments["continuation_id"]: continuation_id = arguments["continuation_id"] logger.debug(f"Resuming conversation thread: {continuation_id}") logger.debug( f"[CONVERSATION_DEBUG] Tool '{name}' resuming thread {continuation_id} with {len(arguments)} arguments" ) logger.debug(f"[CONVERSATION_DEBUG] Original arguments keys: {list(arguments.keys())}") # Log to activity file for monitoring try: mcp_activity_logger = logging.getLogger("mcp_activity") mcp_activity_logger.info(f"CONVERSATION_RESUME: {name} resuming thread {continuation_id}") except Exception: pass arguments = await reconstruct_thread_context(arguments) logger.debug(f"[CONVERSATION_DEBUG] After thread reconstruction, arguments keys: {list(arguments.keys())}") if "_remaining_tokens" in arguments: logger.debug(f"[CONVERSATION_DEBUG] Remaining token budget: {arguments['_remaining_tokens']:,}") # Route to AI-powered tools that require Gemini API calls if name in TOOLS: logger.info(f"Executing tool '{name}' with {len(arguments)} parameter(s)") tool = TOOLS[name] # EARLY MODEL RESOLUTION AT MCP BOUNDARY # Resolve model before passing to tool - this ensures consistent model handling # NOTE: Consensus tool is exempt as it handles multiple models internally from providers.registry import ModelProviderRegistry from utils.file_utils import check_total_file_size from utils.model_context import ModelContext # Get model from arguments or use default model_name = arguments.get("model") or DEFAULT_MODEL logger.debug(f"Initial model for {name}: {model_name}") # Parse model:option format if present model_name, model_option = parse_model_option(model_name) if model_option: logger.info(f"Parsed model format - model: '{model_name}', option: '{model_option}'") else: logger.info(f"Parsed model format - model: '{model_name}'") # Consensus tool handles its own model configuration validation # No special handling needed at server level # Skip model resolution for tools that don't require models (e.g., planner) if not tool.requires_model(): logger.debug(f"Tool {name} doesn't require model resolution - skipping model validation") # Execute tool directly without model context return await tool.execute(arguments) # Handle auto mode at MCP boundary - resolve to specific model if model_name.lower() == "auto": # Get tool category to determine appropriate model tool_category = tool.get_model_category() resolved_model = ModelProviderRegistry.get_preferred_fallback_model(tool_category) logger.info(f"Auto mode resolved to {resolved_model} for {name} (category: {tool_category.value})") model_name = resolved_model # Update arguments with resolved model arguments["model"] = model_name # Validate model availability at MCP boundary provider = ModelProviderRegistry.get_provider_for_model(model_name) if not provider: # Get list of available models for error message available_models = list(ModelProviderRegistry.get_available_models(respect_restrictions=True).keys()) tool_category = tool.get_model_category() suggested_model = ModelProviderRegistry.get_preferred_fallback_model(tool_category) error_message = ( f"Model '{model_name}' is not available with current API keys. " f"Available models: {', '.join(available_models)}. " f"Suggested model for {name}: '{suggested_model}' " f"(category: {tool_category.value})" ) error_output = ToolOutput( status="error", content=error_message, content_type="text", metadata={"tool_name": name, "requested_model": model_name}, ) raise ToolExecutionError(error_output.model_dump_json()) # Create model context with resolved model and option model_context = ModelContext(model_name, model_option) arguments["_model_context"] = model_context arguments["_resolved_model_name"] = model_name logger.debug( f"Model context created for {model_name} with {model_context.capabilities.context_window} token capacity" ) if model_option: logger.debug(f"Model option stored in context: '{model_option}'") # EARLY FILE SIZE VALIDATION AT MCP BOUNDARY # Check file sizes before tool execution using resolved model argument_files = arguments.get("absolute_file_paths") if argument_files: logger.debug(f"Checking file sizes for {len(argument_files)} files with model {model_name}") file_size_check = check_total_file_size(argument_files, model_name) if file_size_check: logger.warning(f"File size check failed for {name} with model {model_name}") raise ToolExecutionError(ToolOutput(**file_size_check).model_dump_json()) # Execute tool with pre-resolved model context result = await tool.execute(arguments) logger.info(f"Tool '{name}' execution completed") # Log completion to activity file try: mcp_activity_logger = logging.getLogger("mcp_activity") mcp_activity_logger.info(f"TOOL_COMPLETED: {name}") except Exception: pass return result # Handle unknown tool requests gracefully else: return [TextContent(type="text", text=f"Unknown tool: {name}")] def parse_model_option(model_string: str) -> tuple[str, Optional[str]]: """ Parse model:option format into model name and option. Handles different formats: - OpenRouter models: preserve :free, :beta, :preview suffixes as part of model name - Ollama/Custom models: split on : to extract tags like :latest - Consensus stance: extract options like :for, :against Args: model_string: String that may contain "model:option" format Returns: tuple: (model_name, option) where option may be None """ if ":" in model_string and not model_string.startswith("http"): # Avoid parsing URLs # Check if this looks like an OpenRouter model (contains /) if "/" in model_string and model_string.count(":") == 1: # Could be openai/gpt-4:something - check what comes after colon parts = model_string.split(":", 1) suffix = parts[1].strip().lower() # Known OpenRouter suffixes to preserve if suffix in ["free", "beta", "preview"]: return model_string.strip(), None # For other patterns (Ollama tags, consensus stances), split normally parts = model_string.split(":", 1) model_name = parts[0].strip() model_option = parts[1].strip() if len(parts) > 1 else None return model_name, model_option return model_string.strip(), None def get_follow_up_instructions(current_turn_count: int, max_turns: int = None) -> str: """ Generate dynamic follow-up instructions based on conversation turn count. Args: current_turn_count: Current number of turns in the conversation max_turns: Maximum allowed turns before conversation ends (defaults to MAX_CONVERSATION_TURNS) Returns: Follow-up instructions to append to the tool prompt """ if max_turns is None: from utils.conversation_memory import MAX_CONVERSATION_TURNS max_turns = MAX_CONVERSATION_TURNS if current_turn_count >= max_turns - 1: # We're at or approaching the turn limit - no more follow-ups return """ IMPORTANT: This is approaching the final exchange in this conversation thread. Do NOT include any follow-up questions in your response. Provide your complete final analysis and recommendations.""" else: # Normal follow-up instructions remaining_turns = max_turns - current_turn_count - 1 return f""" CONVERSATION CONTINUATION: You can continue this discussion with the agent! ({remaining_turns} exchanges remaining) Feel free to ask clarifying questions or suggest areas for deeper exploration naturally within your response. If something needs clarification or you'd benefit from additional context, simply mention it conversationally. IMPORTANT: When you suggest follow-ups or ask questions, you MUST explicitly instruct the agent to use the continuation_id to respond. Use clear, direct language based on urgency: For optional follow-ups: "Please continue this conversation using the continuation_id from this response if you'd " "like to explore this further." For needed responses: "Please respond using the continuation_id from this response - your input is needed to proceed." For essential/critical responses: "RESPONSE REQUIRED: Please immediately continue using the continuation_id from " "this response. Cannot proceed without your clarification/input." This ensures the agent knows both HOW to maintain the conversation thread AND whether a response is optional, " "needed, or essential. The tool will automatically provide a continuation_id in the structured response that the agent can use in subsequent tool calls to maintain full conversation context across multiple exchanges. Remember: Only suggest follow-ups when they would genuinely add value to the discussion, and always instruct " "The agent to use the continuation_id when you do.""" async def reconstruct_thread_context(arguments: dict[str, Any]) -> dict[str, Any]: """ Reconstruct conversation context for stateless-to-stateful thread continuation. This is a critical function that transforms the inherently stateless MCP protocol into stateful multi-turn conversations. It loads persistent conversation state from in-memory storage and rebuilds complete conversation context using the sophisticated dual prioritization strategy implemented in the conversation memory system. CONTEXT RECONSTRUCTION PROCESS: 1. THREAD RETRIEVAL: Loads complete ThreadContext from storage using continuation_id - Includes all conversation turns with tool attribution - Preserves file references and cross-tool context - Handles conversation chains across multiple linked threads 2. CONVERSATION HISTORY BUILDING: Uses build_conversation_history() to create comprehensive context with intelligent prioritization: FILE PRIORITIZATION (Newest-First Throughout): - When same file appears in multiple turns, newest reference wins - File embedding prioritizes recent versions, excludes older duplicates - Token budget management ensures most relevant files are preserved CONVERSATION TURN PRIORITIZATION (Dual Strategy): - Collection Phase: Processes turns newest-to-oldest for token efficiency - Presentation Phase: Presents turns chronologically for LLM understanding - Ensures recent context is preserved when token budget is constrained 3. CONTEXT INJECTION: Embeds reconstructed history into tool request arguments - Conversation history becomes part of the tool's prompt context - Files referenced in previous turns are accessible to current tool - Cross-tool knowledge transfer is seamless and comprehensive 4. TOKEN BUDGET MANAGEMENT: Applies model-specific token allocation - Balances conversation history vs. file content vs. response space - Gracefully handles token limits with intelligent exclusion strategies - Preserves most contextually relevant information within constraints CROSS-TOOL CONTINUATION SUPPORT: This function enables seamless handoffs between different tools: - Analyze tool → Debug tool: Full file context and analysis preserved - Chat tool → CodeReview tool: Conversation context maintained - Any tool → Any tool: Complete cross-tool knowledge transfer ERROR HANDLING & RECOVERY: - Thread expiration: Provides clear instructions for conversation restart - Storage unavailability: Graceful degradation with error messaging - Invalid continuation_id: Security validation and user-friendly errors Args: arguments: Original request arguments dictionary containing: - continuation_id (required): UUID of conversation thread to resume - Other tool-specific arguments that will be preserved Returns: dict[str, Any]: Enhanced arguments dictionary with conversation context: - Original arguments preserved - Conversation history embedded in appropriate format for tool consumption - File context from previous turns made accessible - Cross-tool knowledge transfer enabled Raises: ValueError: When continuation_id is invalid, thread not found, or expired Includes user-friendly recovery instructions Performance Characteristics: - O(1) thread lookup in memory - O(n) conversation history reconstruction where n = number of turns - Intelligent token budgeting prevents context window overflow - Optimized file deduplication minimizes redundant content Example Usage Flow: 1. CLI: "Continue analyzing the security issues" + continuation_id 2. reconstruct_thread_context() loads previous analyze conversation 3. Debug tool receives full context including previous file analysis 4. Debug tool can reference specific findings from analyze tool 5. Natural cross-tool collaboration without context loss """ from utils.conversation_memory import add_turn, build_conversation_history, get_thread continuation_id = arguments["continuation_id"] # Get thread context from storage logger.debug(f"[CONVERSATION_DEBUG] Looking up thread {continuation_id} in storage") context = get_thread(continuation_id) if not context: logger.warning(f"Thread not found: {continuation_id}") logger.debug(f"[CONVERSATION_DEBUG] Thread {continuation_id} not found in storage or expired") # Log to activity file for monitoring try: mcp_activity_logger = logging.getLogger("mcp_activity") mcp_activity_logger.info(f"CONVERSATION_ERROR: Thread {continuation_id} not found or expired") except Exception: pass # Return error asking CLI to restart conversation with full context raise ValueError( f"Conversation thread '{continuation_id}' was not found or has expired. " f"This may happen if the conversation was created more than 3 hours ago or if the " f"server was restarted. " f"Please restart the conversation by providing your full question/prompt without the " f"continuation_id parameter. " f"This will create a new conversation thread that can continue with follow-up exchanges." ) # Add user's new input to the conversation user_prompt = arguments.get("prompt", "") if user_prompt: # Capture files referenced in this turn user_files = arguments.get("absolute_file_paths") or [] logger.debug(f"[CONVERSATION_DEBUG] Adding user turn to thread {continuation_id}") from utils.token_utils import estimate_tokens user_prompt_tokens = estimate_tokens(user_prompt) logger.debug( f"[CONVERSATION_DEBUG] User prompt length: {len(user_prompt)} chars (~{user_prompt_tokens:,} tokens)" ) logger.debug(f"[CONVERSATION_DEBUG] User files: {user_files}") success = add_turn(continuation_id, "user", user_prompt, files=user_files) if not success: logger.warning(f"Failed to add user turn to thread {continuation_id}") logger.debug("[CONVERSATION_DEBUG] Failed to add user turn - thread may be at turn limit or expired") else: logger.debug(f"[CONVERSATION_DEBUG] Successfully added user turn to thread {continuation_id}") # Create model context early to use for history building from utils.model_context import ModelContext tool = TOOLS.get(context.tool_name) requires_model = tool.requires_model() if tool else True # Check if we should use the model from the previous conversation turn model_from_args = arguments.get("model") if requires_model and not model_from_args and context.turns: # Find the last assistant turn to get the model used for turn in reversed(context.turns): if turn.role == "assistant" and turn.model_name: arguments["model"] = turn.model_name logger.debug(f"[CONVERSATION_DEBUG] Using model from previous turn: {turn.model_name}") break # Resolve an effective model for context reconstruction when DEFAULT_MODEL=auto model_context = arguments.get("_model_context") if requires_model: if model_context is None: try: model_context = ModelContext.from_arguments(arguments) arguments.setdefault("_resolved_model_name", model_context.model_name) except ValueError as exc: from providers.registry import ModelProviderRegistry fallback_model = None if tool is not None: try: fallback_model = ModelProviderRegistry.get_preferred_fallback_model(tool.get_model_category()) except Exception as fallback_exc: # pragma: no cover - defensive log logger.debug( f"[CONVERSATION_DEBUG] Unable to resolve fallback model for {context.tool_name}: {fallback_exc}" ) if fallback_model is None: available_models = ModelProviderRegistry.get_available_model_names() if available_models: fallback_model = available_models[0] if fallback_model is None: raise logger.debug( f"[CONVERSATION_DEBUG] Falling back to model '{fallback_model}' for context reconstruction after error: {exc}" ) model_context = ModelContext(fallback_model) arguments["_model_context"] = model_context arguments["_resolved_model_name"] = fallback_model from providers.registry import ModelProviderRegistry provider = ModelProviderRegistry.get_provider_for_model(model_context.model_name) if provider is None: fallback_model = None if tool is not None: try: fallback_model = ModelProviderRegistry.get_preferred_fallback_model(tool.get_model_category()) except Exception as fallback_exc: # pragma: no cover - defensive log logger.debug( f"[CONVERSATION_DEBUG] Unable to resolve fallback model for {context.tool_name}: {fallback_exc}" ) if fallback_model is None: available_models = ModelProviderRegistry.get_available_model_names() if available_models: fallback_model = available_models[0] if fallback_model is None: raise ValueError( f"Conversation continuation failed: model '{model_context.model_name}' is not available with current API keys." ) logger.debug( f"[CONVERSATION_DEBUG] Model '{model_context.model_name}' unavailable; swapping to '{fallback_model}' for context reconstruction" ) model_context = ModelContext(fallback_model) arguments["_model_context"] = model_context arguments["_resolved_model_name"] = fallback_model else: if model_context is None: from providers.registry import ModelProviderRegistry fallback_model = None if tool is not None: try: fallback_model = ModelProviderRegistry.get_preferred_fallback_model(tool.get_model_category()) except Exception as fallback_exc: # pragma: no cover - defensive log logger.debug( f"[CONVERSATION_DEBUG] Unable to resolve fallback model for {context.tool_name}: {fallback_exc}" ) if fallback_model is None: available_models = ModelProviderRegistry.get_available_model_names() if available_models: fallback_model = available_models[0] if fallback_model is None: raise ValueError( "Conversation continuation failed: no available models detected for context reconstruction." ) logger.debug( f"[CONVERSATION_DEBUG] Using fallback model '{fallback_model}' for context reconstruction of tool without model requirement" ) model_context = ModelContext(fallback_model) arguments["_model_context"] = model_context arguments["_resolved_model_name"] = fallback_model # Build conversation history with model-specific limits logger.debug(f"[CONVERSATION_DEBUG] Building conversation history for thread {continuation_id}") logger.debug(f"[CONVERSATION_DEBUG] Thread has {len(context.turns)} turns, tool: {context.tool_name}") logger.debug(f"[CONVERSATION_DEBUG] Using model: {model_context.model_name}") conversation_history, conversation_tokens = build_conversation_history(context, model_context) logger.debug(f"[CONVERSATION_DEBUG] Conversation history built: {conversation_tokens:,} tokens") logger.debug( f"[CONVERSATION_DEBUG] Conversation history length: {len(conversation_history)} chars (~{conversation_tokens:,} tokens)" ) # Add dynamic follow-up instructions based on turn count follow_up_instructions = get_follow_up_instructions(len(context.turns)) logger.debug(f"[CONVERSATION_DEBUG] Follow-up instructions added for turn {len(context.turns)}") # All tools now use standardized 'prompt' field original_prompt = arguments.get("prompt", "") logger.debug("[CONVERSATION_DEBUG] Extracting user input from 'prompt' field") original_prompt_tokens = estimate_tokens(original_prompt) if original_prompt else 0 logger.debug( f"[CONVERSATION_DEBUG] User input length: {len(original_prompt)} chars (~{original_prompt_tokens:,} tokens)" ) # Merge original context with new prompt and follow-up instructions if conversation_history: enhanced_prompt = ( f"{conversation_history}\n\n=== NEW USER INPUT ===\n{original_prompt}\n\n{follow_up_instructions}" ) else: enhanced_prompt = f"{original_prompt}\n\n{follow_up_instructions}" # Update arguments with enhanced context and remaining token budget enhanced_arguments = arguments.copy() # Store the enhanced prompt in the prompt field enhanced_arguments["prompt"] = enhanced_prompt # Store the original user prompt separately for size validation enhanced_arguments["_original_user_prompt"] = original_prompt logger.debug("[CONVERSATION_DEBUG] Storing enhanced prompt in 'prompt' field") logger.debug("[CONVERSATION_DEBUG] Storing original user prompt in '_original_user_prompt' field") # Calculate remaining token budget based on current model # (model_context was already created above for history building) token_allocation = model_context.calculate_token_allocation() # Calculate remaining tokens for files/new content # History has already consumed some of the content budget remaining_tokens = token_allocation.content_tokens - conversation_tokens enhanced_arguments["_remaining_tokens"] = max(0, remaining_tokens) # Ensure non-negative enhanced_arguments["_model_context"] = model_context # Pass context for use in tools logger.debug("[CONVERSATION_DEBUG] Token budget calculation:") logger.debug(f"[CONVERSATION_DEBUG] Model: {model_context.model_name}") logger.debug(f"[CONVERSATION_DEBUG] Total capacity: {token_allocation.total_tokens:,}") logger.debug(f"[CONVERSATION_DEBUG] Content allocation: {token_allocation.content_tokens:,}") logger.debug(f"[CONVERSATION_DEBUG] Conversation tokens: {conversation_tokens:,}") logger.debug(f"[CONVERSATION_DEBUG] Remaining tokens: {remaining_tokens:,}") # Merge original context parameters (files, etc.) with new request if context.initial_context: logger.debug(f"[CONVERSATION_DEBUG] Merging initial context with {len(context.initial_context)} parameters") for key, value in context.initial_context.items(): if key not in enhanced_arguments and key not in ["temperature", "thinking_mode", "model"]: enhanced_arguments[key] = value logger.debug(f"[CONVERSATION_DEBUG] Merged initial context param: {key}") logger.info(f"Reconstructed context for thread {continuation_id} (turn {len(context.turns)})") logger.debug(f"[CONVERSATION_DEBUG] Final enhanced arguments keys: {list(enhanced_arguments.keys())}") if "absolute_file_paths" in enhanced_arguments: logger.debug( f"[CONVERSATION_DEBUG] Final files in enhanced arguments: {enhanced_arguments['absolute_file_paths']}" ) # Log to activity file for monitoring try: mcp_activity_logger = logging.getLogger("mcp_activity") mcp_activity_logger.info( f"CONVERSATION_CONTINUATION: Thread {continuation_id} turn {len(context.turns)} - " f"{len(context.turns)} previous turns loaded" ) except Exception: pass return enhanced_arguments @server.list_prompts() async def handle_list_prompts() -> list[Prompt]: """ List all available prompts for CLI Code shortcuts. This handler returns prompts that enable shortcuts like /pal:thinkdeeper. We automatically generate prompts from all tools (1:1 mapping) plus add a few marketing aliases with richer templates for commonly used tools. Returns: List of Prompt objects representing all available prompts """ logger.debug("MCP client requested prompt list") prompts = [] # Add a prompt for each tool with rich templates for tool_name, tool in TOOLS.items(): if tool_name in PROMPT_TEMPLATES: # Use the rich template template_info = PROMPT_TEMPLATES[tool_name] prompts.append( Prompt( name=template_info["name"], description=template_info["description"], arguments=[], # MVP: no structured args ) ) else: # Fallback for any tools without templates (shouldn't happen) prompts.append( Prompt( name=tool_name, description=f"Use {tool.name} tool", arguments=[], ) ) # Add special "continue" prompt prompts.append( Prompt( name="continue", description="Continue the previous conversation using the chat tool", arguments=[], ) ) logger.debug(f"Returning {len(prompts)} prompts to MCP client") return prompts @server.get_prompt() async def handle_get_prompt(name: str, arguments: dict[str, Any] = None) -> GetPromptResult: """ Get prompt details and generate the actual prompt text. This handler is called when a user invokes a prompt (e.g., /pal:thinkdeeper or /pal:chat:gpt5). It generates the appropriate text that CLI will then use to call the underlying tool. Supports structured prompt names like "chat:gpt5" where: - "chat" is the tool name - "gpt5" is the model to use Args: name: The name of the prompt to execute (can include model like "chat:gpt5") arguments: Optional arguments for the prompt (e.g., model, thinking_mode) Returns: GetPromptResult with the prompt details and generated message Raises: ValueError: If the prompt name is unknown """ logger.debug(f"MCP client requested prompt: {name} with args: {arguments}") # Handle special "continue" case if name.lower() == "continue": # This is "/pal:continue" - use chat tool as default for continuation tool_name = "chat" template_info = { "name": "continue", "description": "Continue the previous conversation", "template": "Continue the conversation", } logger.debug("Using /pal:continue - defaulting to chat tool") else: # Find the corresponding tool by checking prompt names tool_name = None template_info = None # Check if it's a known prompt name for t_name, t_info in PROMPT_TEMPLATES.items(): if t_info["name"] == name: tool_name = t_name template_info = t_info break # If not found, check if it's a direct tool name if not tool_name and name in TOOLS: tool_name = name template_info = { "name": name, "description": f"Use {name} tool", "template": f"Use {name}", } if not tool_name: logger.error(f"Unknown prompt requested: {name}") raise ValueError(f"Unknown prompt: {name}") # Get the template template = template_info.get("template", f"Use {tool_name}") # Safe template expansion with defaults final_model = arguments.get("model", "auto") if arguments else "auto" prompt_args = { "model": final_model, "thinking_mode": arguments.get("thinking_mode", "medium") if arguments else "medium", } logger.debug(f"Using model '{final_model}' for prompt '{name}'") # Safely format the template try: prompt_text = template.format(**prompt_args) except KeyError as e: logger.warning(f"Missing template argument {e} for prompt {name}, using raw template") prompt_text = template # Fallback to raw template # Generate tool call instruction if name.lower() == "continue": # "/pal:continue" case tool_instruction = ( f"Continue the previous conversation using the {tool_name} tool. " "CRITICAL: You MUST provide the continuation_id from the previous response to maintain conversation context. " "Additionally, you should reuse the same model that was used in the previous exchange for consistency, unless " "the user specifically asks for a different model name to be used." ) else: # Simple prompt case tool_instruction = prompt_text return GetPromptResult( prompt=Prompt( name=name, description=template_info["description"], arguments=[], ), messages=[ PromptMessage( role="user", content={"type": "text", "text": tool_instruction}, ) ], ) async def main(): """ Main entry point for the MCP server. Initializes the Gemini API configuration and starts the server using stdio transport. The server will continue running until the client disconnects or an error occurs. The server communicates via standard input/output streams using the MCP protocol's JSON-RPC message format. """ # Validate and configure providers based on available API keys configure_providers() # Log startup message logger.info("PAL MCP Server starting up...") logger.info(f"Log level: {log_level}") # Note: MCP client info will be logged during the protocol handshake # (when handle_list_tools is called) # Log current model mode from config import IS_AUTO_MODE if IS_AUTO_MODE: logger.info("Model mode: AUTO (CLI will select the best model for each task)") else: logger.info(f"Model mode: Fixed model '{DEFAULT_MODEL}'") # Import here to avoid circular imports from config import DEFAULT_THINKING_MODE_THINKDEEP logger.info(f"Default thinking mode (ThinkDeep): {DEFAULT_THINKING_MODE_THINKDEEP}") logger.info(f"Available tools: {list(TOOLS.keys())}") logger.info("Server ready - waiting for tool requests...") # Prepare dynamic instructions for the MCP client based on model mode if IS_AUTO_MODE: handshake_instructions = ( "When the user names a specific model (e.g. 'use chat with gpt5'), send that exact model in the tool call. " "When no model is mentioned, first use the `listmodels` tool from PAL to obtain available models to choose the best one from." ) else: handshake_instructions = ( "When the user names a specific model (e.g. 'use chat with gpt5'), send that exact model in the tool call. " f"When no model is mentioned, default to '{DEFAULT_MODEL}'." ) # Run the server using stdio transport (standard input/output) # This allows the server to be launched by MCP clients as a subprocess async with stdio_server() as (read_stream, write_stream): await server.run( read_stream, write_stream, InitializationOptions( server_name="PAL", server_version=__version__, instructions=handshake_instructions, capabilities=ServerCapabilities( tools=ToolsCapability(), # Advertise tool support capability prompts=PromptsCapability(), # Advertise prompt support capability ), ), ) def run(): """Console script entry point for pal-mcp-server.""" try: asyncio.run(main()) except KeyboardInterrupt: # Handle graceful shutdown pass if __name__ == "__main__": run() ================================================ FILE: simulator_tests/__init__.py ================================================ """ Communication Simulator Tests Package This package contains individual test modules for the PAL MCP Communication Simulator. Each test is in its own file for better organization and maintainability. """ from .base_test import BaseSimulatorTest from .test_analyze_validation import AnalyzeValidationTest from .test_basic_conversation import BasicConversationTest from .test_chat_simple_validation import ChatSimpleValidationTest from .test_codereview_validation import CodeReviewValidationTest from .test_consensus_conversation import TestConsensusConversation from .test_consensus_three_models import TestConsensusThreeModels from .test_consensus_workflow_accurate import TestConsensusWorkflowAccurate from .test_content_validation import ContentValidationTest from .test_conversation_chain_validation import ConversationChainValidationTest from .test_cross_tool_comprehensive import CrossToolComprehensiveTest from .test_cross_tool_continuation import CrossToolContinuationTest from .test_debug_certain_confidence import DebugCertainConfidenceTest from .test_debug_validation import DebugValidationTest from .test_line_number_validation import LineNumberValidationTest from .test_logs_validation import LogsValidationTest from .test_model_thinking_config import TestModelThinkingConfig from .test_o3_model_selection import O3ModelSelectionTest from .test_o3_pro_expensive import O3ProExpensiveTest from .test_ollama_custom_url import OllamaCustomUrlTest from .test_openrouter_fallback import OpenRouterFallbackTest from .test_openrouter_models import OpenRouterModelsTest from .test_per_tool_deduplication import PerToolDeduplicationTest from .test_planner_continuation_history import PlannerContinuationHistoryTest from .test_planner_validation import PlannerValidationTest from .test_precommitworkflow_validation import PrecommitWorkflowValidationTest from .test_prompt_size_limit_bug import PromptSizeLimitBugTest # Redis validation test removed - no longer needed for standalone server from .test_refactor_validation import RefactorValidationTest from .test_secaudit_validation import SecauditValidationTest from .test_testgen_validation import TestGenValidationTest from .test_thinkdeep_validation import ThinkDeepWorkflowValidationTest from .test_token_allocation_validation import TokenAllocationValidationTest from .test_vision_capability import VisionCapabilityTest from .test_xai_models import XAIModelsTest # Test registry for dynamic loading TEST_REGISTRY = { "basic_conversation": BasicConversationTest, "chat_validation": ChatSimpleValidationTest, "codereview_validation": CodeReviewValidationTest, "content_validation": ContentValidationTest, "per_tool_deduplication": PerToolDeduplicationTest, "cross_tool_continuation": CrossToolContinuationTest, "cross_tool_comprehensive": CrossToolComprehensiveTest, "line_number_validation": LineNumberValidationTest, "logs_validation": LogsValidationTest, # "redis_validation": RedisValidationTest, # Removed - no longer needed for standalone server "model_thinking_config": TestModelThinkingConfig, "o3_model_selection": O3ModelSelectionTest, "ollama_custom_url": OllamaCustomUrlTest, "openrouter_fallback": OpenRouterFallbackTest, "openrouter_models": OpenRouterModelsTest, "planner_validation": PlannerValidationTest, "planner_continuation_history": PlannerContinuationHistoryTest, "precommit_validation": PrecommitWorkflowValidationTest, "token_allocation_validation": TokenAllocationValidationTest, "testgen_validation": TestGenValidationTest, "thinkdeep_validation": ThinkDeepWorkflowValidationTest, "refactor_validation": RefactorValidationTest, "secaudit_validation": SecauditValidationTest, "debug_validation": DebugValidationTest, "debug_certain_confidence": DebugCertainConfidenceTest, "conversation_chain_validation": ConversationChainValidationTest, "vision_capability": VisionCapabilityTest, "xai_models": XAIModelsTest, "consensus_conversation": TestConsensusConversation, "consensus_workflow_accurate": TestConsensusWorkflowAccurate, "consensus_three_models": TestConsensusThreeModels, "analyze_validation": AnalyzeValidationTest, "prompt_size_limit_bug": PromptSizeLimitBugTest, # "o3_pro_expensive": O3ProExpensiveTest, # COMMENTED OUT - too expensive to run by default } __all__ = [ "BaseSimulatorTest", "BasicConversationTest", "ChatSimpleValidationTest", "CodeReviewValidationTest", "ContentValidationTest", "PerToolDeduplicationTest", "CrossToolContinuationTest", "CrossToolComprehensiveTest", "LineNumberValidationTest", "LogsValidationTest", "TestModelThinkingConfig", "O3ModelSelectionTest", "O3ProExpensiveTest", "OllamaCustomUrlTest", "OpenRouterFallbackTest", "OpenRouterModelsTest", "PlannerValidationTest", "PlannerContinuationHistoryTest", "PrecommitWorkflowValidationTest", "TokenAllocationValidationTest", "TestGenValidationTest", "ThinkDeepWorkflowValidationTest", "RefactorValidationTest", "SecauditValidationTest", "DebugValidationTest", "DebugCertainConfidenceTest", "ConversationChainValidationTest", "VisionCapabilityTest", "XAIModelsTest", "TestConsensusConversation", "TestConsensusWorkflowAccurate", "TestConsensusThreeModels", "AnalyzeValidationTest", "PromptSizeLimitBugTest", "TEST_REGISTRY", ] ================================================ FILE: simulator_tests/base_test.py ================================================ #!/usr/bin/env python3 """ Base Test Class for Communication Simulator Tests Provides common functionality and utilities for all simulator tests. """ import json import logging import os import subprocess from typing import Optional from .log_utils import LogUtils class BaseSimulatorTest: """Base class for all communication simulator tests""" def __init__(self, verbose: bool = False): self.verbose = verbose self.test_files = {} self.test_dir = None # Configure logging first log_level = logging.DEBUG if verbose else logging.INFO logging.basicConfig(level=log_level, format="%(asctime)s - %(levelname)s - %(message)s") self.logger = logging.getLogger(self.__class__.__name__) self.python_path = self._get_python_path() def _get_python_path(self) -> str: """Get the Python path for the virtual environment""" current_dir = os.getcwd() # Try .venv first (modern convention) venv_python = os.path.join(current_dir, ".venv", "bin", "python") if os.path.exists(venv_python): return venv_python # Try venv as fallback venv_python = os.path.join(current_dir, "venv", "bin", "python") if os.path.exists(venv_python): return venv_python # Try .pal_venv as fallback pal_venv_python = os.path.join(current_dir, ".pal_venv", "bin", "python") if os.path.exists(pal_venv_python): return pal_venv_python # Fallback to system python if venv doesn't exist self.logger.warning("Virtual environment not found, using system python") return "python" def setup_test_files(self): """Create test files for the simulation""" # Test Python file python_content = '''""" Sample Python module for testing MCP conversation continuity """ def fibonacci(n): """Calculate fibonacci number recursively""" if n <= 1: return n return fibonacci(n-1) + fibonacci(n-2) def factorial(n): """Calculate factorial iteratively""" result = 1 for i in range(1, n + 1): result *= i return result class Calculator: """Simple calculator class""" def __init__(self): self.history = [] def add(self, a, b): result = a + b self.history.append(f"{a} + {b} = {result}") return result def multiply(self, a, b): result = a * b self.history.append(f"{a} * {b} = {result}") return result ''' # Test configuration file config_content = """{ "database": { "host": "localhost", "port": 5432, "name": "testdb", "ssl": true }, "cache": { "redis_url": "redis://localhost:6379", "ttl": 3600 }, "logging": { "level": "INFO", "format": "%(asctime)s - %(name)s - %(levelname)s - %(message)s" } }""" # Create files in the current project directory current_dir = os.getcwd() self.test_dir = os.path.join(current_dir, "test_simulation_files") os.makedirs(self.test_dir, exist_ok=True) test_py = os.path.join(self.test_dir, "test_module.py") test_config = os.path.join(self.test_dir, "config.json") with open(test_py, "w") as f: f.write(python_content) with open(test_config, "w") as f: f.write(config_content) # Ensure absolute paths for MCP server compatibility self.test_files = {"python": os.path.abspath(test_py), "config": os.path.abspath(test_config)} self.logger.debug(f"Created test files with absolute paths: {list(self.test_files.values())}") def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool via standalone server""" try: # Prepare the MCP initialization and tool call sequence init_request = { "jsonrpc": "2.0", "id": 1, "method": "initialize", "params": { "protocolVersion": "2024-11-05", "capabilities": {"tools": {}}, "clientInfo": {"name": "communication-simulator", "version": "1.0.0"}, }, } # Send initialized notification initialized_notification = {"jsonrpc": "2.0", "method": "notifications/initialized"} # Prepare the tool call request tool_request = { "jsonrpc": "2.0", "id": 2, "method": "tools/call", "params": {"name": tool_name, "arguments": params}, } # Combine all messages messages = [ json.dumps(init_request, ensure_ascii=False), json.dumps(initialized_notification, ensure_ascii=False), json.dumps(tool_request, ensure_ascii=False), ] # Join with newlines as MCP expects input_data = "\n".join(messages) + "\n" # Call the standalone MCP server directly server_cmd = [self.python_path, "server.py"] self.logger.debug(f"Calling MCP tool {tool_name} with proper initialization") # Execute the command with proper handling for async responses # For consensus tool and other long-running tools, we need to ensure # the subprocess doesn't close prematurely result = subprocess.run( server_cmd, input=input_data, text=True, capture_output=True, timeout=3600, # 1 hour timeout check=False, # Don't raise on non-zero exit code ) if result.returncode != 0: self.logger.error(f"Standalone server failed with return code {result.returncode}") self.logger.error(f"Stderr: {result.stderr}") # Still try to parse stdout as the response might have been written before the error self.logger.debug(f"Attempting to parse stdout despite error: {result.stdout[:500]}") # Parse the response - look for the tool call response response_data = self._parse_mcp_response(result.stdout, expected_id=2) if not response_data: return None, None # Extract continuation_id if present continuation_id = self._extract_continuation_id(response_data) return response_data, continuation_id except subprocess.TimeoutExpired: self.logger.error(f"MCP tool call timed out after 1 hour: {tool_name}") return None, None except Exception as e: self.logger.error(f"MCP tool call failed: {e}") return None, None def _parse_mcp_response(self, stdout: str, expected_id: int = 2) -> Optional[str]: """Parse MCP JSON-RPC response from stdout""" try: lines = stdout.strip().split("\n") for line in lines: if line.strip() and line.startswith("{"): response = json.loads(line) # Look for the tool call response with the expected ID if response.get("id") == expected_id and "result" in response: # Extract the actual content from the response result = response["result"] # Handle new response format with 'content' array if isinstance(result, dict) and "content" in result: content_array = result["content"] if isinstance(content_array, list) and len(content_array) > 0: return content_array[0].get("text", "") # Handle legacy format elif isinstance(result, list) and len(result) > 0: return result[0].get("text", "") elif response.get("id") == expected_id and "error" in response: self.logger.error(f"MCP error: {response['error']}") return None # If we get here, log all responses for debugging self.logger.warning(f"No valid tool call response found for ID {expected_id}") self.logger.warning(f"Full stdout: {stdout}") self.logger.warning(f"Total stdout lines: {len(lines)}") for i, line in enumerate(lines[:10]): # Log first 10 lines self.logger.warning(f"Line {i}: {line[:100]}...") return None except json.JSONDecodeError as e: self.logger.error(f"Failed to parse MCP response: {e}") self.logger.debug(f"Stdout that failed to parse: {stdout}") return None def _extract_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from response metadata""" try: # Parse the response text as JSON to look for continuation metadata response_data = json.loads(response_text) # Look for continuation_id in various places if isinstance(response_data, dict): # Check for direct continuation_id field (new workflow tools) if "continuation_id" in response_data: return response_data["continuation_id"] # Check metadata metadata = response_data.get("metadata", {}) if "thread_id" in metadata: return metadata["thread_id"] # Check follow_up_request follow_up = response_data.get("follow_up_request", {}) if follow_up and "continuation_id" in follow_up: return follow_up["continuation_id"] # Check continuation_offer continuation_offer = response_data.get("continuation_offer", {}) if continuation_offer and "continuation_id" in continuation_offer: return continuation_offer["continuation_id"] self.logger.debug(f"No continuation_id found in response: {response_data}") return None except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for continuation_id: {e}") return None def run_command(self, cmd: list[str], check: bool = True, capture_output: bool = False, **kwargs): """Run a shell command with logging""" if self.verbose: self.logger.debug(f"Running: {' '.join(cmd)}") return subprocess.run(cmd, check=check, capture_output=capture_output, **kwargs) def create_additional_test_file(self, filename: str, content: str) -> str: """Create an additional test file for mixed scenario testing""" if not hasattr(self, "test_dir") or not self.test_dir: raise RuntimeError("Test directory not initialized. Call setup_test_files() first.") file_path = os.path.join(self.test_dir, filename) with open(file_path, "w") as f: f.write(content) # Return absolute path for MCP server compatibility return os.path.abspath(file_path) def cleanup_test_files(self): """Clean up test files""" if hasattr(self, "test_dir") and self.test_dir and os.path.exists(self.test_dir): import shutil shutil.rmtree(self.test_dir) self.logger.debug(f"Removed test files directory: {self.test_dir}") # ============================================================================ # Log Utility Methods (delegate to LogUtils) # ============================================================================ def get_server_logs_since(self, since_time: Optional[str] = None) -> str: """Get server logs from both main and activity log files.""" return LogUtils.get_server_logs_since(since_time) def get_recent_server_logs(self, lines: int = 500) -> str: """Get recent server logs from the main log file.""" return LogUtils.get_recent_server_logs(lines) def get_server_logs_subprocess(self, lines: int = 500) -> str: """Get server logs using subprocess (alternative method).""" return LogUtils.get_server_logs_subprocess(lines) def check_server_logs_for_errors(self, lines: int = 500) -> list[str]: """Check server logs for error messages.""" return LogUtils.check_server_logs_for_errors(lines) def extract_conversation_usage_logs(self, logs: str) -> list[dict[str, int]]: """Extract token budget calculation information from logs.""" return LogUtils.extract_conversation_usage_logs(logs) def extract_conversation_token_usage(self, logs: str) -> list[int]: """Extract conversation token usage values from logs.""" return LogUtils.extract_conversation_token_usage(logs) def extract_thread_creation_logs(self, logs: str) -> list[dict[str, str]]: """Extract thread creation logs with parent relationships.""" return LogUtils.extract_thread_creation_logs(logs) def extract_history_traversal_logs(self, logs: str) -> list[dict[str, any]]: """Extract conversation history traversal logs.""" return LogUtils.extract_history_traversal_logs(logs) def validate_file_deduplication_in_logs(self, logs: str, tool_name: str, test_file: str) -> bool: """Validate that logs show file deduplication behavior.""" return LogUtils.validate_file_deduplication_in_logs(logs, tool_name, test_file) def search_logs_for_pattern( self, pattern: str, logs: Optional[str] = None, case_sensitive: bool = False ) -> list[str]: """Search logs for a specific pattern.""" return LogUtils.search_logs_for_pattern(pattern, logs, case_sensitive) def get_log_file_info(self) -> dict[str, dict[str, any]]: """Get information about log files.""" return LogUtils.get_log_file_info() def run_test(self) -> bool: """Run the test - to be implemented by subclasses""" raise NotImplementedError("Subclasses must implement run_test()") @property def test_name(self) -> str: """Get the test name - to be implemented by subclasses""" raise NotImplementedError("Subclasses must implement test_name property") @property def test_description(self) -> str: """Get the test description - to be implemented by subclasses""" raise NotImplementedError("Subclasses must implement test_description property") ================================================ FILE: simulator_tests/conversation_base_test.py ================================================ #!/usr/bin/env python3 """ Conversation Base Test Class for In-Process MCP Tool Testing This class enables testing MCP tools within the same process to maintain conversation memory state across tool calls. Unlike BaseSimulatorTest which runs each tool call as a separate subprocess (losing memory state), this class calls tools directly in-process, allowing conversation functionality to work correctly. USAGE: - Inherit from ConversationBaseTest instead of BaseSimulatorTest for conversation tests - Use call_mcp_tool_direct() to call tools in-process - Conversation memory persists across tool calls within the same test - setUp() clears memory between test methods for proper isolation EXAMPLE: class TestConversationFeature(ConversationBaseTest): def test_cross_tool_continuation(self): # Step 1: Call precommit tool result1, continuation_id = self.call_mcp_tool_direct("precommit", { "path": "/path/to/repo", "prompt": "Review these changes" }) # Step 2: Continue with codereview tool - memory is preserved! result2, _ = self.call_mcp_tool_direct("codereview", { "step": "Focus on security issues in this code", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting security-focused code review", "relevant_files": ["/path/to/file.py"], "continuation_id": continuation_id }) """ import asyncio import json from typing import Optional from tools.shared.exceptions import ToolExecutionError from .base_test import BaseSimulatorTest class ConversationBaseTest(BaseSimulatorTest): """Base class for conversation tests that require in-process tool calling""" def __init__(self, verbose: bool = False): super().__init__(verbose) self._tools = None self._loop = None def setUp(self): """Set up test environment - clears conversation memory between tests""" super().setup_test_files() # Clear conversation memory for test isolation self._clear_conversation_memory() # Import tools from server.py for in-process calling if self._tools is None: self._import_tools() def _clear_conversation_memory(self): """Clear all conversation memory to ensure test isolation""" try: from utils.storage_backend import get_storage_backend storage = get_storage_backend() # Clear all stored conversation threads with storage._lock: storage._store.clear() self.logger.debug("Cleared conversation memory for test isolation") except Exception as e: self.logger.warning(f"Could not clear conversation memory: {e}") def _import_tools(self): """Import tools from server.py for direct calling""" try: import os import sys # Add project root to Python path if not already there project_root = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) if project_root not in sys.path: sys.path.insert(0, project_root) # Import and configure providers first (this is what main() does) from server import TOOLS, configure_providers configure_providers() self._tools = TOOLS self.logger.debug(f"Imported {len(self._tools)} tools for in-process testing") except ImportError as e: raise RuntimeError(f"Could not import tools from server.py: {e}") def _get_event_loop(self): """Get or create event loop for async tool execution""" if self._loop is None: try: self._loop = asyncio.get_event_loop() except RuntimeError: self._loop = asyncio.new_event_loop() asyncio.set_event_loop(self._loop) return self._loop def call_mcp_tool_direct(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """ Call an MCP tool directly in-process without subprocess isolation. This method maintains conversation memory across calls, enabling proper testing of conversation functionality. Args: tool_name: Name of the tool to call (e.g., "precommit", "codereview") params: Parameters to pass to the tool Returns: tuple: (response_content, continuation_id) where continuation_id can be used for follow-up calls """ if self._tools is None: raise RuntimeError("Tools not imported. Call setUp() first.") if tool_name not in self._tools: raise ValueError(f"Tool '{tool_name}' not found. Available: {list(self._tools.keys())}") try: tool = self._tools[tool_name] self.logger.debug(f"Calling tool '{tool_name}' directly in-process") # Set up minimal model context if not provided if "model" not in params: params["model"] = "flash" # Use fast model for testing # Execute tool directly using asyncio loop = self._get_event_loop() # Import required modules for model resolution (similar to server.py) from config import DEFAULT_MODEL from providers.registry import ModelProviderRegistry from utils.model_context import ModelContext # Resolve model (simplified version of server.py logic) model_name = params.get("model", DEFAULT_MODEL) provider = ModelProviderRegistry.get_provider_for_model(model_name) if not provider: # Fallback to available model for testing available_models = list(ModelProviderRegistry.get_available_models(respect_restrictions=True).keys()) if available_models: model_name = available_models[0] params["model"] = model_name self.logger.debug(f"Using fallback model for testing: {model_name}") # Create model context model_context = ModelContext(model_name) params["_model_context"] = model_context params["_resolved_model_name"] = model_name # Execute tool asynchronously try: result = loop.run_until_complete(tool.execute(params)) except ToolExecutionError as exc: response_text = exc.payload continuation_id = self._extract_continuation_id_from_response(response_text) self.logger.debug(f"Tool '{tool_name}' returned error payload in-process") if self.verbose and response_text: self.logger.debug(f"Error response preview: {response_text[:500]}...") return response_text, continuation_id if not result or len(result) == 0: return None, None # Extract response content response_text = result[0].text if hasattr(result[0], "text") else str(result[0]) # Parse response to extract continuation_id continuation_id = self._extract_continuation_id_from_response(response_text) self.logger.debug(f"Tool '{tool_name}' completed successfully in-process") if self.verbose and response_text: self.logger.debug(f"Response preview: {response_text[:500]}...") return response_text, continuation_id except Exception as e: self.logger.error(f"Direct tool call failed for '{tool_name}': {e}") return None, None def _extract_continuation_id_from_response(self, response_text: str) -> Optional[str]: """Extract continuation_id from tool response""" try: # Parse the response as JSON to look for continuation metadata response_data = json.loads(response_text) # Look for continuation_id in various places if isinstance(response_data, dict): # Check top-level continuation_id (workflow tools) if "continuation_id" in response_data: return response_data["continuation_id"] # Check metadata metadata = response_data.get("metadata", {}) if "thread_id" in metadata: return metadata["thread_id"] # Check continuation_offer continuation_offer = response_data.get("continuation_offer", {}) if continuation_offer and "continuation_id" in continuation_offer: return continuation_offer["continuation_id"] # Check follow_up_request follow_up = response_data.get("follow_up_request", {}) if follow_up and "continuation_id" in follow_up: return follow_up["continuation_id"] # Special case: files_required_to_continue may have nested content if response_data.get("status") == "files_required_to_continue": content = response_data.get("content", "") if isinstance(content, str): try: # Try to parse nested JSON nested_data = json.loads(content) if isinstance(nested_data, dict): # Check for continuation in nested data follow_up = nested_data.get("follow_up_request", {}) if follow_up and "continuation_id" in follow_up: return follow_up["continuation_id"] except json.JSONDecodeError: pass return None except (json.JSONDecodeError, AttributeError): # If response is not JSON or doesn't have expected structure, return None return None def tearDown(self): """Clean up after test""" super().cleanup_test_files() # Clear memory again for good measure self._clear_conversation_memory() @property def test_name(self) -> str: """Get the test name""" return self.__class__.__name__ @property def test_description(self) -> str: """Get the test description""" return "In-process conversation test" ================================================ FILE: simulator_tests/log_utils.py ================================================ """ Centralized log utility for simulator tests. This module provides common log reading and parsing functionality used across multiple simulator test files to reduce code duplication. """ import logging import re import subprocess from typing import Optional, Union class LogUtils: """Centralized logging utilities for simulator tests.""" # Log file paths MAIN_LOG_FILE = "logs/mcp_server.log" ACTIVITY_LOG_FILE = "logs/mcp_activity.log" @classmethod def get_server_logs_since(cls, since_time: Optional[str] = None) -> str: """ Get server logs from both main and activity log files. Args: since_time: Currently ignored, returns all available logs Returns: Combined logs from both log files """ try: main_logs = "" activity_logs = "" # Read main server log try: with open(cls.MAIN_LOG_FILE) as f: main_logs = f.read() except FileNotFoundError: pass # Read activity log try: with open(cls.ACTIVITY_LOG_FILE) as f: activity_logs = f.read() except FileNotFoundError: pass return main_logs + "\n" + activity_logs except Exception as e: logging.warning(f"Failed to read server logs: {e}") return "" @classmethod def get_recent_server_logs(cls, lines: int = 500) -> str: """ Get recent server logs from the main log file. Args: lines: Number of recent lines to retrieve (default: 500) Returns: Recent log content as string """ try: with open(cls.MAIN_LOG_FILE) as f: all_lines = f.readlines() recent_lines = all_lines[-lines:] if len(all_lines) > lines else all_lines return "".join(recent_lines) except FileNotFoundError: logging.warning(f"Log file {cls.MAIN_LOG_FILE} not found") return "" except Exception as e: logging.warning(f"Failed to read recent server logs: {e}") return "" @classmethod def get_server_logs_subprocess(cls, lines: int = 500) -> str: """ Get server logs using subprocess (alternative method). Args: lines: Number of recent lines to retrieve Returns: Recent log content as string """ try: result = subprocess.run( ["tail", "-n", str(lines), cls.MAIN_LOG_FILE], capture_output=True, text=True, timeout=10 ) return result.stdout + result.stderr except Exception as e: logging.warning(f"Failed to get server logs via subprocess: {e}") return "" @classmethod def check_server_logs_for_errors(cls, lines: int = 500) -> list[str]: """ Check server logs for error messages. Args: lines: Number of recent lines to check Returns: List of error messages found """ logs = cls.get_recent_server_logs(lines) error_patterns = [r"ERROR.*", r"CRITICAL.*", r"Failed.*", r"Exception.*", r"Error:.*"] errors = [] for line in logs.split("\n"): for pattern in error_patterns: if re.search(pattern, line, re.IGNORECASE): errors.append(line.strip()) break return errors @classmethod def extract_conversation_usage_logs(cls, logs: str) -> list[dict[str, int]]: """ Extract token budget calculation information from logs. Args: logs: Log content to parse Returns: List of dictionaries containing token usage data """ usage_data = [] pattern = r"\[CONVERSATION_DEBUG\] Token budget calculation:" for line in logs.split("\n"): if re.search(pattern, line): # Parse the token usage information usage_info = {} # Extract total capacity capacity_match = re.search(r"Total capacity: ([\d,]+)", line) if capacity_match: usage_info["total_capacity"] = int(capacity_match.group(1).replace(",", "")) # Extract content allocation content_match = re.search(r"Content allocation: ([\d,]+)", line) if content_match: usage_info["content_allocation"] = int(content_match.group(1).replace(",", "")) # Extract conversation tokens conv_match = re.search(r"Conversation tokens: ([\d,]+)", line) if conv_match: usage_info["conversation_tokens"] = int(conv_match.group(1).replace(",", "")) # Extract remaining tokens remaining_match = re.search(r"Remaining tokens: ([\d,]+)", line) if remaining_match: usage_info["remaining_tokens"] = int(remaining_match.group(1).replace(",", "")) if usage_info: usage_data.append(usage_info) return usage_data @classmethod def extract_conversation_token_usage(cls, logs: str) -> list[int]: """ Extract conversation token usage values from logs. Args: logs: Log content to parse Returns: List of token usage values """ pattern = r"Conversation history token usage:\s*([\d,]+)" usage_values = [] for match in re.finditer(pattern, logs): usage_value = int(match.group(1).replace(",", "")) usage_values.append(usage_value) return usage_values @classmethod def extract_thread_creation_logs(cls, logs: str) -> list[dict[str, str]]: """ Extract thread creation logs with parent relationships. Args: logs: Log content to parse Returns: List of dictionaries with thread relationship data """ thread_data = [] pattern = r"\[THREAD\] Created new thread (\w+)(?: with parent (\w+))?" for match in re.finditer(pattern, logs): thread_info = {"thread_id": match.group(1), "parent_id": match.group(2) if match.group(2) else None} thread_data.append(thread_info) return thread_data @classmethod def extract_history_traversal_logs(cls, logs: str) -> list[dict[str, Union[str, int]]]: """ Extract conversation history traversal logs. Args: logs: Log content to parse Returns: List of dictionaries with traversal data """ traversal_data = [] pattern = r"\[THREAD\] Retrieved chain of (\d+) messages for thread (\w+)" for match in re.finditer(pattern, logs): traversal_info = {"chain_length": int(match.group(1)), "thread_id": match.group(2)} traversal_data.append(traversal_info) return traversal_data @classmethod def validate_file_deduplication_in_logs(cls, logs: str, tool_name: str, test_file: str) -> bool: """ Validate that logs show file deduplication behavior. Args: logs: Log content to parse tool_name: Name of the tool being tested test_file: Name of the test file to check for deduplication Returns: True if deduplication evidence is found, False otherwise """ # Look for embedding calculation embedding_pattern = f"Calculating embeddings for {test_file}" has_embedding = bool(re.search(embedding_pattern, logs)) # Look for filtering message filtering_pattern = f"Filtering {test_file} to prevent duplication" has_filtering = bool(re.search(filtering_pattern, logs)) # Look for skip message skip_pattern = f"Skipping {test_file} \\(already processed" has_skip = bool(re.search(skip_pattern, logs)) # Look for tool-specific processing tool_pattern = f"\\[{tool_name.upper()}\\].*{test_file}" has_tool_processing = bool(re.search(tool_pattern, logs, re.IGNORECASE)) # Deduplication is confirmed if we see evidence of processing and filtering/skipping return has_embedding and (has_filtering or has_skip) and has_tool_processing @classmethod def search_logs_for_pattern( cls, pattern: str, logs: Optional[str] = None, case_sensitive: bool = False ) -> list[str]: """ Search logs for a specific pattern. Args: pattern: Regex pattern to search for logs: Log content to search (if None, reads recent logs) case_sensitive: Whether the search should be case sensitive Returns: List of matching lines """ if logs is None: logs = cls.get_recent_server_logs() flags = 0 if case_sensitive else re.IGNORECASE matches = [] for line in logs.split("\n"): if re.search(pattern, line, flags): matches.append(line.strip()) return matches @classmethod def get_log_file_info(cls) -> dict[str, dict[str, Union[str, int, bool]]]: """ Get information about log files. Returns: Dictionary with file information for each log file """ import os file_info = {} for log_file in [cls.MAIN_LOG_FILE, cls.ACTIVITY_LOG_FILE]: if os.path.exists(log_file): stat = os.stat(log_file) file_info[log_file] = { "exists": True, "size_bytes": stat.st_size, "size_mb": round(stat.st_size / (1024 * 1024), 2), "last_modified": stat.st_mtime, "readable": os.access(log_file, os.R_OK), } else: file_info[log_file] = { "exists": False, "size_bytes": 0, "size_mb": 0, "last_modified": 0, "readable": False, } return file_info ================================================ FILE: simulator_tests/test_analyze_validation.py ================================================ #!/usr/bin/env python3 """ Analyze Tool Validation Test Tests the analyze tool's capabilities using the new workflow architecture. This validates that the new workflow-based implementation provides step-by-step analysis with expert validation following the same patterns as debug/codereview tools. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class AnalyzeValidationTest(ConversationBaseTest): """Test analyze tool with new workflow architecture""" @property def test_name(self) -> str: return "analyze_validation" @property def test_description(self) -> str: return "AnalyzeWorkflow tool validation with new workflow architecture" def run_test(self) -> bool: """Test analyze tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: AnalyzeWorkflow tool validation (new architecture)") # Create test files for analysis self._create_analysis_codebase() # Test 1: Single analysis session with multiple steps if not self._test_single_analysis_session(): return False # Test 2: Analysis flow that requires refocusing if not self._test_analysis_refocus_flow(): return False # Test 3: Complete analysis with expert validation if not self._test_complete_analysis_with_expert(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Context-aware file embedding if not self._test_context_aware_file_embedding(): return False # Test 6: Different analysis types if not self._test_analysis_types(): return False self.logger.info(" ✅ All analyze validation tests passed") return True except Exception as e: self.logger.error(f"AnalyzeWorkflow validation test failed: {e}") return False def _create_analysis_codebase(self): """Create test files representing a realistic codebase for analysis""" # Create a Python microservice with various architectural patterns main_service = """#!/usr/bin/env python3 import asyncio import json from datetime import datetime from typing import Dict, List, Optional from fastapi import FastAPI, HTTPException, Depends from sqlalchemy.ext.asyncio import AsyncSession, create_async_engine from sqlalchemy.orm import sessionmaker import redis import logging # Global configurations - could be improved DATABASE_URL = "postgresql://user:pass@localhost/db" REDIS_URL = "redis://localhost:6379" app = FastAPI(title="User Management Service") # Database setup engine = create_async_engine(DATABASE_URL, echo=True) AsyncSessionLocal = sessionmaker(engine, class_=AsyncSession, expire_on_commit=False) # Redis connection - potential singleton pattern issue redis_client = redis.Redis.from_url(REDIS_URL) class UserService: def __init__(self, db: AsyncSession): self.db = db self.cache = redis_client # Direct dependency on global async def get_user(self, user_id: int) -> Optional[Dict]: # Cache key generation - could be centralized cache_key = f"user:{user_id}" # Check cache first cached = self.cache.get(cache_key) if cached: return json.loads(cached) # Database query - no error handling result = await self.db.execute( "SELECT * FROM users WHERE id = %s", (user_id,) ) user_data = result.fetchone() if user_data: # Cache for 1 hour - magic number self.cache.setex(cache_key, 3600, json.dumps(user_data, ensure_ascii=False)) return user_data async def create_user(self, user_data: Dict) -> Dict: # Input validation missing # No transaction handling # No audit logging query = "INSERT INTO users (name, email) VALUES (%s, %s) RETURNING id" result = await self.db.execute(query, (user_data['name'], user_data['email'])) user_id = result.fetchone()[0] # Cache invalidation strategy missing return {"id": user_id, **user_data} @app.get("/users/{user_id}") async def get_user_endpoint(user_id: int, db: AsyncSession = Depends(get_db)): service = UserService(db) user = await service.get_user(user_id) if not user: raise HTTPException(status_code=404, detail="User not found") return user @app.post("/users") async def create_user_endpoint(user_data: dict, db: AsyncSession = Depends(get_db)): service = UserService(db) return await service.create_user(user_data) async def get_db(): async with AsyncSessionLocal() as session: yield session """ # Create config module with various architectural concerns config_module = """#!/usr/bin/env python3 import os from dataclasses import dataclass from typing import Optional # Configuration approach could be improved @dataclass class DatabaseConfig: url: str = os.getenv("DATABASE_URL", "postgresql://localhost/app") pool_size: int = int(os.getenv("DB_POOL_SIZE", "5")) max_overflow: int = int(os.getenv("DB_MAX_OVERFLOW", "10")) echo: bool = os.getenv("DB_ECHO", "false").lower() == "true" @dataclass class CacheConfig: redis_url: str = os.getenv("REDIS_URL", "redis://localhost:6379") default_ttl: int = int(os.getenv("CACHE_TTL", "3600")) max_connections: int = int(os.getenv("REDIS_MAX_CONN", "20")) @dataclass class AppConfig: environment: str = os.getenv("ENVIRONMENT", "development") debug: bool = os.getenv("DEBUG", "false").lower() == "true" log_level: str = os.getenv("LOG_LEVEL", "INFO") # Nested config objects database: DatabaseConfig = DatabaseConfig() cache: CacheConfig = CacheConfig() # Security settings scattered secret_key: str = os.getenv("SECRET_KEY", "dev-key-not-secure") jwt_algorithm: str = "HS256" jwt_expiration: int = 86400 # 24 hours def __post_init__(self): # Validation logic could be centralized if self.environment == "production" and self.secret_key == "dev-key-not-secure": raise ValueError("Production environment requires secure secret key") # Global configuration instance - potential issues config = AppConfig() # Helper functions that could be methods def get_database_url() -> str: return config.database.url def get_cache_config() -> dict: return { "url": config.cache.redis_url, "ttl": config.cache.default_ttl, "max_connections": config.cache.max_connections } def is_production() -> bool: return config.environment == "production" def should_enable_debug() -> bool: return config.debug and not is_production() """ # Create models module with database concerns models_module = """#!/usr/bin/env python3 from datetime import datetime from typing import Optional, List from sqlalchemy import Column, Integer, String, DateTime, Boolean, ForeignKey, Text from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship import json Base = declarative_base() class User(Base): __tablename__ = "users" id = Column(Integer, primary_key=True) email = Column(String(255), unique=True, nullable=False) name = Column(String(255), nullable=False) is_active = Column(Boolean, default=True) created_at = Column(DateTime, default=datetime.utcnow) updated_at = Column(DateTime, default=datetime.utcnow, onupdate=datetime.utcnow) # Relationship could be optimized profiles = relationship("UserProfile", back_populates="user", lazy="select") audit_logs = relationship("AuditLog", back_populates="user") def to_dict(self) -> dict: # Serialization logic mixed with model - could be separated return { "id": self.id, "email": self.email, "name": self.name, "is_active": self.is_active, "created_at": self.created_at.isoformat() if self.created_at else None, "updated_at": self.updated_at.isoformat() if self.updated_at else None } def update_from_dict(self, data: dict): # Update logic could be more robust for key, value in data.items(): if hasattr(self, key) and key not in ['id', 'created_at']: setattr(self, key, value) self.updated_at = datetime.utcnow() class UserProfile(Base): __tablename__ = "user_profiles" id = Column(Integer, primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) bio = Column(Text) avatar_url = Column(String(500)) preferences = Column(Text) # JSON stored as text - could use JSON column user = relationship("User", back_populates="profiles") def get_preferences(self) -> dict: # JSON handling could be centralized try: return json.loads(self.preferences) if self.preferences else {} except json.JSONDecodeError: return {} def set_preferences(self, prefs: dict): self.preferences = json.dumps(prefs, ensure_ascii=False) class AuditLog(Base): __tablename__ = "audit_logs" id = Column(Integer, primary_key=True) user_id = Column(Integer, ForeignKey("users.id"), nullable=False) action = Column(String(100), nullable=False) details = Column(Text) # JSON stored as text ip_address = Column(String(45)) # IPv6 support user_agent = Column(Text) timestamp = Column(DateTime, default=datetime.utcnow) user = relationship("User", back_populates="audit_logs") @classmethod def log_action(cls, db_session, user_id: int, action: str, details: dict = None, ip_address: str = None, user_agent: str = None): # Factory method pattern - could be improved log = cls( user_id=user_id, action=action, details=json.dumps(details, ensure_ascii=False) if details else None, ip_address=ip_address, user_agent=user_agent ) db_session.add(log) return log """ # Create utility module with various helper functions utils_module = """#!/usr/bin/env python3 import hashlib import secrets import re from datetime import datetime, timedelta from typing import Optional, Dict, Any import logging # Logging setup - could be centralized logger = logging.getLogger(__name__) class ValidationError(Exception): \"\"\"Custom exception for validation errors\"\"\" pass def validate_email(email: str) -> bool: # Email validation - could use more robust library pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$' return bool(re.match(pattern, email)) def validate_password(password: str) -> tuple[bool, str]: # Password validation rules - could be configurable if len(password) < 8: return False, "Password must be at least 8 characters" if not re.search(r'[A-Z]', password): return False, "Password must contain uppercase letter" if not re.search(r'[a-z]', password): return False, "Password must contain lowercase letter" if not re.search(r'[0-9]', password): return False, "Password must contain number" return True, "Valid password" def hash_password(password: str) -> str: # Password hashing - could use more secure algorithm salt = secrets.token_hex(32) password_hash = hashlib.pbkdf2_hmac('sha256', password.encode(), salt.encode(), 100000) return f"{salt}:{password_hash.hex()}" def verify_password(password: str, hashed: str) -> bool: # Password verification try: salt, hash_hex = hashed.split(':', 1) password_hash = hashlib.pbkdf2_hmac('sha256', password.encode(), salt.encode(), 100000) return password_hash.hex() == hash_hex except ValueError: return False def generate_cache_key(*args, prefix: str = "", separator: str = ":") -> str: # Cache key generation - could be more sophisticated parts = [str(arg) for arg in args if arg is not None] if prefix: parts.insert(0, prefix) return separator.join(parts) def parse_datetime(date_string: str) -> Optional[datetime]: # Date parsing with multiple format support formats = [ "%Y-%m-%d %H:%M:%S", "%Y-%m-%dT%H:%M:%S", "%Y-%m-%dT%H:%M:%S.%f", "%Y-%m-%d" ] for fmt in formats: try: return datetime.strptime(date_string, fmt) except ValueError: continue logger.warning(f"Unable to parse datetime: {date_string}") return None def calculate_expiry(hours: int = 24) -> datetime: # Expiry calculation - could be more flexible return datetime.utcnow() + timedelta(hours=hours) def sanitize_input(data: Dict[str, Any]) -> Dict[str, Any]: # Input sanitization - basic implementation sanitized = {} for key, value in data.items(): if isinstance(value, str): # Basic HTML/script tag removal value = re.sub(r'<[^>]*>', '', value) value = value.strip() # Type validation could be more comprehensive if value is not None and value != "": sanitized[key] = value return sanitized def format_response(data: Any, status: str = "success", message: str = None) -> Dict[str, Any]: # Response formatting - could be more standardized response = { "status": status, "data": data, "timestamp": datetime.utcnow().isoformat() } if message: response["message"] = message return response class PerformanceTimer: # Performance measurement utility def __init__(self, name: str): self.name = name self.start_time = None def __enter__(self): self.start_time = datetime.now() return self def __exit__(self, exc_type, exc_val, exc_tb): if self.start_time: duration = datetime.now() - self.start_time logger.info(f"Performance: {self.name} took {duration.total_seconds():.3f}s") """ # Create test files self.main_service_file = self.create_additional_test_file("main_service.py", main_service) self.config_file = self.create_additional_test_file("config.py", config_module) self.models_file = self.create_additional_test_file("models.py", models_module) self.utils_file = self.create_additional_test_file("utils.py", utils_module) self.logger.info(" ✅ Created test codebase with 4 files for analysis") def _test_single_analysis_session(self) -> bool: """Test a complete analysis session with multiple steps""" try: self.logger.info(" 1.1: Testing single analysis session") # Step 1: Start analysis self.logger.info(" 1.1.1: Step 1 - Initial analysis") response1, continuation_id = self.call_mcp_tool( "analyze", { "step": "I need to analyze this Python microservice codebase for architectural patterns, design decisions, and improvement opportunities. Let me start by examining the overall structure and understanding the technology stack.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Starting analysis of FastAPI microservice with PostgreSQL, Redis, and SQLAlchemy. Initial examination shows user management functionality with caching layer.", "files_checked": [self.main_service_file], "relevant_files": [self.main_service_file, self.config_file, self.models_file, self.utils_file], "prompt": "Analyze this microservice architecture for scalability, maintainability, and design patterns", "analysis_type": "architecture", }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial analysis response") return False # Parse and validate JSON response response1_data = self._parse_analyze_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_analysis for next_step_required=True if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_analysis"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Deeper examination self.logger.info(" 1.1.2: Step 2 - Architecture examination") response2, _ = self.call_mcp_tool( "analyze", { "step": "Now examining the configuration and models modules to understand data architecture and configuration management patterns.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Found several architectural concerns: direct Redis dependency in service class, global configuration instance, missing error handling in database operations, and mixed serialization logic in models.", "files_checked": [self.main_service_file, self.config_file, self.models_file], "relevant_files": [self.main_service_file, self.config_file, self.models_file], "relevant_context": ["UserService", "AppConfig", "User.to_dict"], "issues_found": [ { "severity": "medium", "description": "Direct dependency on global Redis client in UserService", }, {"severity": "low", "description": "Global configuration instance could cause testing issues"}, ], "confidence": "medium", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue analysis to step 2") return False response2_data = self._parse_analyze_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_analysis"): return False # Check analysis status tracking analysis_status = response2_data.get("analysis_status", {}) if analysis_status.get("files_checked", 0) < 3: self.logger.error("Files checked count not properly tracked") return False if analysis_status.get("insights_by_severity", {}).get("medium", 0) < 1: self.logger.error("Medium severity insights not properly tracked") return False if analysis_status.get("analysis_confidence") != "medium": self.logger.error("Confidence level not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper tracking") # Store continuation_id for next test self.analysis_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single analysis session test failed: {e}") return False def _test_analysis_refocus_flow(self) -> bool: """Test analysis flow that requires refocusing to revise findings""" try: self.logger.info(" 1.2: Testing analysis refocus workflow") # Start a new analysis for testing refocus behaviour self.logger.info(" 1.2.1: Start analysis for refocus test") response1, continuation_id = self.call_mcp_tool( "analyze", { "step": "Analyzing performance characteristics of the data processing pipeline", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial analysis suggests database queries might be the bottleneck", "files_checked": [self.main_service_file], "relevant_files": [self.main_service_file, self.utils_file], "prompt": "Analyze performance bottlenecks in this microservice", "analysis_type": "performance", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start refocus test analysis") return False # Step 2: Wrong direction self.logger.info(" 1.2.2: Step 2 - Incorrect analysis path") response2, _ = self.call_mcp_tool( "analyze", { "step": "Focusing on database optimization strategies", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Database queries seem reasonable, might be looking in wrong direction", "files_checked": [self.main_service_file, self.models_file], "relevant_files": [], "relevant_context": [], "issues_found": [], "confidence": "low", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Adjust investigation path self.logger.info(" 1.2.3: Step 3 - Refocus the analysis") response3, _ = self.call_mcp_tool( "analyze", { "step": "Refocus - the performance issue might not be database related. Let me examine the caching and serialization patterns instead.", "step_number": 3, "total_steps": 4, "next_step_required": True, "findings": "Found potential performance issues in JSON serialization and cache key generation patterns in utils module", "files_checked": [self.utils_file, self.models_file], "relevant_files": [self.utils_file, self.models_file], "relevant_context": ["generate_cache_key", "User.to_dict", "sanitize_input"], "issues_found": [ {"severity": "medium", "description": "JSON serialization in model classes could be optimized"}, {"severity": "low", "description": "Cache key generation lacks proper escaping"}, ], "confidence": "medium", "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to refocus analysis") return False response3_data = self._parse_analyze_response(response3) if not self._validate_step_response(response3_data, 3, 4, True, "pause_for_analysis"): return False self.logger.info(" ✅ Analysis refocus flow working correctly") return True except Exception as e: self.logger.error(f"Backtracking test failed: {e}") return False def _test_complete_analysis_with_expert(self) -> bool: """Test complete analysis ending with expert validation""" try: self.logger.info(" 1.3: Testing complete analysis with expert validation") # Use the continuation from first test continuation_id = getattr(self, "analysis_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh analysis") response0, continuation_id = self.call_mcp_tool( "analyze", { "step": "Analyzing the microservice architecture for improvement opportunities", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found dependency injection and configuration management issues", "files_checked": [self.main_service_file, self.config_file], "relevant_files": [self.main_service_file, self.config_file], "relevant_context": ["UserService", "AppConfig"], "prompt": "Analyze architectural patterns and improvement opportunities", "analysis_type": "architecture", }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh analysis") return False # Final step - trigger expert validation self.logger.info(" 1.3.1: Final step - complete analysis") response_final, _ = self.call_mcp_tool( "analyze", { "step": "Analysis complete. I have identified key architectural patterns and strategic improvement opportunities across scalability, maintainability, and performance dimensions.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert validation "findings": "Key findings: 1) Tight coupling via global dependencies, 2) Missing error handling and transaction management, 3) Mixed concerns in model classes, 4) Configuration management could be more flexible, 5) Opportunities for dependency injection and better separation of concerns.", "files_checked": [self.main_service_file, self.config_file, self.models_file, self.utils_file], "relevant_files": [self.main_service_file, self.config_file, self.models_file, self.utils_file], "relevant_context": ["UserService", "AppConfig", "User", "validate_email"], "issues_found": [ {"severity": "high", "description": "Tight coupling via global Redis client and configuration"}, {"severity": "medium", "description": "Missing transaction management in create_user"}, {"severity": "medium", "description": "Serialization logic mixed with model classes"}, {"severity": "low", "description": "Magic numbers and hardcoded values scattered throughout"}, ], "confidence": "high", "continuation_id": continuation_id, "model": "flash", # Use flash for expert validation }, ) if not response_final: self.logger.error("Failed to complete analysis") return False response_final_data = self._parse_analyze_response(response_final) if not response_final_data: return False # Validate final response structure - expect calling_expert_analysis for next_step_required=False if response_final_data.get("status") != "calling_expert_analysis": self.logger.error( f"Expected status 'calling_expert_analysis', got '{response_final_data.get('status')}'" ) return False if not response_final_data.get("analysis_complete"): self.logger.error("Expected analysis_complete=true for final step") return False # Check for expert analysis if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) # Check for expected analysis content (checking common patterns) analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower() # Look for architectural analysis indicators arch_indicators = ["architecture", "pattern", "coupling", "dependency", "scalability", "maintainability"] found_indicators = sum(1 for indicator in arch_indicators if indicator in analysis_text) if found_indicators >= 3: self.logger.info(" ✅ Expert analysis identified architectural patterns correctly") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully analyzed architecture (found {found_indicators}/6 indicators)" ) # Check complete analysis summary if "complete_analysis" not in response_final_data: self.logger.error("Missing complete_analysis in final response") return False complete_analysis = response_final_data["complete_analysis"] if not complete_analysis.get("relevant_context"): self.logger.error("Missing relevant context in complete analysis") return False if "UserService" not in complete_analysis["relevant_context"]: self.logger.error("Expected context not found in analysis summary") return False self.logger.info(" ✅ Complete analysis with expert validation successful") return True except Exception as e: self.logger.error(f"Complete analysis test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test final step analysis completion (analyze tool doesn't use confidence levels)""" try: self.logger.info(" 1.4: Testing final step analysis completion") # Test final step - analyze tool doesn't use confidence levels, but we test completion self.logger.info(" 1.4.1: Final step analysis") response_final, _ = self.call_mcp_tool( "analyze", { "step": "I have completed a comprehensive analysis of the architectural patterns and improvement opportunities.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step - should trigger expert analysis "findings": "Complete architectural analysis reveals: FastAPI microservice with clear separation needs, dependency injection opportunities, and performance optimization potential. Key patterns identified: service layer, repository-like data access, configuration management, and utility functions.", "files_checked": [self.main_service_file, self.config_file, self.models_file, self.utils_file], "relevant_files": [self.main_service_file, self.config_file, self.models_file, self.utils_file], "relevant_context": ["UserService", "AppConfig", "User", "validate_email"], "issues_found": [ {"severity": "high", "description": "Global dependencies create tight coupling"}, {"severity": "medium", "description": "Transaction management missing in critical operations"}, ], "prompt": "Comprehensive architectural analysis", "analysis_type": "architecture", "model": "flash", }, ) if not response_final: self.logger.error("Failed to test final step analysis") return False response_final_data = self._parse_analyze_response(response_final) if not response_final_data: return False # Validate final step response - should trigger expert analysis expected_status = "calling_expert_analysis" if response_final_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_final_data.get('status')}'") return False # Check that expert analysis was performed expert_analysis = response_final_data.get("expert_analysis", {}) if not expert_analysis: self.logger.error("Expert analysis should be present for final step") return False # Expert analysis should complete successfully if expert_analysis.get("status") != "analysis_complete": self.logger.error( f"Expert analysis status: {expert_analysis.get('status')} (expected analysis_complete)" ) return False self.logger.info(" ✅ Final step analysis completion working correctly") return True except Exception as e: self.logger.error(f"Final step analysis test failed: {e}") return False def _test_context_aware_file_embedding(self) -> bool: """Test context-aware file embedding optimization""" try: self.logger.info(" 1.5: Testing context-aware file embedding") # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "analyze", { "step": "Starting architectural analysis of microservice components", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of service layer and configuration patterns", "files_checked": [self.main_service_file, self.config_file], "relevant_files": [self.main_service_file], # This should be referenced, not embedded "relevant_context": ["UserService"], "issues_found": [{"severity": "medium", "description": "Direct Redis dependency in service class"}], "confidence": "low", "prompt": "Analyze service architecture patterns", "analysis_type": "architecture", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_analyze_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False if "Files referenced but not embedded" not in file_context.get("context_optimization", ""): self.logger.error("Expected context optimization message for reference_only") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Final step - should embed files for expert validation self.logger.info(" 1.5.2: Final step (should embed files)") response2, _ = self.call_mcp_tool( "analyze", { "step": "Analysis complete - identified key architectural patterns and improvement opportunities", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete analysis reveals dependency injection opportunities, configuration management improvements, and separation of concerns enhancements", "files_checked": [self.main_service_file, self.config_file, self.models_file], "relevant_files": [self.main_service_file, self.config_file], # Should be fully embedded "relevant_context": ["UserService", "AppConfig"], "issues_found": [ {"severity": "high", "description": "Global dependencies create architectural coupling"}, {"severity": "medium", "description": "Configuration management lacks flexibility"}, ], "confidence": "high", "model": "flash", }, ) if not response2: self.logger.error("Failed to complete to final step") return False response2_data = self._parse_analyze_response(response2) if not response2_data: return False # Check file context - should be fully_embedded for final step file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context2.get('type')}" ) return False if "Full file content embedded for expert analysis" not in file_context2.get("context_optimization", ""): self.logger.error("Expected expert analysis optimization message for fully_embedded") return False # Verify expert analysis was called for final step if response2_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False if "expert_analysis" not in response2_data: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Context-aware file embedding test completed successfully") return True except Exception as e: self.logger.error(f"Context-aware file embedding test failed: {e}") return False def _test_analysis_types(self) -> bool: """Test different analysis types (architecture, performance, security, quality)""" try: self.logger.info(" 1.6: Testing different analysis types") # Test security analysis self.logger.info(" 1.6.1: Security analysis") response_security, _ = self.call_mcp_tool( "analyze", { "step": "Conducting security analysis of authentication and data handling patterns", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Security analysis reveals: password hashing implementation, input validation patterns, SQL injection prevention via parameterized queries, but missing input sanitization in some areas and weak default secret key handling.", "files_checked": [self.main_service_file, self.utils_file], "relevant_files": [self.main_service_file, self.utils_file], "relevant_context": ["hash_password", "validate_email", "sanitize_input"], "issues_found": [ {"severity": "critical", "description": "Weak default secret key in production detection"}, {"severity": "medium", "description": "Input sanitization not consistently applied"}, ], "confidence": "high", "prompt": "Analyze security patterns and vulnerabilities", "analysis_type": "security", "model": "flash", }, ) if not response_security: self.logger.error("Failed security analysis test") return False response_security_data = self._parse_analyze_response(response_security) if not response_security_data: return False # Check that security analysis was processed issues = response_security_data.get("complete_analysis", {}).get("issues_found", []) critical_issues = [issue for issue in issues if issue.get("severity") == "critical"] if not critical_issues: self.logger.warning("Security analysis should have identified critical security issues") else: self.logger.info(" ✅ Security analysis identified critical issues") # Test quality analysis self.logger.info(" 1.6.2: Quality analysis") response_quality, _ = self.call_mcp_tool( "analyze", { "step": "Conducting code quality analysis focusing on maintainability and best practices", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Code quality analysis shows: good use of type hints, proper error handling in some areas but missing in others, mixed separation of concerns, and opportunities for better abstraction.", "files_checked": [self.models_file, self.utils_file], "relevant_files": [self.models_file, self.utils_file], "relevant_context": ["User.to_dict", "ValidationError", "PerformanceTimer"], "issues_found": [ {"severity": "medium", "description": "Serialization logic mixed with model classes"}, {"severity": "low", "description": "Inconsistent error handling patterns"}, ], "confidence": "high", "prompt": "Analyze code quality and maintainability patterns", "analysis_type": "quality", "model": "flash", }, ) if not response_quality: self.logger.error("Failed quality analysis test") return False response_quality_data = self._parse_analyze_response(response_quality) if not response_quality_data: return False # Verify quality analysis was processed quality_context = response_quality_data.get("complete_analysis", {}).get("relevant_context", []) if not any("User" in ctx for ctx in quality_context): self.logger.warning("Quality analysis should have analyzed model classes") else: self.logger.info(" ✅ Quality analysis examined relevant code elements") self.logger.info(" ✅ Different analysis types test completed successfully") return True except Exception as e: self.logger.error(f"Analysis types test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for analyze-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from analyze response specifically continuation_id = self._extract_analyze_continuation_id(response_text) return response_text, continuation_id def _extract_analyze_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from analyze response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for analyze continuation_id: {e}") return None def _parse_analyze_response(self, response_text: str) -> dict: """Parse analyze tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse analyze response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate an analyze investigation step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check analysis_status exists if "analysis_status" not in response_data: self.logger.error("Missing analysis_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False ================================================ FILE: simulator_tests/test_basic_conversation.py ================================================ #!/usr/bin/env python3 """ Basic Conversation Flow Test Tests basic conversation continuity with the chat tool, including: - Initial chat with file analysis - Continuing conversation with same file (deduplication) - Adding additional files to ongoing conversation """ from .base_test import BaseSimulatorTest class BasicConversationTest(BaseSimulatorTest): """Test basic conversation flow with chat tool""" @property def test_name(self) -> str: return "basic_conversation" @property def test_description(self) -> str: return "Basic conversation flow with chat tool" def run_test(self) -> bool: """Test basic conversation flow with chat tool""" try: self.logger.info("Test: Basic conversation flow") # Setup test files self.setup_test_files() # Initial chat tool call with file self.logger.info(" 1.1: Initial chat with file analysis") response1, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Please use low thinking mode. Analyze this Python code and explain what it does", "absolute_file_paths": [self.test_files["python"]], "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial response with continuation_id") return False self.logger.info(f" ✅ Got continuation_id: {continuation_id}") # Continue conversation with same file (should be deduplicated) self.logger.info(" 1.2: Continue conversation with same file") response2, _ = self.call_mcp_tool( "chat", { "prompt": "Please use low thinking mode. Now focus on the Calculator class specifically. Are there any improvements you'd suggest?", "absolute_file_paths": [self.test_files["python"]], # Same file - should be deduplicated "continuation_id": continuation_id, "model": "flash", }, ) if not response2: self.logger.error("Failed to continue conversation") return False # Continue with additional file self.logger.info(" 1.3: Continue conversation with additional file") response3, _ = self.call_mcp_tool( "chat", { "prompt": "Please use low thinking mode. Now also analyze this configuration file and see how it might relate to the Python code", "absolute_file_paths": [self.test_files["python"], self.test_files["config"]], "continuation_id": continuation_id, "model": "flash", }, ) if not response3: self.logger.error("Failed to continue with additional file") return False self.logger.info(" ✅ Basic conversation flow working") return True except Exception as e: self.logger.error(f"Basic conversation flow test failed: {e}") return False finally: self.cleanup_test_files() ================================================ FILE: simulator_tests/test_chat_simple_validation.py ================================================ #!/usr/bin/env python3 """ Chat Simple Tool Validation Test Comprehensive test for the new ChatSimple tool implementation that validates: - Basic conversation flow without continuation_id (new chats) - Continuing existing conversations with continuation_id (continued chats) - File handling with conversation context (chats with files) - Image handling in conversations (chat with images) - Continuing conversations with files from previous turns (continued chats with files previously) - Temperature validation for different models - Image limit validation per model - Conversation context preservation across turns """ from .conversation_base_test import ConversationBaseTest class ChatSimpleValidationTest(ConversationBaseTest): """Test ChatSimple tool functionality and validation""" @property def test_name(self) -> str: return "_validation" @property def test_description(self) -> str: return "Comprehensive validation of ChatSimple tool implementation" def run_test(self) -> bool: """Run comprehensive ChatSimple validation tests""" try: # Set up the test environment for in-process testing self.setUp() self.logger.info("Test: ChatSimple tool validation") # Run all test scenarios if not self.test_new_conversation_no_continuation(): return False if not self.test_continue_existing_conversation(): return False if not self.test_file_handling_with_conversation(): return False if not self.test_temperature_validation_edge_cases(): return False if not self.test_image_limits_per_model(): return False if not self.test_conversation_context_preservation(): return False if not self.test_chat_with_images(): return False if not self.test_continued_chat_with_previous_files(): return False self.logger.info(" ✅ All ChatSimple validation tests passed") return True except Exception as e: self.logger.error(f"ChatSimple validation test failed: {e}") return False def test_new_conversation_no_continuation(self) -> bool: """Test ChatSimple creates new conversation without continuation_id""" try: self.logger.info(" 1. Test new conversation without continuation_id") # Call chat without continuation_id response, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": "Hello! Please use low thinking mode. Can you explain what MCP tools are?", "model": "flash", "temperature": 0.7, "thinking_mode": "low", }, ) if not response: self.logger.error(" ❌ Failed to get response from chat") return False if not continuation_id: self.logger.error(" ❌ No continuation_id returned for new conversation") return False # Verify response mentions MCP or tools if "MCP" not in response and "tool" not in response.lower(): self.logger.error(" ❌ Response doesn't seem to address the question about MCP tools") return False self.logger.info(f" ✅ New conversation created with continuation_id: {continuation_id}") self.new_continuation_id = continuation_id # Store for next test return True except Exception as e: self.logger.error(f" ❌ New conversation test failed: {e}") return False def test_continue_existing_conversation(self) -> bool: """Test ChatSimple continues conversation with valid continuation_id""" try: self.logger.info(" 2. Test continuing existing conversation") if not hasattr(self, "new_continuation_id"): self.logger.error(" ❌ No continuation_id from previous test") return False # Continue the conversation response, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Can you give me a specific example of how an MCP tool might work?", "continuation_id": self.new_continuation_id, "model": "flash", "thinking_mode": "low", }, ) if not response: self.logger.error(" ❌ Failed to continue conversation") return False # Continuation ID should be the same if continuation_id != self.new_continuation_id: self.logger.error(f" ❌ Continuation ID changed: {self.new_continuation_id} -> {continuation_id}") return False # Response should be contextual (mentioning previous discussion) if "example" not in response.lower(): self.logger.error(" ❌ Response doesn't seem to provide an example as requested") return False self.logger.info(" ✅ Successfully continued conversation with same continuation_id") return True except Exception as e: self.logger.error(f" ❌ Continue conversation test failed: {e}") return False def test_file_handling_with_conversation(self) -> bool: """Test ChatSimple handles files correctly in conversation context""" try: self.logger.info(" 3. Test file handling with conversation") # Setup test files self.setup_test_files() # Start new conversation with a file response1, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Analyze this Python code and tell me what the Calculator class does", "absolute_file_paths": [self.test_files["python"]], "model": "flash", "thinking_mode": "low", }, ) if not response1 or not continuation_id: self.logger.error(" ❌ Failed to start conversation with file") return False # Continue with same file (should be deduplicated) response2, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. What methods does the Calculator class have?", "absolute_file_paths": [self.test_files["python"]], # Same file "continuation_id": continuation_id, "model": "flash", "thinking_mode": "low", }, ) if not response2: self.logger.error(" ❌ Failed to continue with same file") return False # Response should mention add and multiply methods if "add" not in response2.lower() or "multiply" not in response2.lower(): self.logger.error(" ❌ Response doesn't mention Calculator methods") return False self.logger.info(" ✅ File handling with conversation working correctly") return True except Exception as e: self.logger.error(f" ❌ File handling test failed: {e}") return False finally: self.cleanup_test_files() def test_temperature_validation_edge_cases(self) -> bool: """Test temperature is corrected for model limits (too high/low)""" try: self.logger.info(" 4. Test temperature validation edge cases") # Test 1: Temperature exactly at limit (should work) response1, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Hello, this is a test with max temperature", "model": "flash", "temperature": 1.0, # At the limit "thinking_mode": "low", }, ) if not response1: self.logger.error(" ❌ Failed with temperature 1.0") return False # Test 2: Temperature at minimum (should work) response2, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Another test message with min temperature", "model": "flash", "temperature": 0.0, # At minimum "thinking_mode": "low", }, ) if not response2: self.logger.error(" ❌ Failed with temperature 0.0") return False # Test 3: Check that invalid temperatures are rejected by validation # This should result in an error response from the tool, not a crash try: response3, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Test with invalid temperature", "model": "flash", "temperature": 1.5, # Too high - should be validated "thinking_mode": "low", }, ) # If we get here, check if it's an error response if response3 and "validation error" in response3.lower(): self.logger.info(" ✅ Invalid temperature properly rejected by validation") else: self.logger.warning(" ⚠️ High temperature not properly validated") except Exception: # Expected - validation should reject this self.logger.info(" ✅ Invalid temperature properly rejected") self.logger.info(" ✅ Temperature validation working correctly") return True except Exception as e: self.logger.error(f" ❌ Temperature validation test failed: {e}") return False def test_image_limits_per_model(self) -> bool: """Test image validation respects model-specific limits""" try: self.logger.info(" 5. Test image limits per model") # Create test image data URLs (small base64 images) small_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" # Test 1: Model that doesn't support images response1, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Can you see this image?", "model": "local-llama", # Text-only model "images": [small_image], "thinking_mode": "low", }, ) # Should get an error about image support if response1 and "does not support image" not in response1: self.logger.warning(" ⚠️ Model without image support didn't reject images properly") # Test 2: Too many images for a model many_images = [small_image] * 25 # Most models support max 20 response2, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Analyze these images", "model": "gemini-2.5-flash", # Supports max 16 images "images": many_images, "thinking_mode": "low", }, ) # Should get an error about too many images if response2 and "too many images" not in response2.lower(): self.logger.warning(" ⚠️ Model didn't reject excessive image count") # Test 3: Valid image count response3, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. This is a test with one image", "model": "gemini-2.5-flash", "images": [small_image], "thinking_mode": "low", }, ) if not response3: self.logger.error(" ❌ Failed with valid image count") return False self.logger.info(" ✅ Image validation working correctly") return True except Exception as e: self.logger.error(f" ❌ Image limits test failed: {e}") return False def test_conversation_context_preservation(self) -> bool: """Test ChatSimple preserves context across turns""" try: self.logger.info(" 6. Test conversation context preservation") # Start conversation with specific context response1, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. My name is TestUser and I'm working on a Python project called TestProject", "model": "flash", "thinking_mode": "low", }, ) if not response1 or not continuation_id: self.logger.error(" ❌ Failed to start conversation") return False # Continue and reference previous context response2, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. What's my name and what project am I working on?", "continuation_id": continuation_id, "model": "flash", "thinking_mode": "low", }, ) if not response2: self.logger.error(" ❌ Failed to continue conversation") return False # Check if context was preserved if "TestUser" not in response2 or "TestProject" not in response2: self.logger.error(" ❌ Context not preserved across conversation turns") self.logger.debug(f" Response: {response2[:200]}...") return False self.logger.info(" ✅ Conversation context preserved correctly") return True except Exception as e: self.logger.error(f" ❌ Context preservation test failed: {e}") return False def test_chat_with_images(self) -> bool: """Test ChatSimple handles images correctly in conversation""" try: self.logger.info(" 7. Test chat with images") # Create test image data URL (small base64 image) small_image = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" # Start conversation with image response1, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. I'm sharing an image with you. Can you acknowledge that you received it?", "images": [small_image], "model": "gemini-2.5-flash", # Model that supports images "thinking_mode": "low", }, ) if not response1 or not continuation_id: self.logger.error(" ❌ Failed to start conversation with image") return False # Verify response acknowledges the image if "image" not in response1.lower(): self.logger.warning(" ⚠️ Response doesn't acknowledge receiving image") # Continue conversation referencing the image response2, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. What did you see in that image I shared earlier?", "continuation_id": continuation_id, "model": "gemini-2.5-flash", "thinking_mode": "low", }, ) if not response2: self.logger.error(" ❌ Failed to continue conversation about image") return False # Test with multiple images multiple_images = [small_image, small_image] # Two identical small images response3, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Here are two images for comparison", "images": multiple_images, "model": "gemini-2.5-flash", "thinking_mode": "low", }, ) if not response3: self.logger.error(" ❌ Failed with multiple images") return False self.logger.info(" ✅ Chat with images working correctly") return True except Exception as e: self.logger.error(f" ❌ Chat with images test failed: {e}") return False def test_continued_chat_with_previous_files(self) -> bool: """Test continuing conversation where files were shared in previous turns""" try: self.logger.info(" 8. Test continued chat with files from previous turns") # Setup test files self.setup_test_files() # Start conversation with files response1, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Here are some files for you to analyze", "absolute_file_paths": [self.test_files["python"], self.test_files["config"]], "model": "flash", "thinking_mode": "low", }, ) if not response1 or not continuation_id: self.logger.error(" ❌ Failed to start conversation with files") return False # Continue conversation without new files (should remember previous files) response2, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. From the files I shared earlier, what types of files were there?", "continuation_id": continuation_id, "model": "flash", "thinking_mode": "low", }, ) if not response2: self.logger.error(" ❌ Failed to continue conversation") return False # Check if response references the files from previous turn if "python" not in response2.lower() and "config" not in response2.lower(): self.logger.warning(" ⚠️ Response doesn't reference previous files properly") # Continue with a different question about same files (should still remember them) response3, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Please use low thinking mode. Can you tell me what functions were defined in the Python file from our earlier discussion?", "continuation_id": continuation_id, "model": "flash", "thinking_mode": "low", }, ) if not response3: self.logger.error(" ❌ Failed to continue conversation about Python file") return False # Should reference functions from the Python file (fibonacci, factorial, Calculator, etc.) response_lower = response3.lower() if not ("fibonacci" in response_lower or "factorial" in response_lower or "calculator" in response_lower): self.logger.warning(" ⚠️ Response doesn't reference Python file contents from earlier turn") self.logger.info(" ✅ Continued chat with previous files working correctly") return True except Exception as e: self.logger.error(f" ❌ Continued chat with files test failed: {e}") return False finally: self.cleanup_test_files() ================================================ FILE: simulator_tests/test_codereview_validation.py ================================================ #!/usr/bin/env python3 """ CodeReview Tool Validation Test Tests the codereview tool's capabilities using the new workflow architecture. This validates that the workflow-based code review provides step-by-step analysis with proper investigation guidance and expert analysis integration. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class CodeReviewValidationTest(ConversationBaseTest): """Test codereview tool with new workflow architecture""" @property def test_name(self) -> str: return "codereview_validation" @property def test_description(self) -> str: return "CodeReview tool validation with new workflow architecture" def run_test(self) -> bool: """Test codereview tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: CodeReviewWorkflow tool validation (new architecture)") # Create test code with various issues for review self._create_test_code_for_review() # Test 1: Single review session with multiple steps if not self._test_single_review_session(): return False # Test 2: Review flow that requires refocusing if not self._test_review_refocus_flow(): return False # Test 3: Complete review with expert analysis if not self._test_complete_review_with_analysis(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Context-aware file embedding if not self._test_context_aware_file_embedding(): return False # Test 6: Multi-step file context optimization if not self._test_multi_step_file_context(): return False self.logger.info(" ✅ All codereview validation tests passed") return True except Exception as e: self.logger.error(f"CodeReviewWorkflow validation test failed: {e}") return False def _create_test_code_for_review(self): """Create test files with various code quality issues for review""" # Create a payment processing module with multiple issues payment_code = """#!/usr/bin/env python3 import hashlib import requests import json from datetime import datetime class PaymentProcessor: def __init__(self, api_key): self.api_key = api_key # Security issue: API key stored in plain text self.base_url = "https://payment-gateway.example.com" self.session = requests.Session() self.failed_payments = [] # Performance issue: unbounded list def process_payment(self, amount, card_number, cvv, user_id): \"\"\"Process a payment transaction\"\"\" # Security issue: No input validation # Performance issue: Inefficient nested loops for attempt in range(3): for retry in range(5): try: # Security issue: Logging sensitive data print(f"Processing payment: {card_number}, CVV: {cvv}") # Over-engineering: Complex hashing that's not needed payment_hash = self._generate_complex_hash(amount, card_number, cvv, user_id, datetime.now()) # Security issue: Insecure HTTP request construction url = f"{self.base_url}/charge?amount={amount}&card={card_number}&api_key={self.api_key}" response = self.session.get(url) # Security issue: using GET for sensitive data if response.status_code == 200: return {"status": "success", "hash": payment_hash} else: # Code smell: Generic exception handling without specific error types self.failed_payments.append({"amount": amount, "timestamp": datetime.now()}) except Exception as e: # Code smell: Bare except clause and poor error handling print(f"Payment failed: {e}") continue return {"status": "failed"} def _generate_complex_hash(self, amount, card_number, cvv, user_id, timestamp): \"\"\"Over-engineered hash generation with unnecessary complexity\"\"\" # Over-engineering: Overly complex for no clear benefit combined = f"{amount}-{card_number}-{cvv}-{user_id}-{timestamp}" # Security issue: Weak hashing algorithm hash1 = hashlib.md5(combined.encode()).hexdigest() hash2 = hashlib.sha1(hash1.encode()).hexdigest() hash3 = hashlib.md5(hash2.encode()).hexdigest() # Performance issue: Unnecessary string operations in loop result = "" for i in range(len(hash3)): for j in range(3): # Arbitrary nested loop result += hash3[i] if i % 2 == 0 else hash3[i].upper() return result[:32] # Arbitrary truncation def get_payment_history(self, user_id): \"\"\"Get payment history - has scalability issues\"\"\" # Performance issue: No pagination, could return massive datasets # Performance issue: Inefficient algorithm O(n²) all_payments = self._fetch_all_payments() # Could be millions of records user_payments = [] for payment in all_payments: for field in payment: # Unnecessary nested iteration if field == "user_id" and payment[field] == user_id: user_payments.append(payment) break return user_payments def _fetch_all_payments(self): \"\"\"Simulated method that would fetch all payments\"\"\" # Maintainability issue: Hard-coded test data return [ {"user_id": 1, "amount": 100, "status": "success"}, {"user_id": 2, "amount": 200, "status": "failed"}, {"user_id": 1, "amount": 150, "status": "success"}, ] """ # Create test file with multiple issues self.payment_file = self.create_additional_test_file("payment_processor.py", payment_code) self.logger.info(f" ✅ Created test file with code issues: {self.payment_file}") # Create configuration file with additional issues config_code = """#!/usr/bin/env python3 import os # Security issue: Hardcoded secrets DATABASE_PASSWORD = "admin123" SECRET_KEY = "my-secret-key-12345" # Over-engineering: Unnecessarily complex configuration class class ConfigurationManager: def __init__(self): self.config_cache = {} self.config_hierarchy = {} self.config_validators = {} self.config_transformers = {} self.config_listeners = [] def get_config(self, key, default=None): # Over-engineering: Complex caching for simple config lookup if key in self.config_cache: cached_value = self.config_cache[key] if self._validate_cached_value(cached_value): return self._transform_value(key, cached_value) # Code smell: Complex nested conditionals if key in self.config_hierarchy: hierarchy = self.config_hierarchy[key] for level in hierarchy: if level == "env": value = os.getenv(key.upper(), default) elif level == "file": value = self._read_from_file(key, default) elif level == "database": value = self._read_from_database(key, default) else: value = default if value is not None: self.config_cache[key] = value return self._transform_value(key, value) return default def _validate_cached_value(self, value): # Maintainability issue: Unclear validation logic if isinstance(value, str) and len(value) > 1000: return False return True def _transform_value(self, key, value): # Code smell: Unnecessary abstraction if key in self.config_transformers: transformer = self.config_transformers[key] return transformer(value) return value def _read_from_file(self, key, default): # Maintainability issue: No error handling for file operations with open(f"/etc/app/{key}.conf") as f: return f.read().strip() def _read_from_database(self, key, default): # Performance issue: Database query for every config read # No connection pooling or caching import sqlite3 conn = sqlite3.connect("config.db") cursor = conn.cursor() cursor.execute("SELECT value FROM config WHERE key = ?", (key,)) result = cursor.fetchone() conn.close() return result[0] if result else default """ self.config_file = self.create_additional_test_file("config.py", config_code) self.logger.info(f" ✅ Created configuration file with issues: {self.config_file}") def _test_single_review_session(self) -> bool: """Test a complete code review session with multiple steps""" try: self.logger.info(" 1.1: Testing single code review session") # Step 1: Start review self.logger.info(" 1.1.1: Step 1 - Initial review") response1, continuation_id = self.call_mcp_tool( "codereview", { "step": "I need to perform a comprehensive code review of the payment processing module. Let me start by examining the code structure and identifying potential issues.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial examination reveals a payment processing class with potential security and performance concerns.", "files_checked": [self.payment_file], "relevant_files": [self.payment_file], "absolute_file_paths": [self.payment_file], # Required for step 1 "review_type": "full", "severity_filter": "all", }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial review response") return False # Parse and validate JSON response response1_data = self._parse_review_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_code_review for next_step_required=True if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_code_review"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Detailed analysis self.logger.info(" 1.1.2: Step 2 - Detailed security analysis") response2, _ = self.call_mcp_tool( "codereview", { "step": "Now performing detailed security analysis of the payment processor code to identify vulnerabilities and code quality issues.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Found multiple security issues: API key stored in plain text, sensitive data logging, insecure HTTP methods, and weak hashing algorithms.", "files_checked": [self.payment_file], "relevant_files": [self.payment_file], "relevant_context": ["PaymentProcessor.__init__", "PaymentProcessor.process_payment"], "issues_found": [ {"severity": "critical", "description": "API key stored in plain text in memory"}, {"severity": "critical", "description": "Credit card and CVV logged in plain text"}, {"severity": "high", "description": "Using GET method for sensitive payment data"}, {"severity": "medium", "description": "Weak MD5 hashing algorithm used"}, ], "confidence": "high", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue review to step 2") return False response2_data = self._parse_review_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_code_review"): return False # Check review status tracking review_status = response2_data.get("code_review_status", {}) if review_status.get("files_checked", 0) < 1: self.logger.error("Files checked count not properly tracked") return False if review_status.get("relevant_context", 0) != 2: self.logger.error("Relevant context not properly tracked") return False # Check issues by severity issues_by_severity = review_status.get("issues_by_severity", {}) if issues_by_severity.get("critical", 0) != 2: self.logger.error("Critical issues not properly tracked") return False if issues_by_severity.get("high", 0) != 1: self.logger.error("High severity issues not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper issue tracking") # Store continuation_id for next test self.review_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single review session test failed: {e}") return False def _test_review_refocus_flow(self) -> bool: """Test code review flow that revises findings by refocusing""" try: self.logger.info(" 1.2: Testing code review refocus workflow") # Start a new review for testing refocus behaviour self.logger.info(" 1.2.1: Start review for refocus test") response1, continuation_id = self.call_mcp_tool( "codereview", { "step": "Reviewing configuration management code for best practices", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial analysis shows complex configuration class", "files_checked": [self.config_file], "relevant_files": [self.config_file], "absolute_file_paths": [self.config_file], "review_type": "full", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start refocus test review") return False # Step 2: Initial direction self.logger.info(" 1.2.2: Step 2 - Initial analysis direction") response2, _ = self.call_mcp_tool( "codereview", { "step": "Focusing on configuration architecture patterns", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Architecture seems overly complex, but need to look more carefully at security issues", "files_checked": [self.config_file], "relevant_files": [self.config_file], "issues_found": [ {"severity": "medium", "description": "Complex configuration hierarchy"}, ], "confidence": "low", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Shift focus based on new evidence self.logger.info(" 1.2.3: Step 3 - Refocus on security issues") response3, _ = self.call_mcp_tool( "codereview", { "step": "Refocusing - need to concentrate on the critical security issues I initially missed. Found hardcoded secrets and credentials in plain text.", "step_number": 3, "total_steps": 4, "next_step_required": True, "findings": "Found critical security vulnerabilities: hardcoded DATABASE_PASSWORD and SECRET_KEY in plain text", "files_checked": [self.config_file], "relevant_files": [self.config_file], "relevant_context": ["ConfigurationManager.__init__"], "issues_found": [ {"severity": "critical", "description": "Hardcoded database password in source code"}, {"severity": "critical", "description": "Hardcoded secret key in source code"}, {"severity": "high", "description": "Over-engineered configuration system"}, ], "confidence": "high", "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to refocus") return False response3_data = self._parse_review_response(response3) if not self._validate_step_response(response3_data, 3, 4, True, "pause_for_code_review"): return False self.logger.info(" ✅ Refocus flow working correctly") return True except Exception as e: self.logger.error(f"Refocus test failed: {e}") return False def _test_complete_review_with_analysis(self) -> bool: """Test complete code review ending with expert analysis""" try: self.logger.info(" 1.3: Testing complete review with expert analysis") # Use the continuation from first test continuation_id = getattr(self, "review_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh review") response0, continuation_id = self.call_mcp_tool( "codereview", { "step": "Reviewing payment processor for security and quality issues", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found multiple security and performance issues", "files_checked": [self.payment_file], "relevant_files": [self.payment_file], "absolute_file_paths": [self.payment_file], "relevant_context": ["PaymentProcessor.process_payment"], }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh review") return False # Final step - trigger expert analysis self.logger.info(" 1.3.1: Final step - complete review") response_final, _ = self.call_mcp_tool( "codereview", { "step": "Code review complete. Identified comprehensive security, performance, and maintainability issues throughout the payment processing module.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert analysis "findings": "Complete analysis reveals critical security vulnerabilities, performance bottlenecks, over-engineering patterns, and maintainability concerns. All issues documented with severity levels.", "files_checked": [self.payment_file], "relevant_files": [self.payment_file], "relevant_context": [ "PaymentProcessor.process_payment", "PaymentProcessor._generate_complex_hash", "PaymentProcessor.get_payment_history", ], "issues_found": [ {"severity": "critical", "description": "API key stored in plain text"}, {"severity": "critical", "description": "Sensitive payment data logged"}, {"severity": "high", "description": "SQL injection vulnerability potential"}, {"severity": "medium", "description": "Over-engineered hash generation"}, {"severity": "low", "description": "Poor error handling patterns"}, ], "confidence": "high", "continuation_id": continuation_id, "model": "flash", # Use flash for expert analysis }, ) if not response_final: self.logger.error("Failed to complete review") return False response_final_data = self._parse_review_response(response_final) if not response_final_data: return False # Validate final response structure - expect calling_expert_analysis for next_step_required=False if response_final_data.get("status") != "calling_expert_analysis": self.logger.error( f"Expected status 'calling_expert_analysis', got '{response_final_data.get('status')}'" ) return False if not response_final_data.get("code_review_complete"): self.logger.error("Expected code_review_complete=true for final step") return False # Check for expert analysis if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) # Check for expected analysis content (checking common patterns) analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower() # Look for code review identification review_indicators = ["security", "vulnerability", "performance", "critical", "api", "key"] found_indicators = sum(1 for indicator in review_indicators if indicator in analysis_text) if found_indicators >= 3: self.logger.info(" ✅ Expert analysis identified the issues correctly") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully identified the issues (found {found_indicators}/6 indicators)" ) # Check complete review summary if "complete_code_review" not in response_final_data: self.logger.error("Missing complete_code_review in final response") return False complete_review = response_final_data["complete_code_review"] if not complete_review.get("relevant_context"): self.logger.error("Missing relevant context in complete review") return False if "PaymentProcessor.process_payment" not in complete_review["relevant_context"]: self.logger.error("Expected method not found in review summary") return False self.logger.info(" ✅ Complete review with expert analysis successful") return True except Exception as e: self.logger.error(f"Complete review test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test certain confidence behavior - should skip expert analysis""" try: self.logger.info(" 1.4: Testing certain confidence behavior") # Test certain confidence - should skip expert analysis self.logger.info(" 1.4.1: Certain confidence review") response_certain, _ = self.call_mcp_tool( "codereview", { "step": "I have completed a thorough code review with 100% certainty of all issues identified.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "Complete review identified all critical security issues, performance problems, and code quality concerns. All issues are documented with clear severity levels and specific recommendations.", "files_checked": [self.payment_file], "relevant_files": [self.payment_file], "absolute_file_paths": [self.payment_file], "relevant_context": ["PaymentProcessor.process_payment"], "issues_found": [ {"severity": "critical", "description": "Hardcoded API key security vulnerability"}, {"severity": "high", "description": "Performance bottleneck in payment history"}, ], "review_validation_type": "internal", # This should skip expert analysis "model": "flash", }, ) if not response_certain: self.logger.error("Failed to test certain confidence") return False response_certain_data = self._parse_review_response(response_certain) if not response_certain_data: return False # Validate certain confidence response - should skip expert analysis if response_certain_data.get("status") != "code_review_complete_ready_for_implementation": self.logger.error( f"Expected status 'code_review_complete_ready_for_implementation', got '{response_certain_data.get('status')}'" ) return False if not response_certain_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for certain confidence") return False expert_analysis = response_certain_data.get("expert_analysis", {}) if expert_analysis.get("status") not in [ "skipped_due_to_certain_review_confidence", "skipped_due_to_internal_analysis_type", ]: self.logger.error("Expert analysis should be skipped for certain confidence") return False self.logger.info(" ✅ Certain confidence behavior working correctly") return True except Exception as e: self.logger.error(f"Certain confidence test failed: {e}") return False def _test_context_aware_file_embedding(self) -> bool: """Test context-aware file embedding optimization""" try: self.logger.info(" 1.5: Testing context-aware file embedding") # Create multiple test files for context testing utils_content = """#!/usr/bin/env python3 def calculate_discount(price, discount_percent): \"\"\"Calculate discount amount\"\"\" if discount_percent < 0 or discount_percent > 100: raise ValueError("Invalid discount percentage") return price * (discount_percent / 100) def format_currency(amount): \"\"\"Format amount as currency\"\"\" return f"${amount:.2f}" """ validator_content = """#!/usr/bin/env python3 import re def validate_email(email): \"\"\"Validate email format\"\"\" pattern = r'^[a-zA-Z0-9._%+-]+@[a-zA-Z0-9.-]+\\.[a-zA-Z]{2,}$' return re.match(pattern, email) is not None def validate_credit_card(card_number): \"\"\"Basic credit card validation\"\"\" # Remove spaces and dashes card_number = re.sub(r'[\\s-]', '', card_number) # Check if all digits if not card_number.isdigit(): return False # Basic length check return len(card_number) in [13, 14, 15, 16] """ # Create test files utils_file = self.create_additional_test_file("utils.py", utils_content) validator_file = self.create_additional_test_file("validator.py", validator_content) # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "codereview", { "step": "Starting comprehensive code review of utility modules", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of utility and validation functions", "files_checked": [utils_file, validator_file], "relevant_files": [utils_file], # This should be referenced, not embedded "absolute_file_paths": [utils_file, validator_file], # Required for step 1 "relevant_context": ["calculate_discount"], "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_review_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False if "Files referenced but not embedded" not in file_context.get("context_optimization", ""): self.logger.error("Expected context optimization message for reference_only") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Final step - should embed files for expert analysis self.logger.info(" 1.5.2: Final step (should embed files)") response3, _ = self.call_mcp_tool( "codereview", { "step": "Code review complete - identified all issues and recommendations", "step_number": 3, "total_steps": 3, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete review: utility functions have proper error handling, validation functions are robust", "files_checked": [utils_file, validator_file], "relevant_files": [utils_file, validator_file], # Should be fully embedded "relevant_context": ["calculate_discount", "validate_email", "validate_credit_card"], "issues_found": [ {"severity": "low", "description": "Could add more comprehensive email validation"}, {"severity": "medium", "description": "Credit card validation logic could be more robust"}, ], "confidence": "medium", "model": "flash", }, ) if not response3: self.logger.error("Failed to complete to final step") return False response3_data = self._parse_review_response(response3) if not response3_data: return False # Check file context - should be fully_embedded for final step file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context3.get('type')}" ) return False if "Full file content embedded for expert analysis" not in file_context3.get("context_optimization", ""): self.logger.error("Expected expert analysis optimization message for fully_embedded") return False self.logger.info(" ✅ Final step correctly uses fully_embedded file context") # Verify expert analysis was called for final step if response3_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False if "expert_analysis" not in response3_data: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Context-aware file embedding test completed successfully") return True except Exception as e: self.logger.error(f"Context-aware file embedding test failed: {e}") return False def _test_multi_step_file_context(self) -> bool: """Test multi-step workflow with proper file context transitions""" try: self.logger.info(" 1.6: Testing multi-step file context optimization") # Use existing payment and config files for multi-step test files_to_review = [self.payment_file, self.config_file] # Step 1: Start review (new conversation) self.logger.info(" 1.6.1: Step 1 - Start comprehensive review") response1, continuation_id = self.call_mcp_tool( "codereview", { "step": "Starting comprehensive security and quality review of payment system components", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial review of payment processor and configuration management modules", "files_checked": files_to_review, "relevant_files": [self.payment_file], "absolute_file_paths": files_to_review, "relevant_context": [], "confidence": "low", "review_type": "security", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start multi-step file context test") return False response1_data = self._parse_review_response(response1) # Validate step 1 - should use reference_only file_context1 = response1_data.get("file_context", {}) if file_context1.get("type") != "reference_only": self.logger.error("Step 1 should use reference_only file context") return False self.logger.info(" ✅ Step 1: reference_only file context") # Step 2: Security analysis self.logger.info(" 1.6.2: Step 2 - Security analysis") response2, _ = self.call_mcp_tool( "codereview", { "step": "Focusing on critical security vulnerabilities across both modules", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Found critical security issues: hardcoded secrets in config, API key exposure in payment processor", "files_checked": files_to_review, "relevant_files": files_to_review, "relevant_context": ["PaymentProcessor.__init__", "ConfigurationManager"], "issues_found": [ {"severity": "critical", "description": "Hardcoded database password"}, {"severity": "critical", "description": "API key stored in plain text"}, ], "confidence": "medium", "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False response2_data = self._parse_review_response(response2) # Validate step 2 - should still use reference_only file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "reference_only": self.logger.error("Step 2 should use reference_only file context") return False self.logger.info(" ✅ Step 2: reference_only file context") # Step 3: Performance and architecture analysis self.logger.info(" 1.6.3: Step 3 - Performance and architecture analysis") response3, _ = self.call_mcp_tool( "codereview", { "step": "Analyzing performance bottlenecks and architectural concerns", "step_number": 3, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Performance issues: unbounded lists, inefficient algorithms, over-engineered patterns", "files_checked": files_to_review, "relevant_files": files_to_review, "relevant_context": [ "PaymentProcessor.get_payment_history", "PaymentProcessor._generate_complex_hash", ], "issues_found": [ {"severity": "high", "description": "O(n²) algorithm in payment history"}, {"severity": "medium", "description": "Over-engineered hash generation"}, {"severity": "medium", "description": "Unbounded failed_payments list"}, ], "confidence": "high", "model": "flash", }, ) if not response3: self.logger.error("Failed to continue to step 3") return False response3_data = self._parse_review_response(response3) # Validate step 3 - should still use reference_only file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "reference_only": self.logger.error("Step 3 should use reference_only file context") return False self.logger.info(" ✅ Step 3: reference_only file context") # Step 4: Final comprehensive analysis self.logger.info(" 1.6.4: Step 4 - Final comprehensive analysis") response4, _ = self.call_mcp_tool( "codereview", { "step": "Code review complete - comprehensive analysis of all security, performance, and quality issues", "step_number": 4, "total_steps": 4, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete review: identified critical security vulnerabilities, performance bottlenecks, over-engineering patterns, and maintainability concerns across payment and configuration modules.", "files_checked": files_to_review, "relevant_files": files_to_review, "relevant_context": ["PaymentProcessor.process_payment", "ConfigurationManager.get_config"], "issues_found": [ {"severity": "critical", "description": "Multiple hardcoded secrets"}, {"severity": "high", "description": "Performance and security issues in payment processing"}, {"severity": "medium", "description": "Over-engineered architecture patterns"}, ], "confidence": "high", "model": "flash", }, ) if not response4: self.logger.error("Failed to complete to final step") return False response4_data = self._parse_review_response(response4) # Validate step 4 - should use fully_embedded for expert analysis file_context4 = response4_data.get("file_context", {}) if file_context4.get("type") != "fully_embedded": self.logger.error("Step 4 (final) should use fully_embedded file context") return False if "expert analysis" not in file_context4.get("context_optimization", "").lower(): self.logger.error("Final step should mention expert analysis in context optimization") return False # Verify expert analysis was triggered if response4_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False # Check that expert analysis has content expert_analysis = response4_data.get("expert_analysis", {}) if not expert_analysis: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Step 4: fully_embedded file context with expert analysis") # Validate the complete workflow progression progression_summary = { "step_1": "reference_only (new conversation, intermediate)", "step_2": "reference_only (continuation, intermediate)", "step_3": "reference_only (continuation, intermediate)", "step_4": "fully_embedded (continuation, final)", } self.logger.info(" 📋 File context progression:") for step, context_type in progression_summary.items(): self.logger.info(f" {step}: {context_type}") self.logger.info(" ✅ Multi-step file context optimization test completed successfully") return True except Exception as e: self.logger.error(f"Multi-step file context test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for codereview-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from codereview response specifically continuation_id = self._extract_review_continuation_id(response_text) return response_text, continuation_id def _extract_review_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from codereview response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for review continuation_id: {e}") return None def _parse_review_response(self, response_text: str) -> dict: """Parse codereview tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse review response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a codereview step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check code_review_status exists if "code_review_status" not in response_data: self.logger.error("Missing code_review_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False ================================================ FILE: simulator_tests/test_consensus_conversation.py ================================================ #!/usr/bin/env python3 """ Consensus Conversation Continuation Test Tests that the consensus tool properly handles conversation continuation and builds conversation context correctly when using continuation_id. """ import json from .conversation_base_test import ConversationBaseTest class TestConsensusConversation(ConversationBaseTest): """Test consensus tool conversation continuation functionality""" def call_mcp_tool(self, tool_name: str, params: dict) -> tuple: """Call an MCP tool in-process""" response_text, continuation_id = self.call_mcp_tool_direct(tool_name, params) return response_text, continuation_id @property def test_name(self) -> str: return "consensus_conversation" @property def test_description(self) -> str: return "Test consensus tool conversation building and continuation" def get_server_logs(self): """Get server logs from local log file""" try: log_file_path = "logs/mcp_server.log" with open(log_file_path) as f: lines = f.readlines() # Return last 100 lines return [line.strip() for line in lines[-100:]] except Exception as e: self.logger.warning(f"Exception getting server logs: {e}") return [] def run_test(self) -> bool: """Test consensus conversation continuation""" try: self.logger.info("Testing consensus tool conversation continuation") # Initialize for in-process tool calling self.setUp() # Setup test files for context self.setup_test_files() # Phase 1: Start conversation with chat tool (which properly creates continuation_id) self.logger.info("Phase 1: Starting conversation with chat tool") initial_response, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Please use low thinking mode. I'm working on a web application and need advice on authentication. Can you look at this code?", "absolute_file_paths": [self.test_files["python"]], "model": "flash", }, ) # Validate initial response if not initial_response: self.logger.error("Failed to get initial chat response") return False if not continuation_id: self.logger.error("Failed to get continuation_id from initial chat") return False self.logger.info(f"Initial chat response preview: {initial_response[:200]}...") self.logger.info(f"Got continuation_id: {continuation_id}") # Phase 2: Use consensus with continuation_id to test conversation building self.logger.info("Phase 2: Using consensus with continuation_id to test conversation building") consensus_response, _ = self.call_mcp_tool( "consensus", { "step": "Based on our previous discussion about authentication, I need expert consensus: Should we implement OAuth2 or stick with simple session-based auth?", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Initial analysis needed on OAuth2 vs session-based authentication approaches for our web application", "models": [ { "model": "flash", "stance": "for", "stance_prompt": "Focus on OAuth2 benefits: security, scalability, and industry standards.", }, { "model": "flash", "stance": "against", "stance_prompt": "Focus on OAuth2 complexity: implementation challenges and simpler alternatives.", }, ], "continuation_id": continuation_id, "model": "flash", }, ) # Validate consensus response if not consensus_response: self.logger.error("Failed to get consensus response with continuation_id") return False self.logger.info(f"Consensus response preview: {consensus_response[:300]}...") # Log the full response for debugging if it's not JSON if not consensus_response.startswith("{"): self.logger.error(f"Consensus response is not JSON. Full response: {consensus_response}") return False # Parse consensus response try: consensus_data = json.loads(consensus_response) except json.JSONDecodeError: self.logger.error(f"Failed to parse consensus response as JSON. Full response: {consensus_response}") return False # Check for step 1 status (Claude analysis + first model consultation) expected_status = "analysis_and_first_model_consulted" if consensus_data.get("status") != expected_status: self.logger.error( f"Consensus step 1 failed with status: {consensus_data.get('status')}, expected: {expected_status}" ) if "error" in consensus_data: self.logger.error(f"Error: {consensus_data['error']}") return False # Phase 3: Check server logs for conversation building self.logger.info("Phase 3: Checking server logs for conversation building") # Check for conversation-related log entries logs = self.get_server_logs() if not logs: self.logger.warning("Could not retrieve server logs for verification") else: # Look for conversation building indicators conversation_logs = [ line for line in logs if any( keyword in line for keyword in [ "CONVERSATION HISTORY", "continuation_id", "build_conversation_history", "ThreadContext", f"thread:{continuation_id}", ] ) ] if conversation_logs: self.logger.info(f"Found {len(conversation_logs)} conversation-related log entries") # Show a few examples (truncated) for i, log in enumerate(conversation_logs[:3]): self.logger.info(f" Conversation log {i+1}: {log[:100]}...") else: self.logger.warning( "No conversation-related logs found (may indicate conversation not properly built)" ) # Check for any ERROR entries related to consensus error_logs = [ line for line in logs if "ERROR" in line and any(keyword in line for keyword in ["consensus", "conversation", continuation_id]) ] if error_logs: self.logger.error(f"Found {len(error_logs)} error logs related to consensus conversation:") for error in error_logs: self.logger.error(f" ERROR: {error}") return False # Phase 4: Verify response structure self.logger.info("Phase 4: Verifying consensus response structure") # Check that we have model response from step 1 model_response = consensus_data.get("model_response") if not model_response: self.logger.error("Consensus step 1 response missing model_response") return False # Check that model response has expected structure if not model_response.get("model") or not model_response.get("verdict"): self.logger.error("Model response missing required fields (model or verdict)") return False # Check step information if consensus_data.get("step_number") != 1: self.logger.error(f"Expected step_number 1, got: {consensus_data.get('step_number')}") return False if not consensus_data.get("next_step_required"): self.logger.error("Expected next_step_required=True for step 1") return False self.logger.info(f"Consensus step 1 consulted model: {model_response.get('model')}") self.logger.info(f"Model stance: {model_response.get('stance', 'neutral')}") self.logger.info(f"Response status: {model_response.get('status', 'unknown')}") # Phase 5: Cross-tool continuation test self.logger.info("Phase 5: Testing cross-tool continuation from consensus") # Try to continue the conversation with a different tool chat_response, _ = self.call_mcp_tool( "chat", { "prompt": "Based on our consensus discussion about authentication, can you summarize the key points?", "continuation_id": continuation_id, "model": "flash", }, ) if not chat_response: self.logger.warning("Cross-tool continuation from consensus failed") # Don't fail the test for this - it's a bonus check else: self.logger.info("✓ Cross-tool continuation from consensus working") self.logger.info(f"Chat continuation preview: {chat_response[:200]}...") self.logger.info("✓ Consensus conversation continuation test completed successfully") return True except Exception as e: self.logger.error(f"Consensus conversation test failed with exception: {str(e)}") import traceback self.logger.error(f"Traceback: {traceback.format_exc()}") return False finally: self.cleanup_test_files() ================================================ FILE: simulator_tests/test_consensus_three_models.py ================================================ """ Test consensus tool with three models demonstrating sequential processing """ import json from .base_test import BaseSimulatorTest class TestConsensusThreeModels(BaseSimulatorTest): """Test consensus tool functionality with three models (testing sequential processing)""" @property def test_name(self) -> str: return "consensus_three_models" @property def test_description(self) -> str: return "Test consensus tool with three models using flash:against, flash:for, local-llama:neutral" def run_test(self) -> bool: """Run three-model consensus test""" try: self.logger.info("Testing consensus tool with three models: flash:against, flash:for, local-llama:neutral") # Send request with three objects using new workflow parameters response, continuation_id = self.call_mcp_tool( "consensus", { "step": "Is a sync manager class a good idea for my CoolTodos app?", "step_number": 1, "total_steps": 3, # 3 models = 3 steps "next_step_required": True, "findings": "Initial analysis needed on sync manager class architecture decision for CoolTodos app", "models": [ { "model": "flash", "stance": "against", "stance_prompt": "You are a software architecture critic. Focus on the potential downsides of adding a sync manager class: complexity overhead, maintenance burden, potential for over-engineering, and whether simpler alternatives exist. Consider if this adds unnecessary abstraction layers.", }, { "model": "flash", "stance": "for", "stance_prompt": "You are a software architecture advocate. Focus on the benefits of a sync manager class: separation of concerns, testability, maintainability, and how it can improve the overall architecture. Consider scalability and code organization advantages.", }, { "model": "local-llama", "stance": "neutral", "stance_prompt": "You are a pragmatic software engineer. Provide a balanced analysis considering both the benefits and drawbacks. Focus on the specific context of a CoolTodos app and what factors would determine if this is the right choice.", }, ], "model": "flash", # Default model for Claude's execution }, ) # Validate response if not response: self.logger.error("Failed to get response from three-model consensus tool") return False self.logger.info(f"Three-model consensus response preview: {response[:500]}...") # Parse the JSON response try: consensus_data = json.loads(response) except json.JSONDecodeError: self.logger.error(f"Failed to parse three-model consensus response as JSON: {response}") return False # Validate consensus structure if "status" not in consensus_data: self.logger.error("Missing 'status' field in three-model consensus response") return False # Check for step 1 status (Claude analysis + first model consultation) expected_status = "analysis_and_first_model_consulted" if consensus_data["status"] != expected_status: self.logger.error( f"Three-model consensus step 1 failed with status: {consensus_data['status']}, expected: {expected_status}" ) # Log additional error details for debugging if "error" in consensus_data: self.logger.error(f"Error message: {consensus_data['error']}") if "models_errored" in consensus_data: self.logger.error(f"Models that errored: {consensus_data['models_errored']}") if "models_skipped" in consensus_data: self.logger.error(f"Models skipped: {consensus_data['models_skipped']}") if "next_steps" in consensus_data: self.logger.error(f"Suggested next steps: {consensus_data['next_steps']}") return False # Check that we have model response from step 1 model_response = consensus_data.get("model_response") if not model_response: self.logger.error("Three-model consensus step 1 response missing model_response") return False # Check that model response has expected structure if not model_response.get("model") or not model_response.get("verdict"): self.logger.error("Model response missing required fields (model or verdict)") return False # Check step information if consensus_data.get("step_number") != 1: self.logger.error(f"Expected step_number 1, got: {consensus_data.get('step_number')}") return False if not consensus_data.get("next_step_required"): self.logger.error("Expected next_step_required=True for step 1") return False self.logger.info(f"Consensus step 1 consulted model: {model_response.get('model')}") self.logger.info(f"Model stance: {model_response.get('stance', 'neutral')}") self.logger.info(f"Response status: {model_response.get('status', 'unknown')}") # Check metadata contains model name metadata = consensus_data.get("metadata", {}) if not metadata.get("model_name"): self.logger.error("Missing model_name in metadata") return False self.logger.info(f"Model name in metadata: {metadata.get('model_name')}") # Verify we have analysis from Claude agent_analysis = consensus_data.get("agent_analysis") if not agent_analysis: self.logger.error("Missing Claude's analysis in step 1") return False analysis_text = agent_analysis.get("initial_analysis", "") self.logger.info(f"Claude analysis length: {len(analysis_text)} characters") self.logger.info("✓ Three-model consensus tool test completed successfully") self.logger.info(f"✓ Step 1 completed with model: {model_response.get('model')}") self.logger.info(f"✓ Analysis provided: {len(analysis_text)} characters") self.logger.info(f"✓ Model metadata properly included: {metadata.get('model_name')}") self.logger.info("✓ Ready for step 2 continuation") return True except Exception as e: self.logger.error(f"Three-model consensus test failed with exception: {str(e)}") return False ================================================ FILE: simulator_tests/test_consensus_workflow_accurate.py ================================================ """ Accurate Consensus Workflow Test This test validates the complete consensus workflow step-by-step to ensure: 1. Step 1: Claude provides its own analysis 2. Step 2: Tool consults first model and returns response to Claude 3. Step 3: Tool consults second model and returns response to Claude 4. Step 4: Claude synthesizes all perspectives This replaces the old faulty test that used non-workflow parameters. """ import json from .conversation_base_test import ConversationBaseTest class TestConsensusWorkflowAccurate(ConversationBaseTest): """Test complete consensus workflow with accurate step-by-step behavior""" @property def test_name(self) -> str: return "consensus_workflow_accurate" @property def test_description(self) -> str: return "Test NEW efficient consensus workflow: 2 models = 2 steps (Claude+model1, model2+synthesis)" def run_test(self) -> bool: """Run complete consensus workflow test""" # Set up the test environment self.setUp() try: self.logger.info("Testing complete consensus workflow step-by-step") self.logger.info("Expected NEW flow: Step1(Claude+Model1) -> Step2(Model2+Synthesis)") # ============================================================================ # STEP 1: Claude analysis + first model consultation # ============================================================================ self.logger.info("=== STEP 1: Claude analysis + flash:for consultation ===") step1_response, continuation_id = self.call_mcp_tool_direct( "consensus", { "step": "Should we add a new AI-powered search feature to our application? Please analyze the technical feasibility, user value, and implementation complexity.", "step_number": 1, "total_steps": 2, # 2 models (each step includes consultation + analysis) "next_step_required": True, "findings": "Initial assessment of AI search feature proposal considering user needs, technical constraints, and business value.", "models": [ { "model": "flash", "stance": "for", "stance_prompt": "Focus on innovation benefits and competitive advantages.", }, { "model": "flash", "stance": "against", "stance_prompt": "Focus on implementation complexity and resource requirements.", }, ], "model": "flash", # Claude's execution model }, ) if not step1_response: self.logger.error("Step 1 failed - no response") return False step1_data = json.loads(step1_response) self.logger.info(f"Step 1 status: {step1_data.get('status')}") # Validate step 1 response (should include Claude's analysis + first model consultation) if step1_data.get("status") != "analysis_and_first_model_consulted": self.logger.error( f"Expected status 'analysis_and_first_model_consulted', got: {step1_data.get('status')}" ) return False if step1_data.get("step_number") != 1: self.logger.error(f"Expected step_number 1, got: {step1_data.get('step_number')}") return False if not step1_data.get("next_step_required"): self.logger.error("Expected next_step_required=True for step 1") return False # Verify Claude's analysis is included if "agent_analysis" not in step1_data: self.logger.error("Expected agent_analysis in step 1 response") return False # Verify first model response is included if "model_response" not in step1_data: self.logger.error("Expected model_response in step 1 response") return False model1_response = step1_data["model_response"] if model1_response.get("model") != "flash" or model1_response.get("stance") != "for": self.logger.error( f"Expected flash:for model response in step 1, got: {model1_response.get('model')}:{model1_response.get('stance')}" ) return False self.logger.info("✓ Step 1 completed - Claude analysis + first model (flash:for) consulted") # ============================================================================ # STEP 2: Final step - second model consultation + synthesis # ============================================================================ self.logger.info("=== STEP 2: Final step - second model (flash:against) + synthesis ===") step2_response, _ = self.call_mcp_tool_direct( "consensus", { "step": "I need to review the second model's perspective and provide final synthesis.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step "findings": "Analyzed first model's 'for' perspective. Now ready for second model's 'against' stance and final synthesis.", "continuation_id": continuation_id, "model": "flash", }, ) if not step2_response: self.logger.error("Step 2 failed - no response") return False self.logger.info(f"Step 2 raw response: {step2_response[:500]}...") step2_data = json.loads(step2_response) self.logger.info(f"Step 2 status: {step2_data.get('status')}") # Validate step 2 - should show consensus completion if step2_data.get("status") != "consensus_workflow_complete": self.logger.error(f"Expected status 'consensus_workflow_complete', got: {step2_data.get('status')}") return False if step2_data.get("model_consulted") != "flash": self.logger.error(f"Expected model_consulted 'flash', got: {step2_data.get('model_consulted')}") return False if step2_data.get("model_stance") != "against": self.logger.error(f"Expected model_stance 'against', got: {step2_data.get('model_stance')}") return False # Verify model response is included if "model_response" not in step2_data: self.logger.error("Expected model_response in step 2") return False model2_response = step2_data["model_response"] if model2_response.get("model") != "flash": self.logger.error(f"Expected model_response.model 'flash', got: {model2_response.get('model')}") return False # Verify consensus completion data if not step2_data.get("consensus_complete"): self.logger.error("Expected consensus_complete=True in final step") return False if "complete_consensus" not in step2_data: self.logger.error("Expected complete_consensus data in final step") return False self.logger.info("✓ Step 2 completed - Second model (flash:against) consulted and consensus complete") self.logger.info(f"Model 2 verdict preview: {model2_response.get('verdict', 'No verdict')[:100]}...") # Validate final consensus completion data complete_consensus = step2_data["complete_consensus"] if complete_consensus.get("total_responses") != 2: self.logger.error(f"Expected 2 model responses, got: {complete_consensus.get('total_responses')}") return False models_consulted = complete_consensus.get("models_consulted", []) expected_models = ["flash:for", "flash:against"] if models_consulted != expected_models: self.logger.error(f"Expected models {expected_models}, got: {models_consulted}") return False # ============================================================================ # VALIDATION: Check accumulated responses are available # ============================================================================ self.logger.info("=== VALIDATION: Checking accumulated responses ===") if "accumulated_responses" not in step2_data: self.logger.error("Expected accumulated_responses in final step") return False accumulated = step2_data["accumulated_responses"] if len(accumulated) != 2: self.logger.error(f"Expected 2 accumulated responses, got: {len(accumulated)}") return False # Verify first response (flash:for) response1 = accumulated[0] if response1.get("model") != "flash" or response1.get("stance") != "for": self.logger.error(f"First response incorrect: {response1}") return False # Verify second response (flash:against) response2 = accumulated[1] if response2.get("model") != "flash" or response2.get("stance") != "against": self.logger.error(f"Second response incorrect: {response2}") return False self.logger.info("✓ All accumulated responses validated") # ============================================================================ # SUCCESS # ============================================================================ self.logger.info("🎉 CONSENSUS WORKFLOW TEST PASSED") self.logger.info("✓ Step 1: Claude analysis + first model (flash:for) consulted") self.logger.info("✓ Step 2: Second model (flash:against) consulted + synthesis completed") self.logger.info("✓ All model responses accumulated correctly") self.logger.info("✓ New efficient workflow: 2 models = 2 steps (not 4)") self.logger.info("✓ Workflow progression validated at each step") return True except Exception as e: self.logger.error(f"Consensus workflow test failed with exception: {str(e)}") import traceback self.logger.error(f"Traceback: {traceback.format_exc()}") return False ================================================ FILE: simulator_tests/test_content_validation.py ================================================ #!/usr/bin/env python3 """ Content Validation Test Tests that tools don't duplicate file content in their responses. This test is specifically designed to catch content duplication bugs. """ import os from .base_test import BaseSimulatorTest class ContentValidationTest(BaseSimulatorTest): """Test that tools don't duplicate file content in their responses""" @property def test_name(self) -> str: return "content_validation" @property def test_description(self) -> str: return "Content validation and duplicate detection" def run_test(self) -> bool: """Test that file processing system properly handles file deduplication""" try: self.logger.info("📄 Test: Content validation and file processing deduplication") # Setup test files first self.setup_test_files() # Create a test file for validation validation_content = '''""" Configuration file for content validation testing """ # Configuration constants MAX_CONTENT_TOKENS = 800_000 TEMPERATURE_ANALYTICAL = 0.2 UNIQUE_VALIDATION_MARKER = "CONTENT_VALIDATION_TEST_12345" # Database settings DATABASE_CONFIG = { "host": "localhost", "port": 5432, "name": "validation_test_db" } ''' validation_file = os.path.join(self.test_dir, "validation_config.py") with open(validation_file, "w") as f: f.write(validation_content) # Ensure absolute path for MCP server compatibility validation_file = os.path.abspath(validation_file) # Get timestamp for log filtering import datetime start_time = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") # Test 1: Initial tool call with validation file self.logger.info(" 1: Testing initial tool call with file") # Call chat tool with the validation file response1, thread_id = self.call_mcp_tool( "chat", { "prompt": "Analyze this configuration file briefly", "absolute_file_paths": [validation_file], "model": "flash", }, ) if not response1: self.logger.error(" ❌ Initial tool call failed") return False self.logger.info(" ✅ Initial tool call completed") # Test 2: Continuation with same file (should be deduplicated) self.logger.info(" 2: Testing continuation with same file") if thread_id: response2, _ = self.call_mcp_tool( "chat", { "prompt": "Continue analyzing this configuration file", "absolute_file_paths": [validation_file], # Same file should be deduplicated "continuation_id": thread_id, "model": "flash", }, ) if response2: self.logger.info(" ✅ Continuation with same file completed") else: self.logger.warning(" ⚠️ Continuation failed") # Test 3: Different tool with same file (new conversation) self.logger.info(" 3: Testing different tool with same file") response3, _ = self.call_mcp_tool( "codereview", { "step": "Review this configuration file for quality and potential issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting code review of configuration file", "relevant_files": [validation_file], "model": "flash", }, ) if response3: self.logger.info(" ✅ Different tool with same file completed") else: self.logger.warning(" ⚠️ Different tool failed") # Validate file processing behavior from server logs self.logger.info(" 4: Validating file processing logs") logs = self.get_server_logs_since(start_time) # Check for proper file embedding logs embedding_logs = [ line for line in logs.split("\n") if "[FILE_PROCESSING]" in line or "embedding" in line.lower() or "[FILES]" in line ] # Check for deduplication evidence deduplication_logs = [ line for line in logs.split("\n") if ("skipping" in line.lower() and "already in conversation" in line.lower()) or "No new files to embed" in line ] # Check for file processing patterns new_file_logs = [ line for line in logs.split("\n") if "will embed new files" in line or "New conversation" in line or "[FILE_PROCESSING]" in line ] # Validation criteria validation_file_mentioned = any("validation_config.py" in line for line in logs.split("\n")) embedding_found = len(embedding_logs) > 0 (len(deduplication_logs) > 0 or len(new_file_logs) >= 2) # Should see new conversation patterns self.logger.info(f" Embedding logs found: {len(embedding_logs)}") self.logger.info(f" Deduplication evidence: {len(deduplication_logs)}") self.logger.info(f" New conversation patterns: {len(new_file_logs)}") self.logger.info(f" Validation file mentioned: {validation_file_mentioned}") # Log sample evidence for debugging if self.verbose and embedding_logs: self.logger.debug(" 📋 Sample embedding logs:") for log in embedding_logs[:5]: self.logger.debug(f" {log}") # Success criteria success_criteria = [ ("Embedding logs found", embedding_found), ("File processing evidence", validation_file_mentioned), ("Multiple tool calls", len(new_file_logs) >= 2), ] passed_criteria = sum(1 for _, passed in success_criteria if passed) self.logger.info(f" Success criteria met: {passed_criteria}/{len(success_criteria)}") # Cleanup os.remove(validation_file) if passed_criteria >= 2: # At least 2 out of 3 criteria self.logger.info(" ✅ File processing validation passed") return True else: self.logger.error(" ❌ File processing validation failed") return False except Exception as e: self.logger.error(f"Content validation test failed: {e}") return False finally: self.cleanup_test_files() ================================================ FILE: simulator_tests/test_conversation_chain_validation.py ================================================ #!/usr/bin/env python3 """ Conversation Chain and Threading Validation Test This test validates that: 1. Multiple tool invocations create proper parent->parent->parent chains 2. New conversations can be started independently 3. Original conversation chains can be resumed from any point 4. History traversal works correctly for all scenarios 5. Thread relationships are properly maintained in Redis Test Flow: Chain A: chat -> analyze -> debug (3 linked threads) Chain B: chat -> analyze (2 linked threads, independent) Chain A Branch: debug (continue from original chat, creating branch) This validates the conversation threading system's ability to: - Build linear chains - Create independent conversation threads - Branch from earlier points in existing chains - Properly traverse parent relationships for history reconstruction """ from .conversation_base_test import ConversationBaseTest class ConversationChainValidationTest(ConversationBaseTest): """Test conversation chain and threading functionality""" @property def test_name(self) -> str: return "conversation_chain_validation" @property def test_description(self) -> str: return "Conversation chain and threading validation" def run_test(self) -> bool: """Test conversation chain and threading functionality""" # Set up the test environment self.setUp() try: self.logger.info("Test: Conversation chain and threading validation") # Create test file for consistent context test_file_content = """def example_function(): '''Simple test function for conversation continuity testing''' return "Hello from conversation chain test" def buggy_function(x, y): '''Function with a bug - incorrect operator''' return x - y # BUG: Should be x + y for addition class TestClass: def method(self): return "Method in test class" """ test_file_path = self.create_additional_test_file("chain_test.py", test_file_content) # Track all continuation IDs and their relationships conversation_chains = {} # === CHAIN A: Build linear conversation chain === self.logger.info(" Chain A: Building linear conversation chain") # Step A1: Start with chat tool (creates thread_id_1) self.logger.info(" Step A1: Chat tool - start new conversation") response_a1, continuation_id_a1 = self.call_mcp_tool( "chat", { "prompt": "Analyze this test file and explain what it does.", "absolute_file_paths": [test_file_path], "model": "flash", "temperature": 0.7, }, ) if not response_a1 or not continuation_id_a1: self.logger.error(" ❌ Step A1 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step A1 completed - thread_id: {continuation_id_a1[:8]}...") conversation_chains["A1"] = continuation_id_a1 # Step A2: Continue with analyze tool (creates thread_id_2 with parent=thread_id_1) self.logger.info(" Step A2: Analyze tool - continue Chain A") response_a2, continuation_id_a2 = self.call_mcp_tool( "analyze", { "step": "Now analyze the code quality and suggest improvements.", "step_number": 1, "total_steps": 2, "next_step_required": False, "findings": "Continuing analysis from previous chat conversation to analyze code quality.", "relevant_files": [test_file_path], "continuation_id": continuation_id_a1, "model": "flash", }, ) if not response_a2 or not continuation_id_a2: self.logger.error(" ❌ Step A2 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step A2 completed - thread_id: {continuation_id_a2[:8]}...") conversation_chains["A2"] = continuation_id_a2 # Step A3: Continue with chat tool (creates thread_id_3 with parent=thread_id_2) self.logger.info(" Step A3: Chat tool - continue Chain A") response_a3, continuation_id_a3 = self.call_mcp_tool( "chat", { "prompt": "Thank you for the analysis. Can you summarize the key points?", "continuation_id": continuation_id_a2, "model": "flash", "temperature": 0.7, }, ) if not response_a3 or not continuation_id_a3: self.logger.error(" ❌ Step A3 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step A3 completed - thread_id: {continuation_id_a3[:8]}...") conversation_chains["A3"] = continuation_id_a3 # === CHAIN B: Start independent conversation === self.logger.info(" Chain B: Starting independent conversation") # Step B1: Start new chat conversation (creates thread_id_4, no parent) self.logger.info(" Step B1: Chat tool - start NEW independent conversation") response_b1, continuation_id_b1 = self.call_mcp_tool( "chat", { "prompt": "This is a completely new conversation. Please greet me.", "model": "flash", "temperature": 0.7, }, ) if not response_b1 or not continuation_id_b1: self.logger.error(" ❌ Step B1 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step B1 completed - thread_id: {continuation_id_b1[:8]}...") conversation_chains["B1"] = continuation_id_b1 # Step B2: Continue the new conversation (creates thread_id_5 with parent=thread_id_4) self.logger.info(" Step B2: Analyze tool - continue Chain B") response_b2, continuation_id_b2 = self.call_mcp_tool( "analyze", { "step": "Analyze the previous greeting and suggest improvements.", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Analyzing the greeting from previous conversation and suggesting improvements.", "relevant_files": [test_file_path], "continuation_id": continuation_id_b1, "model": "flash", }, ) if not response_b2 or not continuation_id_b2: self.logger.error(" ❌ Step B2 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step B2 completed - thread_id: {continuation_id_b2[:8]}...") conversation_chains["B2"] = continuation_id_b2 # === CHAIN A BRANCH: Go back to original conversation === self.logger.info(" Chain A Branch: Resume original conversation from A1") # Step A1-Branch: Use original continuation_id_a1 to branch (creates thread_id_6 with parent=thread_id_1) self.logger.info(" Step A1-Branch: Chat tool - branch from original Chain A") response_a1_branch, continuation_id_a1_branch = self.call_mcp_tool( "chat", { "prompt": "Going back to our original discussion, I have another question about the code structure.", "continuation_id": continuation_id_a1, # Go back to original! "model": "flash", "temperature": 0.7, }, ) if not response_a1_branch or not continuation_id_a1_branch: self.logger.error(" ❌ Step A1-Branch failed - no response or continuation ID") return False self.logger.info(f" ✅ Step A1-Branch completed - thread_id: {continuation_id_a1_branch[:8]}...") conversation_chains["A1_Branch"] = continuation_id_a1_branch # === ANALYSIS: Validate thread relationships and history traversal === self.logger.info(" Analyzing conversation chain structure...") # Get logs and extract thread relationships logs = self.get_recent_server_logs() thread_creation_logs = self.extract_thread_creation_logs(logs) history_traversal_logs = self.extract_history_traversal_logs(logs) self.logger.info(f" Found {len(thread_creation_logs)} thread creation logs") self.logger.info(f" Found {len(history_traversal_logs)} history traversal logs") # Debug: Show what we found if self.verbose: self.logger.debug(" Thread creation logs found:") for log in thread_creation_logs: self.logger.debug( f" {log['thread_id'][:8]}... parent: {log['parent_id'][:8] if log['parent_id'] else 'None'}..." ) self.logger.debug(" History traversal logs found:") for log in history_traversal_logs: self.logger.debug(f" {log['thread_id'][:8]}... chain length: {log['chain_length']}") # Build expected thread relationships expected_relationships = [] # Note: A1 and B1 won't appear in thread creation logs because they're new conversations (no parent) # Only continuation threads (A2, A3, B2, A1-Branch) will appear in creation logs # Find logs for each continuation thread a2_log = next((log for log in thread_creation_logs if log["thread_id"] == continuation_id_a2), None) a3_log = next((log for log in thread_creation_logs if log["thread_id"] == continuation_id_a3), None) b2_log = next((log for log in thread_creation_logs if log["thread_id"] == continuation_id_b2), None) a1_branch_log = next( (log for log in thread_creation_logs if log["thread_id"] == continuation_id_a1_branch), None ) # A2 should have A1 as parent if a2_log: expected_relationships.append(("A2 has A1 as parent", a2_log["parent_id"] == continuation_id_a1)) # A3 should have A2 as parent if a3_log: expected_relationships.append(("A3 has A2 as parent", a3_log["parent_id"] == continuation_id_a2)) # B2 should have B1 as parent (independent chain) if b2_log: expected_relationships.append(("B2 has B1 as parent", b2_log["parent_id"] == continuation_id_b1)) # A1-Branch should have A1 as parent (branching) if a1_branch_log: expected_relationships.append( ("A1-Branch has A1 as parent", a1_branch_log["parent_id"] == continuation_id_a1) ) # Validate history traversal traversal_validations = [] # History traversal logs are only generated when conversation history is built from scratch # (not when history is already embedded in the prompt by server.py) # So we should expect at least 1 traversal log, but not necessarily for every continuation if len(history_traversal_logs) > 0: # Validate that any traversal logs we find have reasonable chain lengths for log in history_traversal_logs: thread_id = log["thread_id"] chain_length = log["chain_length"] # Chain length should be at least 2 for any continuation thread # (original thread + continuation thread) is_valid_length = chain_length >= 2 # Try to identify which thread this is for better validation thread_description = f"Thread {thread_id[:8]}" if thread_id == continuation_id_a1: thread_description = "A1 (original thread)" is_valid_length = chain_length == 1 elif thread_id == continuation_id_a2: thread_description = "A2 (2-thread chain)" is_valid_length = chain_length == 2 elif thread_id == continuation_id_a3: thread_description = "A3 (3-thread chain)" is_valid_length = chain_length == 3 elif thread_id == continuation_id_b1: thread_description = "B1 (original thread)" is_valid_length = chain_length == 1 elif thread_id == continuation_id_b2: thread_description = "B2 (2-thread chain)" is_valid_length = chain_length == 2 elif thread_id == continuation_id_a1_branch: thread_description = "A1-Branch (2-thread chain)" is_valid_length = chain_length == 2 traversal_validations.append((f"{thread_description} has valid chain length", is_valid_length)) # Also validate we found at least one traversal (shows the system is working) traversal_validations.append( ("At least one history traversal occurred", len(history_traversal_logs) >= 1) ) # === VALIDATION RESULTS === self.logger.info(" Thread Relationship Validation:") relationship_passed = 0 for desc, passed in expected_relationships: status = "✅" if passed else "❌" self.logger.info(f" {status} {desc}") if passed: relationship_passed += 1 self.logger.info(" History Traversal Validation:") traversal_passed = 0 for desc, passed in traversal_validations: status = "✅" if passed else "❌" self.logger.info(f" {status} {desc}") if passed: traversal_passed += 1 # === SUCCESS CRITERIA === total_relationship_checks = len(expected_relationships) total_traversal_checks = len(traversal_validations) self.logger.info(" Validation Summary:") self.logger.info(f" Thread relationships: {relationship_passed}/{total_relationship_checks}") self.logger.info(f" History traversal: {traversal_passed}/{total_traversal_checks}") # Success requires at least 80% of validations to pass relationship_success = relationship_passed >= (total_relationship_checks * 0.8) # If no traversal checks were possible, it means no traversal logs were found # This could indicate an issue since we expect at least some history building if total_traversal_checks == 0: self.logger.warning( " No history traversal logs found - this may indicate conversation history is always pre-embedded" ) # Still consider it successful since the thread relationships are what matter most traversal_success = True else: # For traversal success, we need at least 50% to pass since chain lengths can vary # The important thing is that traversal is happening and relationships are correct traversal_success = traversal_passed >= (total_traversal_checks * 0.5) overall_success = relationship_success and traversal_success self.logger.info(" Conversation Chain Structure:") self.logger.info( f" Chain A: {continuation_id_a1[:8]} → {continuation_id_a2[:8]} → {continuation_id_a3[:8]}" ) self.logger.info(f" Chain B: {continuation_id_b1[:8]} → {continuation_id_b2[:8]}") self.logger.info(f" Branch: {continuation_id_a1[:8]} → {continuation_id_a1_branch[:8]}") if overall_success: self.logger.info(" ✅ Conversation chain validation test PASSED") return True else: self.logger.error(" ❌ Conversation chain validation test FAILED") return False except Exception as e: self.logger.error(f"Conversation chain validation test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple: """Call an MCP tool in-process""" # Use in-process implementation to maintain conversation memory response_text, continuation_id = self.call_mcp_tool_direct(tool_name, params) return response_text, continuation_id def main(): """Run the conversation chain validation test""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = ConversationChainValidationTest(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_cross_tool_comprehensive.py ================================================ #!/usr/bin/env python3 """ Comprehensive Cross-Tool Test Tests file deduplication, conversation continuation, and file handling across all available MCP tools using realistic workflows with low thinking mode. Validates: 1. Cross-tool conversation continuation 2. File deduplication across different tools 3. Mixed file scenarios (old + new files) 4. Conversation history preservation 5. Proper tool chaining with context """ from .conversation_base_test import ConversationBaseTest class CrossToolComprehensiveTest(ConversationBaseTest): """Comprehensive test across all MCP tools""" def call_mcp_tool(self, tool_name: str, params: dict) -> tuple: """Call an MCP tool in-process""" # Use the new method for workflow tools workflow_tools = ["analyze", "debug", "codereview", "precommit", "refactor", "thinkdeep"] if tool_name in workflow_tools: response_text, continuation_id = super().call_mcp_tool(tool_name, params) else: response_text, continuation_id = self.call_mcp_tool_direct(tool_name, params) return response_text, continuation_id @property def test_name(self) -> str: return "cross_tool_comprehensive" @property def test_description(self) -> str: return "Comprehensive cross-tool file deduplication and continuation" def run_test(self) -> bool: """Comprehensive cross-tool test with all MCP tools""" try: self.logger.info("📄 Test: Comprehensive cross-tool file deduplication and continuation") # Initialize for in-process tool calling self.setUp() # Setup test files self.setup_test_files() # Create short test files for quick testing python_code = """def login(user, pwd): # Security issue: plain text password if user == "admin" and pwd == "123": return True return False def hash_pwd(pwd): # Weak hashing return str(hash(pwd)) """ config_file = """{ "db_password": "weak123", "debug": true, "secret_key": "test" }""" auth_file = self.create_additional_test_file("auth.py", python_code) config_file_path = self.create_additional_test_file("config.json", config_file) # Get timestamp for log filtering import datetime start_time = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") # Tool chain: chat → analyze → debug → codereview → precommit # Each step builds on the previous with cross-tool continuation current_continuation_id = None responses = [] # Step 1: Start with chat tool to understand the codebase self.logger.info(" Step 1: chat tool - Initial codebase exploration") chat_params = { "prompt": "List security issues in auth.py", "absolute_file_paths": [auth_file], "thinking_mode": "low", "model": "flash", } response1, continuation_id1 = self.call_mcp_tool("chat", chat_params) if not response1 or not continuation_id1: self.logger.error(" ❌ Step 1: chat tool failed") return False self.logger.info(f" ✅ Step 1: chat completed with continuation_id: {continuation_id1[:8]}...") responses.append(("chat", response1, continuation_id1)) current_continuation_id = continuation_id1 # Step 2: Use analyze tool to do deeper analysis (fresh conversation) self.logger.info(" Step 2: analyze tool - Deep code analysis (fresh)") analyze_params = { "step": "Starting comprehensive code analysis to find security vulnerabilities in the authentication system", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Initial analysis will focus on security vulnerabilities in authentication code", "relevant_files": [auth_file], "thinking_mode": "low", "model": "flash", } response2, continuation_id2 = self.call_mcp_tool("analyze", analyze_params) if not response2: self.logger.error(" ❌ Step 2: analyze tool failed") return False self.logger.info( f" ✅ Step 2: analyze completed with continuation_id: {continuation_id2[:8] if continuation_id2 else 'None'}..." ) responses.append(("analyze", response2, continuation_id2)) # Step 3: Continue chat conversation with config file self.logger.info(" Step 3: chat continuation - Add config file context") chat_continue_params = { "continuation_id": current_continuation_id, "prompt": "Check config.json too", "absolute_file_paths": [auth_file, config_file_path], # Old + new file "thinking_mode": "low", "model": "flash", } response3, _ = self.call_mcp_tool("chat", chat_continue_params) if not response3: self.logger.error(" ❌ Step 3: chat continuation failed") return False self.logger.info(" ✅ Step 3: chat continuation completed") responses.append(("chat_continue", response3, current_continuation_id)) # Step 4: Use debug tool to identify specific issues self.logger.info(" Step 4: debug tool - Identify specific problems") debug_params = { "step": "Starting debug investigation to identify and fix authentication security issues", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Investigating authentication vulnerabilities found in previous analysis", "relevant_files": [auth_file, config_file_path], "thinking_mode": "low", "model": "flash", } response4, continuation_id4 = self.call_mcp_tool("debug", debug_params) if not response4: self.logger.error(" ❌ Step 4: debug tool failed") return False self.logger.info( f" ✅ Step 4: debug completed with continuation_id: {continuation_id4[:8] if continuation_id4 else 'None'}..." ) responses.append(("debug", response4, continuation_id4)) # Step 5: Cross-tool continuation - continue debug with chat context if continuation_id4: self.logger.info(" Step 5: debug continuation - Additional analysis") debug_continue_params = { "step": "Continuing debug investigation to fix password hashing implementation", "step_number": 2, "total_steps": 2, "next_step_required": False, "findings": "Building on previous analysis to fix weak password hashing", "continuation_id": continuation_id4, "relevant_files": [auth_file, config_file_path], "thinking_mode": "low", "model": "flash", } response5, _ = self.call_mcp_tool("debug", debug_continue_params) if response5: self.logger.info(" ✅ Step 5: debug continuation completed") responses.append(("debug_continue", response5, continuation_id4)) # Step 6: Use codereview for comprehensive review self.logger.info(" Step 6: codereview tool - Comprehensive code review") codereview_params = { "step": "Starting comprehensive security code review of authentication system", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Performing thorough security review of authentication code and configuration", "relevant_files": [auth_file, config_file_path], "thinking_mode": "low", "model": "flash", } response6, continuation_id6 = self.call_mcp_tool("codereview", codereview_params) if not response6: self.logger.error(" ❌ Step 6: codereview tool failed") return False self.logger.info( f" ✅ Step 6: codereview completed with continuation_id: {continuation_id6[:8] if continuation_id6 else 'None'}..." ) responses.append(("codereview", response6, continuation_id6)) # Step 7: Create improved version and use precommit self.logger.info(" Step 7: precommit tool - Pre-commit validation") # Create a short improved version improved_code = """import hashlib def secure_login(user, pwd): # Better: hashed password check hashed = hashlib.sha256(pwd.encode()).hexdigest() if user == "admin" and hashed == "expected_hash": return True return False """ improved_file = self.create_additional_test_file("auth_improved.py", improved_code) precommit_params = { "step": "Starting pre-commit validation of improved authentication code", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Validating improved authentication implementation before commit", "path": self.test_dir, "relevant_files": [auth_file, config_file_path, improved_file], "thinking_mode": "low", "model": "flash", } response7, continuation_id7 = self.call_mcp_tool("precommit", precommit_params) if not response7: self.logger.error(" ❌ Step 7: precommit tool failed") return False self.logger.info( f" ✅ Step 7: precommit completed with continuation_id: {continuation_id7[:8] if continuation_id7 else 'None'}..." ) responses.append(("precommit", response7, continuation_id7)) # Validate comprehensive results self.logger.info(" 📋 Validating comprehensive cross-tool results...") logs = self.get_server_logs_since(start_time) # Validation criteria tools_used = [r[0] for r in responses] continuation_ids_created = [r[2] for r in responses if r[2]] # Check for various log patterns conversation_logs = [ line for line in logs.split("\n") if "conversation" in line.lower() or "history" in line.lower() ] embedding_logs = [ line for line in logs.split("\n") if "📁" in line or "embedding" in line.lower() or "file" in line.lower() ] continuation_logs = [ line for line in logs.split("\n") if "continuation" in line.lower() or "resuming" in line.lower() ] cross_tool_logs = [ line for line in logs.split("\n") if any(tool in line.lower() for tool in ["chat", "analyze", "debug", "codereview", "precommit"]) ] # File mentions auth_file_mentioned = any("auth.py" in line for line in logs.split("\n")) config_file_mentioned = any("config.json" in line for line in logs.split("\n")) improved_file_mentioned = any("auth_improved.py" in line for line in logs.split("\n")) # Print comprehensive diagnostics self.logger.info(f" Tools used: {len(tools_used)} ({', '.join(tools_used)})") self.logger.info(f" Continuation IDs created: {len(continuation_ids_created)}") self.logger.info(f" Conversation logs found: {len(conversation_logs)}") self.logger.info(f" File embedding logs found: {len(embedding_logs)}") self.logger.info(f" Continuation logs found: {len(continuation_logs)}") self.logger.info(f" Cross-tool activity logs: {len(cross_tool_logs)}") self.logger.info(f" Auth file mentioned: {auth_file_mentioned}") self.logger.info(f" Config file mentioned: {config_file_mentioned}") self.logger.info(f" Improved file mentioned: {improved_file_mentioned}") if self.verbose: self.logger.debug(" 📋 Sample tool activity logs:") for log in cross_tool_logs[:10]: # Show first 10 if log.strip(): self.logger.debug(f" {log.strip()}") self.logger.debug(" 📋 Sample continuation logs:") for log in continuation_logs[:5]: # Show first 5 if log.strip(): self.logger.debug(f" {log.strip()}") # Comprehensive success criteria success_criteria = [ len(tools_used) >= 5, # Used multiple tools len(continuation_ids_created) >= 3, # Created multiple continuation threads len(embedding_logs) > 10, # Significant file embedding activity len(continuation_logs) > 0, # Evidence of continuation auth_file_mentioned, # Original file processed config_file_mentioned, # Additional file processed improved_file_mentioned, # New file processed len(conversation_logs) > 5, # Conversation history activity ] passed_criteria = sum(success_criteria) total_criteria = len(success_criteria) self.logger.info(f" Success criteria met: {passed_criteria}/{total_criteria}") # Allow for slight variations in log output (7/8 is sufficient for comprehensive test) if passed_criteria >= total_criteria - 1: # Allow 1 missing criterion self.logger.info(" ✅ Comprehensive cross-tool test: PASSED") if passed_criteria < total_criteria: self.logger.info( f" ℹ️ Note: {total_criteria - passed_criteria} criterion not met (acceptable variation)" ) return True else: self.logger.warning(" ⚠️ Comprehensive cross-tool test: FAILED") self.logger.warning(" 💡 Check logs for detailed cross-tool activity") return False except Exception as e: self.logger.error(f"Comprehensive cross-tool test failed: {e}") return False finally: self.cleanup_test_files() ================================================ FILE: simulator_tests/test_cross_tool_continuation.py ================================================ #!/usr/bin/env python3 """ Cross-Tool Continuation Test Tests comprehensive cross-tool continuation scenarios to ensure conversation context is maintained when switching between different tools. """ from .conversation_base_test import ConversationBaseTest class CrossToolContinuationTest(ConversationBaseTest): """Test comprehensive cross-tool continuation scenarios""" @property def test_name(self) -> str: return "cross_tool_continuation" @property def test_description(self) -> str: return "Cross-tool conversation continuation scenarios" def run_test(self) -> bool: """Test comprehensive cross-tool continuation scenarios""" try: self.logger.info("🔧 Test: Cross-tool continuation scenarios") # Setup test environment for conversation testing self.setUp() success_count = 0 total_scenarios = 3 # Scenario 1: chat -> thinkdeep -> codereview if self._test_chat_thinkdeep_codereview(): success_count += 1 # Scenario 2: analyze -> debug -> thinkdeep if self._test_analyze_debug_thinkdeep(): success_count += 1 # Scenario 3: Multi-file cross-tool continuation if self._test_multi_file_continuation(): success_count += 1 self.logger.info( f" ✅ Cross-tool continuation scenarios completed: {success_count}/{total_scenarios} scenarios passed" ) # Consider successful if at least one scenario worked return success_count > 0 except Exception as e: self.logger.error(f"Cross-tool continuation test failed: {e}") return False finally: self.cleanup_test_files() def _test_chat_thinkdeep_codereview(self) -> bool: """Test chat -> thinkdeep -> codereview scenario""" try: self.logger.info(" 1: Testing chat -> thinkdeep -> codereview") # Start with chat chat_response, chat_id = self.call_mcp_tool( "chat", { "prompt": "Please use low thinking mode. Look at this Python code and tell me what you think about it", "absolute_file_paths": [self.test_files["python"]], "model": "flash", }, ) if not chat_response or not chat_id: self.logger.error("Failed to start chat conversation") return False # Continue with thinkdeep thinkdeep_response, _ = self.call_mcp_tool( "thinkdeep", { "step": "Think deeply about potential performance issues in this code. Please use low thinking mode.", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Building on previous chat analysis to examine performance issues", "relevant_files": [self.test_files["python"]], # Same file should be deduplicated "continuation_id": chat_id, "model": "flash", }, ) if not thinkdeep_response: self.logger.error("Failed chat -> thinkdeep continuation") return False # Continue with codereview codereview_response, _ = self.call_mcp_tool( "codereview", { "step": "Building on our previous analysis, provide a comprehensive code review", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Continuing from previous chat and thinkdeep analysis for comprehensive review", "relevant_files": [self.test_files["python"]], # Same file should be deduplicated "continuation_id": chat_id, "model": "flash", }, ) if not codereview_response: self.logger.error("Failed thinkdeep -> codereview continuation") return False self.logger.info(" ✅ chat -> thinkdeep -> codereview working") return True except Exception as e: self.logger.error(f"Chat -> thinkdeep -> codereview scenario failed: {e}") return False def _test_analyze_debug_thinkdeep(self) -> bool: """Test analyze -> debug -> thinkdeep scenario""" try: self.logger.info(" 2: Testing analyze -> debug -> thinkdeep") # Start with analyze analyze_response, analyze_id = self.call_mcp_tool( "analyze", { "step": "Analyze this code for quality and performance issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting analysis of Python code for quality and performance issues", "relevant_files": [self.test_files["python"]], "model": "flash", }, ) if not analyze_response or not analyze_id: self.logger.warning("Failed to start analyze conversation, skipping scenario 2") return False # Continue with debug debug_response, _ = self.call_mcp_tool( "debug", { "step": "Based on our analysis, help debug the performance issue in fibonacci", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Building on previous analysis to debug specific performance issue", "relevant_files": [self.test_files["python"]], # Same file should be deduplicated "continuation_id": analyze_id, "model": "flash", }, ) if not debug_response: self.logger.warning(" ⚠️ analyze -> debug continuation failed") return False # Continue with thinkdeep final_response, _ = self.call_mcp_tool( "thinkdeep", { "step": "Think deeply about the architectural implications of the issues we've found. Please use low thinking mode.", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Building on analysis and debug findings to explore architectural implications", "relevant_files": [self.test_files["python"]], # Same file should be deduplicated "continuation_id": analyze_id, "model": "flash", }, ) if not final_response: self.logger.warning(" ⚠️ debug -> thinkdeep continuation failed") return False self.logger.info(" ✅ analyze -> debug -> thinkdeep working") return True except Exception as e: self.logger.error(f"Analyze -> debug -> thinkdeep scenario failed: {e}") return False def _test_multi_file_continuation(self) -> bool: """Test multi-file cross-tool continuation""" try: self.logger.info(" 3: Testing multi-file cross-tool continuation") # Start with both files multi_response, multi_id = self.call_mcp_tool( "chat", { "prompt": "Please use low thinking mode. Analyze both the Python code and configuration file", "absolute_file_paths": [self.test_files["python"], self.test_files["config"]], "model": "flash", }, ) if not multi_response or not multi_id: self.logger.warning("Failed to start multi-file conversation, skipping scenario 3") return False # Switch to codereview with same files (should use conversation history) multi_review, _ = self.call_mcp_tool( "codereview", { "step": "Review both files in the context of our previous discussion", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Continuing multi-file analysis with code review perspective", "relevant_files": [self.test_files["python"], self.test_files["config"]], # Same files "continuation_id": multi_id, "model": "flash", }, ) if not multi_review: self.logger.warning(" ⚠️ Multi-file cross-tool continuation failed") return False self.logger.info(" ✅ Multi-file cross-tool continuation working") return True except Exception as e: self.logger.error(f"Multi-file continuation scenario failed: {e}") return False ================================================ FILE: simulator_tests/test_debug_certain_confidence.py ================================================ #!/usr/bin/env python3 """ Debug Tool Certain Confidence Simulator Test Tests the debug tool's 'certain' confidence feature in a realistic simulation: - Multi-step investigation leading to certain confidence - Validation that expert analysis is skipped for obvious bugs - Verification that certain confidence is always trusted - Ensures token optimization works correctly for minimal fixes """ import json from typing import Optional from tools.shared.exceptions import ToolExecutionError from .conversation_base_test import ConversationBaseTest class DebugCertainConfidenceTest(ConversationBaseTest): """Test debug tool's certain confidence optimization feature""" @property def test_name(self) -> str: return "debug_certain_confidence" @property def test_description(self) -> str: return "Debug tool certain confidence optimization validation" def run_test(self) -> bool: """Test debug tool certain confidence capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: Debug tool certain confidence validation") # Create test files with obvious bugs for certain scenarios self._create_obvious_bug_scenarios() # Test 1: Obvious import error with certain confidence if not self._test_obvious_import_error_certain(): return False # Test 2: Certain confidence is always trusted if not self._test_certain_always_trusted(): return False # Test 3: Regular high confidence still triggers expert analysis if not self._test_regular_high_confidence_expert_analysis(): return False # Test 4: Multi-step investigation ending in certain if not self._test_multi_step_investigation_certain(): return False self.logger.info(" ✅ All debug certain confidence tests passed") return True except Exception as e: self.logger.error(f"Debug certain confidence test failed: {e}") return False def _create_obvious_bug_scenarios(self): """Create test files with obvious bugs perfect for certain confidence""" # Scenario 1: Missing import statement (very obvious) missing_import_code = """#!/usr/bin/env python3 import os import sys # import hashlib # <-- Missing import! class UserAuth: def __init__(self, secret_key): self.secret_key = secret_key def hash_password(self, password): # This will fail with NameError: name 'hashlib' is not defined salt = os.urandom(32) return hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 100000) def verify_password(self, password, stored_hash): # This function also uses hashlib return hashlib.pbkdf2_hmac('sha256', password.encode(), stored_hash[:32], 100000) == stored_hash[32:] """ # Scenario 2: Typo in method name (obvious once spotted) typo_bug_code = """#!/usr/bin/env python3 class Calculator: def __init__(self): self.history = [] def add_numbers(self, a, b): result = a + b self.history.append(f"{a} + {b} = {result}") return result def calculate_total(self, numbers): total = 0 for num in numbers: # Typo: should be add_numbers, not add_number total = self.add_number(total, num) # NameError: no method 'add_number' return total """ # Scenario 3: Indentation error (Python syntax error) indentation_error_code = """#!/usr/bin/env python3 def process_data(data_list): results = [] for item in data_list: if item > 0: processed = item * 2 results.append(processed) # IndentationError: unindent does not match any outer indentation level return results def main(): data = [1, 2, 3, 4, 5] print(process_data(data)) """ # Create test files self.missing_import_file = self.create_additional_test_file("user_auth.py", missing_import_code) self.typo_bug_file = self.create_additional_test_file("calculator.py", typo_bug_code) self.indentation_file = self.create_additional_test_file("data_processor.py", indentation_error_code) self.logger.info(" ✅ Created obvious bug scenarios:") self.logger.info(f" - Missing import: {self.missing_import_file}") self.logger.info(f" - Method typo: {self.typo_bug_file}") self.logger.info(f" - Indentation error: {self.indentation_file}") # Create error logs for context import_error_log = """ERROR: User authentication failing during login Traceback (most recent call last): File "user_auth.py", line 12, in hash_password return hashlib.pbkdf2_hmac('sha256', password.encode(), salt, 100000) NameError: name 'hashlib' is not defined This happens every time a user tries to log in. The error occurs in the password hashing function. """ self.error_log_file = self.create_additional_test_file("error.log", import_error_log) self.logger.info(f" - Error log: {self.error_log_file}") def _test_obvious_import_error_certain(self) -> bool: """Test certain confidence with obvious missing import error""" try: self.logger.info(" 1.1: Testing obvious import error with certain confidence") # Step 1: Initial investigation self.logger.info(" 1.1.1: Step 1 - Initial problem description") response1, continuation_id = self.call_mcp_tool_direct( "debug", { "step": "Investigating NameError in user authentication - users cannot log in due to 'name hashlib is not defined' error.", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "NameError occurs in hash_password method when trying to use hashlib.pbkdf2_hmac. Error happens on every login attempt.", "files_checked": [self.error_log_file], "relevant_files": [self.error_log_file], "hypothesis": "Missing import statement for hashlib module", "confidence": "medium", }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial investigation response") return False response1_data = self._parse_debug_response(response1) if not self._validate_investigation_response(response1_data, 1, True, "pause_for_investigation"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Examine code and identify obvious fix - use certain confidence self.logger.info(" 1.1.2: Step 2 - Found exact issue and simple fix (certain)") response2, _ = self.call_mcp_tool_direct( "debug", { "step": "Found the exact issue and the minimal fix required", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step "findings": "Missing 'import hashlib' statement at the top of user_auth.py file. The error occurs because hashlib is used in hash_password() method on line 12 but never imported. Simple one-line fix: add 'import hashlib' after line 2.", "files_checked": [self.error_log_file, self.missing_import_file], "relevant_files": [self.missing_import_file], "relevant_context": ["UserAuth.hash_password", "UserAuth.verify_password"], "hypothesis": "Missing 'import hashlib' statement causes NameError when hash_password method executes", "confidence": "certain", # Use certain - should skip expert analysis "continuation_id": continuation_id, "model": "flash", # Specify model for consistency }, ) if not response2: self.logger.error("Failed to complete investigation with certain confidence") return False response2_data = self._parse_debug_response(response2) if not response2_data: return False # Validate certain response structure expected_status = "certain_confidence_proceed_with_fix" if response2_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response2_data.get('status')}'") return False if not response2_data.get("investigation_complete"): self.logger.error("Expected investigation_complete=true for certain confidence") return False if not response2_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for certain confidence") return False # Verify expert analysis is marked as skipped expert_analysis = response2_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_certain_confidence": self.logger.error("Expert analysis should be marked as skipped for certain confidence") return False # Check for proper investigation summary complete_investigation = response2_data.get("complete_investigation", {}) if complete_investigation.get("confidence_level") != "certain": self.logger.error("Expected confidence_level='certain' in complete investigation") return False if complete_investigation.get("steps_taken") != 2: self.logger.error("Expected steps_taken=2 in complete investigation") return False # Verify next steps guidance next_steps = response2_data.get("next_steps", "") if "CERTAIN confidence" not in next_steps: self.logger.error("Expected 'CERTAIN confidence' in next_steps guidance") return False if "minimal fix" not in next_steps: self.logger.error("Expected 'minimal fix' guidance in next_steps") return False self.logger.info(" ✅ Certain confidence skipped expert analysis correctly") return True except Exception as e: self.logger.error(f"Obvious import error certain test failed: {e}") return False def _test_certain_always_trusted(self) -> bool: """Test that certain confidence is always trusted regardless of complexity""" try: self.logger.info(" 1.2: Testing that certain confidence is always trusted") # Single step investigation with certain - should always be trusted self.logger.info(" 1.2.1: Direct certain confidence (always trusted)") response, _ = self.call_mcp_tool_direct( "debug", { "step": "Found the exact root cause and minimal fix for this complex issue", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "After thorough investigation, identified that the issue is caused by method name typo in Calculator.calculate_total() - calls self.add_number() instead of self.add_numbers(). Simple fix: change line 14 from 'add_number' to 'add_numbers'.", "files_checked": [self.typo_bug_file], "relevant_files": [self.typo_bug_file], "relevant_context": ["Calculator.calculate_total", "Calculator.add_numbers"], "hypothesis": "Method name typo in calculate_total() calls non-existent add_number() instead of add_numbers()", "confidence": "certain", # Should always be trusted "model": "flash", }, ) if not response: self.logger.error("Failed to get certain confidence response") return False response_data = self._parse_debug_response(response) if not response_data: return False # Verify certain is trusted regardless of complexity if response_data.get("status") != "certain_confidence_proceed_with_fix": self.logger.error("Certain confidence should always be trusted") return False if not response_data.get("skip_expert_analysis"): self.logger.error("Expert analysis should be skipped for certain confidence") return False # Ensure expert analysis is marked as skipped expert_analysis = response_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_certain_confidence": self.logger.error("Expert analysis status should indicate certain skip") return False self.logger.info(" ✅ Certain confidence always trusted correctly") return True except Exception as e: self.logger.error(f"Certain always trusted test failed: {e}") return False def _test_regular_high_confidence_expert_analysis(self) -> bool: """Test that regular 'high' confidence still triggers expert analysis""" try: self.logger.info(" 1.3: Testing that regular 'high' confidence triggers expert analysis") # Investigation with regular high confidence (not certain) self.logger.info(" 1.3.1: High confidence (not certain) - should trigger expert analysis") response, _ = self.call_mcp_tool_direct( "debug", { "step": "Identified likely root cause with strong evidence", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "IndentationError in data_processor.py line 8 - results.append(processed) is incorrectly indented. Should align with the 'if' statement above it.", "files_checked": [self.indentation_file], "relevant_files": [self.indentation_file], "relevant_context": ["process_data"], "hypothesis": "Incorrect indentation causes IndentationError in process_data function", "confidence": "high", # Regular high confidence, NOT certain "model": "flash", }, ) if not response: self.logger.error("Failed to get high confidence response") return False response_data = self._parse_debug_response(response) if not response_data: return False # Verify that regular high confidence triggers expert analysis if response_data.get("status") != "calling_expert_analysis": self.logger.error( f"Expected 'calling_expert_analysis' for high confidence, got '{response_data.get('status')}'" ) return False if response_data.get("skip_expert_analysis"): self.logger.error("Expert analysis should NOT be skipped for regular high confidence") return False # Verify expert analysis was called expert_analysis = response_data.get("expert_analysis", {}) if not expert_analysis: self.logger.error("Expected expert analysis for regular high confidence") return False # Check that expert analysis has content if "status" not in expert_analysis: self.logger.error("Expert analysis should have status field") return False self.logger.info(" ✅ Regular high confidence triggers expert analysis correctly") return True except Exception as e: self.logger.error(f"Regular high confidence test failed: {e}") return False def _test_multi_step_investigation_certain(self) -> bool: """Test multi-step investigation that ends with certain confidence""" try: self.logger.info(" 1.4: Testing multi-step investigation ending with certain") # Step 1: Start investigation self.logger.info(" 1.4.1: Step 1 - Initial investigation") response1, continuation_id = self.call_mcp_tool_direct( "debug", { "step": "Investigating Python syntax error in data processing module", "step_number": 1, "total_steps": 3, "next_step_required": True, "findings": "IndentationError reported when running data_processor.py - 'unindent does not match any outer indentation level'", "files_checked": [self.indentation_file], "relevant_files": [], "hypothesis": "Indentation inconsistency in Python code", "confidence": "low", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start multi-step investigation") return False # Step 2: Examine code structure self.logger.info(" 1.4.2: Step 2 - Code examination") response2, _ = self.call_mcp_tool_direct( "debug", { "step": "Examining the indentation structure in process_data function", "step_number": 2, "total_steps": 3, "next_step_required": True, "findings": "Found the issue: line 8 'results.append(processed)' is indented incorrectly. It should align with the 'if' statement, not be at the same level as the 'for' loop.", "files_checked": [self.indentation_file], "relevant_files": [self.indentation_file], "relevant_context": ["process_data"], "hypothesis": "Line 8 has incorrect indentation level causing IndentationError", "confidence": "medium", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Confirm fix with certain confidence self.logger.info(" 1.4.3: Step 3 - Confirmed fix (certain)") response3, _ = self.call_mcp_tool_direct( "debug", { "step": "Confirmed the exact issue and simple fix", "step_number": 3, "total_steps": 3, "next_step_required": False, # Final step "findings": "Confirmed: line 8 'results.append(processed)' needs to be indented 4 more spaces to align with line 6 'if item > 0:'. This is a simple indentation fix.", "files_checked": [self.indentation_file], "relevant_files": [self.indentation_file], "relevant_context": ["process_data"], "hypothesis": "IndentationError on line 8 due to incorrect indentation level - needs 4 more spaces", "confidence": "certain", # Final step with certain "continuation_id": continuation_id, "model": "flash", }, ) if not response3: self.logger.error("Failed to complete multi-step investigation") return False response3_data = self._parse_debug_response(response3) if not response3_data: return False # Validate multi-step certain response if response3_data.get("status") != "certain_confidence_proceed_with_fix": self.logger.error("Expected certain status for final step") return False if not response3_data.get("skip_expert_analysis"): self.logger.error("Expected expert analysis to be skipped for certain") return False # Verify investigation preserves steps (at least the current step) complete_investigation = response3_data.get("complete_investigation", {}) steps_taken = complete_investigation.get("steps_taken", 0) if steps_taken < 1: self.logger.error("Expected at least 1 step in complete investigation") return False # Check that work summary includes progression work_summary = complete_investigation.get("work_summary", "") if "Total steps:" not in work_summary and "Steps taken:" not in work_summary: self.logger.error("Work summary should show steps information") return False self.logger.info(" ✅ Multi-step investigation with certain ending successful") return True except Exception as e: self.logger.error(f"Multi-step investigation certain test failed: {e}") return False def call_mcp_tool_direct(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool directly in-process to maintain conversation memory""" try: # Get the tool instance if tool_name not in self._tools: self.logger.error(f"Tool '{tool_name}' not found in available tools") return None, None tool = self._tools[tool_name] # Execute the tool with proper async handling loop = self._get_event_loop() # Call the tool's execute method try: result = loop.run_until_complete(tool.execute(params)) except ToolExecutionError as exc: response_text = exc.payload continuation_id = self._extract_debug_continuation_id(response_text) return response_text, continuation_id if not result or len(result) == 0: self.logger.error(f"Tool '{tool_name}' returned empty result") return None, None # Extract the text content from the result response_text = result[0].text if hasattr(result[0], "text") else str(result[0]) # Extract continuation_id from debug response if present continuation_id = self._extract_debug_continuation_id(response_text) return response_text, continuation_id except Exception as e: self.logger.error(f"Failed to call tool '{tool_name}' directly: {e}") return None, None def _extract_debug_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from debug response""" try: response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for debug continuation_id: {e}") return None def _parse_debug_response(self, response_text: str) -> dict: """Parse debug tool JSON response""" try: return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse debug response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_investigation_response( self, response_data: dict, expected_step: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate debug investigation response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Basic structure checks if "investigation_status" not in response_data: self.logger.error("Missing investigation_status in response") return False if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating investigation response: {e}") return False ================================================ FILE: simulator_tests/test_debug_validation.py ================================================ #!/usr/bin/env python3 """ DebugWorkflow Tool Validation Test Tests the debug tool's capabilities using the new workflow architecture. This validates that the new workflow-based implementation maintains all the functionality of the original debug tool. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class DebugValidationTest(ConversationBaseTest): """Test debug tool with new workflow architecture""" @property def test_name(self) -> str: return "debug_validation" @property def test_description(self) -> str: return "Debug tool validation with new workflow architecture" def run_test(self) -> bool: """Test debug tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: DebugWorkflow tool validation (new architecture)") # Create a Python file with a subtle but realistic bug self._create_buggy_code() # Test 1: Single investigation session with multiple steps if not self._test_single_investigation_session(): return False # Test 2: Investigation flow that requires refinement if not self._test_investigation_refine_flow(): return False # Test 3: Complete investigation with expert analysis if not self._test_complete_investigation_with_analysis(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Context-aware file embedding if not self._test_context_aware_file_embedding(): return False # Test 6: Multi-step file context optimization if not self._test_multi_step_file_context(): return False self.logger.info(" ✅ All debug validation tests passed") return True except Exception as e: self.logger.error(f"DebugWorkflow validation test failed: {e}") return False def _create_buggy_code(self): """Create test files with a subtle bug for debugging""" # Create a Python file with dictionary iteration bug buggy_code = """#!/usr/bin/env python3 import json from datetime import datetime, timedelta class SessionManager: def __init__(self): self.active_sessions = {} self.session_timeout = 30 * 60 # 30 minutes in seconds def create_session(self, user_id, user_data): \"\"\"Create a new user session\"\"\" session_id = f"sess_{user_id}_{datetime.now().timestamp()}" session_info = { 'user_id': user_id, 'user_data': user_data, 'created_at': datetime.now(), 'expires_at': datetime.now() + timedelta(seconds=self.session_timeout) } self.active_sessions[session_id] = session_info return session_id def validate_session(self, session_id): \"\"\"Check if session is valid and not expired\"\"\" if session_id not in self.active_sessions: return False session = self.active_sessions[session_id] current_time = datetime.now() # Check if session has expired if current_time > session['expires_at']: del self.active_sessions[session_id] return False return True def cleanup_expired_sessions(self): \"\"\"Remove expired sessions from memory\"\"\" current_time = datetime.now() expired_count = 0 # BUG: Modifying dictionary while iterating over it for session_id, session in self.active_sessions.items(): if current_time > session['expires_at']: del self.active_sessions[session_id] # This causes RuntimeError expired_count += 1 return expired_count """ # Create test file with subtle bug self.buggy_file = self.create_additional_test_file("session_manager.py", buggy_code) self.logger.info(f" ✅ Created test file with subtle bug: {self.buggy_file}") # Create error description error_description = """ISSUE DESCRIPTION: Our session management system is experiencing intermittent failures during cleanup operations. SYMPTOMS: - Random RuntimeError: dictionary changed size during iteration - Occurs during high load when many sessions expire simultaneously - Error happens in cleanup_expired_sessions method - Affects about 5% of cleanup operations ERROR LOG: RuntimeError: dictionary changed size during iteration File "session_manager.py", line 44, in cleanup_expired_sessions for session_id, session in self.active_sessions.items(): """ self.error_file = self.create_additional_test_file("error_description.txt", error_description) self.logger.info(f" ✅ Created error description file: {self.error_file}") def _test_single_investigation_session(self) -> bool: """Test a complete investigation session with multiple steps""" try: self.logger.info(" 1.1: Testing single investigation session") # Step 1: Start investigation self.logger.info(" 1.1.1: Step 1 - Initial investigation") response1, continuation_id = self.call_mcp_tool( "debug", { "step": "I need to investigate intermittent RuntimeError during session cleanup. Let me start by examining the error description and understanding the symptoms.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "RuntimeError occurs during dictionary iteration in cleanup_expired_sessions method. Error happens intermittently during high load.", "files_checked": [self.error_file], "relevant_files": [self.error_file], }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial investigation response") return False # Parse and validate JSON response response1_data = self._parse_debug_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_investigation for next_step_required=True if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_investigation"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Examine the code self.logger.info(" 1.1.2: Step 2 - Code examination") response2, _ = self.call_mcp_tool( "debug", { "step": "Now examining the session_manager.py file to understand the cleanup_expired_sessions implementation and identify the root cause.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Found the issue: cleanup_expired_sessions modifies self.active_sessions dictionary while iterating over it with .items(). This causes RuntimeError when del is called during iteration.", "files_checked": [self.error_file, self.buggy_file], "relevant_files": [self.buggy_file], "relevant_context": ["SessionManager.cleanup_expired_sessions"], "hypothesis": "Dictionary is being modified during iteration causing RuntimeError", "confidence": "high", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue investigation to step 2") return False response2_data = self._parse_debug_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_investigation"): return False # Check investigation status tracking investigation_status = response2_data.get("investigation_status", {}) if investigation_status.get("files_checked", 0) < 2: self.logger.error("Files checked count not properly tracked") return False if investigation_status.get("relevant_context", 0) != 1: self.logger.error("Relevant context not properly tracked") return False if investigation_status.get("current_confidence") != "high": self.logger.error("Confidence level not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper tracking") # Store continuation_id for next test self.investigation_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single investigation session test failed: {e}") return False def _test_investigation_refine_flow(self) -> bool: """Test investigation flow that requires refining the approach""" try: self.logger.info(" 1.2: Testing investigation refinement workflow") # Start a new investigation for testing refinement behaviour self.logger.info(" 1.2.1: Start investigation for refinement test") response1, continuation_id = self.call_mcp_tool( "debug", { "step": "Investigating performance degradation in data processing pipeline", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial analysis shows slow database queries", "files_checked": ["/db/queries.py"], "relevant_files": ["/db/queries.py"], }, ) if not response1 or not continuation_id: self.logger.error("Failed to start refinement test investigation") return False # Step 2: Wrong direction self.logger.info(" 1.2.2: Step 2 - Wrong investigation path") response2, _ = self.call_mcp_tool( "debug", { "step": "Focusing on database optimization strategies", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Database queries seem optimized, might be looking in wrong place", "files_checked": ["/db/queries.py", "/db/indexes.py"], "relevant_files": [], "hypothesis": "Database performance issues", "confidence": "low", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Backtrack from step 2 self.logger.info(" 1.2.3: Step 3 - Refine investigation path") response3, _ = self.call_mcp_tool( "debug", { "step": "Refocusing - the issue might not be database related. Let me investigate the data processing algorithm instead.", "step_number": 3, "total_steps": 4, "next_step_required": True, "findings": "Found inefficient nested loops in data processor causing O(n²) complexity", "files_checked": ["/processor/algorithm.py"], "relevant_files": ["/processor/algorithm.py"], "relevant_context": ["DataProcessor.process_batch"], "hypothesis": "Inefficient algorithm causing performance issues", "confidence": "medium", "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to refine investigation") return False response3_data = self._parse_debug_response(response3) if not self._validate_step_response(response3_data, 3, 4, True, "pause_for_investigation"): return False self.logger.info(" ✅ Investigation refinement working correctly") return True except Exception as e: self.logger.error(f"Investigation refinement test failed: {e}") return False def _test_complete_investigation_with_analysis(self) -> bool: """Test complete investigation ending with expert analysis""" try: self.logger.info(" 1.3: Testing complete investigation with expert analysis") # Use the continuation from first test continuation_id = getattr(self, "investigation_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh investigation") response0, continuation_id = self.call_mcp_tool( "debug", { "step": "Investigating the dictionary iteration bug in session cleanup", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found dictionary modification during iteration", "files_checked": [self.buggy_file], "relevant_files": [self.buggy_file], "relevant_context": ["SessionManager.cleanup_expired_sessions"], }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh investigation") return False # Final step - trigger expert analysis self.logger.info(" 1.3.1: Final step - complete investigation") response_final, _ = self.call_mcp_tool( "debug", { "step": "Investigation complete. The root cause is confirmed: cleanup_expired_sessions modifies the dictionary while iterating, causing RuntimeError.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert analysis "findings": "Root cause identified: del self.active_sessions[session_id] on line 46 modifies dictionary during iteration starting at line 44. Fix: collect expired IDs first, then delete.", "files_checked": [self.buggy_file], "relevant_files": [self.buggy_file], "relevant_context": ["SessionManager.cleanup_expired_sessions"], "hypothesis": "Dictionary modification during iteration causes RuntimeError in cleanup_expired_sessions", "confidence": "high", "continuation_id": continuation_id, "model": "flash", # Use flash for expert analysis }, ) if not response_final: self.logger.error("Failed to complete investigation") return False response_final_data = self._parse_debug_response(response_final) if not response_final_data: return False # Validate final response structure - expect calling_expert_analysis for next_step_required=False if response_final_data.get("status") != "calling_expert_analysis": self.logger.error( f"Expected status 'calling_expert_analysis', got '{response_final_data.get('status')}'" ) return False if not response_final_data.get("investigation_complete"): self.logger.error("Expected investigation_complete=true for final step") return False # Check for expert analysis if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) # Check for expected analysis content (checking common patterns) analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower() # Look for bug identification bug_indicators = ["dictionary", "iteration", "modify", "runtime", "error", "del"] found_indicators = sum(1 for indicator in bug_indicators if indicator in analysis_text) if found_indicators >= 3: self.logger.info(" ✅ Expert analysis identified the bug correctly") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully identified the bug (found {found_indicators}/6 indicators)" ) # Check complete investigation summary if "complete_investigation" not in response_final_data: self.logger.error("Missing complete_investigation in final response") return False complete_investigation = response_final_data["complete_investigation"] if not complete_investigation.get("relevant_context"): self.logger.error("Missing relevant context in complete investigation") return False if "SessionManager.cleanup_expired_sessions" not in complete_investigation["relevant_context"]: self.logger.error("Expected method not found in investigation summary") return False self.logger.info(" ✅ Complete investigation with expert analysis successful") return True except Exception as e: self.logger.error(f"Complete investigation test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test certain confidence behavior - should skip expert analysis""" try: self.logger.info(" 1.4: Testing certain confidence behavior") # Test certain confidence - should skip expert analysis self.logger.info(" 1.4.1: Certain confidence investigation") response_certain, _ = self.call_mcp_tool( "debug", { "step": "I have confirmed the exact root cause with 100% certainty: dictionary modification during iteration.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "The bug is on line 44-47: for loop iterates over dict.items() while del modifies the dict inside the loop. Fix is simple: collect expired IDs first, then delete after iteration.", "files_checked": [self.buggy_file], "relevant_files": [self.buggy_file], "relevant_context": ["SessionManager.cleanup_expired_sessions"], "hypothesis": "Dictionary modification during iteration causes RuntimeError - fix is straightforward", "confidence": "certain", # This should skip expert analysis "model": "flash", }, ) if not response_certain: self.logger.error("Failed to test certain confidence") return False response_certain_data = self._parse_debug_response(response_certain) if not response_certain_data: return False # Validate certain confidence response - should skip expert analysis if response_certain_data.get("status") != "certain_confidence_proceed_with_fix": self.logger.error( f"Expected status 'certain_confidence_proceed_with_fix', got '{response_certain_data.get('status')}'" ) return False if not response_certain_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for certain confidence") return False expert_analysis = response_certain_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_certain_confidence": self.logger.error("Expert analysis should be skipped for certain confidence") return False self.logger.info(" ✅ Certain confidence behavior working correctly") return True except Exception as e: self.logger.error(f"Certain confidence test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for debug-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from debug response specifically continuation_id = self._extract_debug_continuation_id(response_text) return response_text, continuation_id def _extract_debug_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from debug response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for debug continuation_id: {e}") return None def _parse_debug_response(self, response_text: str) -> dict: """Parse debug tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse debug response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a debug investigation step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check investigation_status exists if "investigation_status" not in response_data: self.logger.error("Missing investigation_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False def _test_context_aware_file_embedding(self) -> bool: """Test context-aware file embedding optimization""" try: self.logger.info(" 1.5: Testing context-aware file embedding") # Create multiple test files for context testing file1_content = """#!/usr/bin/env python3 def process_data(data): \"\"\"Process incoming data\"\"\" result = [] for item in data: if item.get('valid'): result.append(item['value']) return result """ file2_content = """#!/usr/bin/env python3 def validate_input(data): \"\"\"Validate input data\"\"\" if not isinstance(data, list): raise ValueError("Data must be a list") for item in data: if not isinstance(item, dict): raise ValueError("Items must be dictionaries") if 'value' not in item: raise ValueError("Items must have 'value' key") return True """ # Create test files file1 = self.create_additional_test_file("data_processor.py", file1_content) file2 = self.create_additional_test_file("validator.py", file2_content) # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "debug", { "step": "Starting investigation of data processing pipeline", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of data processing components", "files_checked": [file1, file2], "relevant_files": [file1], # This should be referenced, not embedded "relevant_context": ["process_data"], "hypothesis": "Investigating data flow", "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_debug_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False if "Files referenced but not embedded" not in file_context.get("context_optimization", ""): self.logger.error("Expected context optimization message for reference_only") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Intermediate step with continuation - should still only reference self.logger.info(" 1.5.2: Intermediate step with continuation (should reference only)") response2, _ = self.call_mcp_tool( "debug", { "step": "Continuing investigation with more detailed analysis", "step_number": 2, "total_steps": 3, "next_step_required": True, # Still intermediate "continuation_id": continuation_id, "findings": "Found potential issues in validation logic", "files_checked": [file1, file2], "relevant_files": [file1, file2], # Both files referenced "relevant_context": ["process_data", "validate_input"], "hypothesis": "Validation might be too strict", "confidence": "medium", "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False response2_data = self._parse_debug_response(response2) if not response2_data: return False # Check file context - should still be reference_only file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context for step 2, got: {file_context2.get('type')}") return False # Should include reference note if not file_context2.get("note"): self.logger.error("Expected file reference note for intermediate step") return False reference_note = file_context2.get("note", "") if "data_processor.py" not in reference_note or "validator.py" not in reference_note: self.logger.error("File reference note should mention both files") return False self.logger.info(" ✅ Intermediate step with continuation correctly uses reference_only") # Test 3: Final step - should embed files for expert analysis self.logger.info(" 1.5.3: Final step (should embed files)") response3, _ = self.call_mcp_tool( "debug", { "step": "Investigation complete - identified the root cause", "step_number": 3, "total_steps": 3, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Root cause: validator is rejecting valid data due to strict type checking", "files_checked": [file1, file2], "relevant_files": [file1, file2], # Should be fully embedded "relevant_context": ["process_data", "validate_input"], "hypothesis": "Validation logic is too restrictive for valid edge cases", "confidence": "high", "model": "flash", }, ) if not response3: self.logger.error("Failed to complete to final step") return False response3_data = self._parse_debug_response(response3) if not response3_data: return False # Check file context - should be fully_embedded for final step file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context3.get('type')}" ) return False if "Full file content embedded for expert analysis" not in file_context3.get("context_optimization", ""): self.logger.error("Expected expert analysis optimization message for fully_embedded") return False # Should show files embedded count files_embedded = file_context3.get("files_embedded", 0) if files_embedded == 0: # This is OK - files might already be in conversation history self.logger.info( " ℹ️ Files embedded count is 0 - files already in conversation history (smart deduplication)" ) else: self.logger.info(f" ✅ Files embedded count: {files_embedded}") self.logger.info(" ✅ Final step correctly uses fully_embedded file context") # Verify expert analysis was called for final step if response3_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False if "expert_analysis" not in response3_data: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Context-aware file embedding test completed successfully") return True except Exception as e: self.logger.error(f"Context-aware file embedding test failed: {e}") return False def _test_multi_step_file_context(self) -> bool: """Test multi-step workflow with proper file context transitions""" try: self.logger.info(" 1.6: Testing multi-step file context optimization") # Create a complex scenario with multiple files config_content = """#!/usr/bin/env python3 import os DATABASE_URL = os.getenv('DATABASE_URL', 'sqlite:///app.db') DEBUG_MODE = os.getenv('DEBUG', 'False').lower() == 'true' MAX_CONNECTIONS = int(os.getenv('MAX_CONNECTIONS', '10')) # Bug: This will cause issues when MAX_CONNECTIONS is not a valid integer CACHE_SIZE = MAX_CONNECTIONS * 2 # Problematic if MAX_CONNECTIONS is invalid """ server_content = """#!/usr/bin/env python3 from config import DATABASE_URL, DEBUG_MODE, CACHE_SIZE import sqlite3 class DatabaseServer: def __init__(self): self.connection_pool = [] self.cache_size = CACHE_SIZE # This will fail if CACHE_SIZE is invalid def connect(self): try: conn = sqlite3.connect(DATABASE_URL) self.connection_pool.append(conn) return conn except Exception as e: print(f"Connection failed: {e}") return None """ # Create test files config_file = self.create_additional_test_file("config.py", config_content) server_file = self.create_additional_test_file("database_server.py", server_content) # Step 1: Start investigation (new conversation) self.logger.info(" 1.6.1: Step 1 - Start investigation") response1, continuation_id = self.call_mcp_tool( "debug", { "step": "Investigating application startup failures in production environment", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Application fails to start with configuration errors", "files_checked": [config_file], "relevant_files": [config_file], "relevant_context": [], "hypothesis": "Configuration issue causing startup failure", "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start multi-step file context test") return False response1_data = self._parse_debug_response(response1) # Validate step 1 - should use reference_only file_context1 = response1_data.get("file_context", {}) if file_context1.get("type") != "reference_only": self.logger.error("Step 1 should use reference_only file context") return False self.logger.info(" ✅ Step 1: reference_only file context") # Step 2: Expand investigation self.logger.info(" 1.6.2: Step 2 - Expand investigation") response2, _ = self.call_mcp_tool( "debug", { "step": "Found configuration issue - investigating database server initialization", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "MAX_CONNECTIONS environment variable contains invalid value, causing CACHE_SIZE calculation to fail", "files_checked": [config_file, server_file], "relevant_files": [config_file, server_file], "relevant_context": ["DatabaseServer.__init__"], "hypothesis": "Invalid environment variable causing integer conversion error", "confidence": "medium", "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False response2_data = self._parse_debug_response(response2) # Validate step 2 - should still use reference_only file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "reference_only": self.logger.error("Step 2 should use reference_only file context") return False # Should reference both files reference_note = file_context2.get("note", "") if "config.py" not in reference_note or "database_server.py" not in reference_note: self.logger.error("Step 2 should reference both files in note") return False self.logger.info(" ✅ Step 2: reference_only file context with multiple files") # Step 3: Deep analysis self.logger.info(" 1.6.3: Step 3 - Deep analysis") response3, _ = self.call_mcp_tool( "debug", { "step": "Analyzing the exact error propagation path and impact", "step_number": 3, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Error occurs in config.py line 8 when MAX_CONNECTIONS is not numeric, then propagates to DatabaseServer.__init__", "files_checked": [config_file, server_file], "relevant_files": [config_file, server_file], "relevant_context": ["DatabaseServer.__init__"], "hypothesis": "Need proper error handling and validation for environment variables", "confidence": "high", "model": "flash", }, ) if not response3: self.logger.error("Failed to continue to step 3") return False response3_data = self._parse_debug_response(response3) # Validate step 3 - should still use reference_only file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "reference_only": self.logger.error("Step 3 should use reference_only file context") return False self.logger.info(" ✅ Step 3: reference_only file context") # Step 4: Final analysis with expert consultation self.logger.info(" 1.6.4: Step 4 - Final step with expert analysis") response4, _ = self.call_mcp_tool( "debug", { "step": "Investigation complete - root cause identified with solution", "step_number": 4, "total_steps": 4, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Root cause: config.py assumes MAX_CONNECTIONS env var is always a valid integer. Fix: add try/except with default value and proper validation.", "files_checked": [config_file, server_file], "relevant_files": [config_file, server_file], "relevant_context": ["DatabaseServer.__init__"], "hypothesis": "Environment variable validation needed with proper error handling", "confidence": "high", "model": "flash", }, ) if not response4: self.logger.error("Failed to complete to final step") return False response4_data = self._parse_debug_response(response4) # Validate step 4 - should use fully_embedded for expert analysis file_context4 = response4_data.get("file_context", {}) if file_context4.get("type") != "fully_embedded": self.logger.error("Step 4 (final) should use fully_embedded file context") return False if "expert analysis" not in file_context4.get("context_optimization", "").lower(): self.logger.error("Final step should mention expert analysis in context optimization") return False # Verify expert analysis was triggered if response4_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False # Check that expert analysis has file context expert_analysis = response4_data.get("expert_analysis", {}) if not expert_analysis: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Step 4: fully_embedded file context with expert analysis") # Validate the complete workflow progression progression_summary = { "step_1": "reference_only (new conversation, intermediate)", "step_2": "reference_only (continuation, intermediate)", "step_3": "reference_only (continuation, intermediate)", "step_4": "fully_embedded (continuation, final)", } self.logger.info(" 📋 File context progression:") for step, context_type in progression_summary.items(): self.logger.info(f" {step}: {context_type}") self.logger.info(" ✅ Multi-step file context optimization test completed successfully") return True except Exception as e: self.logger.error(f"Multi-step file context test failed: {e}") return False ================================================ FILE: simulator_tests/test_line_number_validation.py ================================================ """ Test to validate line number handling across different tools """ import json import os from .base_test import BaseSimulatorTest class LineNumberValidationTest(BaseSimulatorTest): """Test that validates correct line number handling in chat, analyze, and refactor tools""" @property def test_name(self) -> str: return "line_number_validation" @property def test_description(self) -> str: return "Line number handling validation across tools" def run_test(self) -> bool: """Test line number handling in different tools""" try: self.logger.info("Test: Line number handling validation") # Setup test files self.setup_test_files() # Create a test file with known content test_file_content = '''# Example code with specific elements def calculate_total(items): """Calculate total with tax""" subtotal = 0 tax_rate = 0.08 # Line 5 - tax_rate defined for item in items: # Line 7 - loop starts if item.price > 0: subtotal += item.price tax_amount = subtotal * tax_rate # Line 11 return subtotal + tax_amount def validate_data(data): """Validate input data""" # Line 15 required_fields = ["name", "email", "age"] # Line 16 for field in required_fields: if field not in data: raise ValueError(f"Missing field: {field}") return True # Line 22 ''' test_file_path = os.path.join(self.test_dir, "line_test.py") with open(test_file_path, "w") as f: f.write(test_file_content) self.logger.info(f"Created test file: {test_file_path}") # Test 1: Chat tool asking about specific line self.logger.info(" 1.1: Testing chat tool with line number question") content, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Where is tax_rate defined in this file? Please tell me the exact line number.", "absolute_file_paths": [test_file_path], "model": "flash", }, ) if content: # Check if the response mentions line 5 if "line 5" in content.lower() or "line 5" in content: self.logger.info(" ✅ Chat tool correctly identified tax_rate at line 5") else: self.logger.warning(f" ⚠️ Chat tool response didn't mention line 5: {content[:200]}...") else: self.logger.error(" ❌ Chat tool request failed") return False # Test 2: Analyze tool with line number reference self.logger.info(" 1.2: Testing analyze tool with line number analysis") content, continuation_id = self.call_mcp_tool( "analyze", { "prompt": "What happens between lines 7-11 in this code? Focus on the loop logic.", "absolute_file_paths": [test_file_path], "model": "flash", }, ) if content: # Check if the response references the loop if any(term in content.lower() for term in ["loop", "iterate", "line 7", "lines 7"]): self.logger.info(" ✅ Analyze tool correctly analyzed the specified line range") else: self.logger.warning(" ⚠️ Analyze tool response unclear about line range") else: self.logger.error(" ❌ Analyze tool request failed") return False # Test 3: Refactor tool with line number precision self.logger.info(" 1.3: Testing refactor tool line number precision") content, continuation_id = self.call_mcp_tool( "refactor", { "prompt": "Analyze this code for refactoring opportunities", "absolute_file_paths": [test_file_path], "refactor_type": "codesmells", "model": "flash", }, ) if content: try: # Parse the JSON response result = json.loads(content) if result.get("status") == "refactor_analysis_complete": opportunities = result.get("refactor_opportunities", []) if opportunities: # Check if line numbers are precise has_line_refs = any( opp.get("start_line") is not None and opp.get("end_line") is not None for opp in opportunities ) if has_line_refs: self.logger.info(" ✅ Refactor tool provided precise line number references") # Log some examples for opp in opportunities[:2]: if opp.get("start_line"): self.logger.info( f" - Issue at lines {opp['start_line']}-{opp['end_line']}: {opp.get('issue', '')[:50]}..." ) else: self.logger.warning(" ⚠️ Refactor tool response missing line numbers") else: self.logger.info(" ℹ️ No refactoring opportunities found (code might be too clean)") except json.JSONDecodeError: self.logger.warning(" ⚠️ Refactor tool response not valid JSON") else: self.logger.error(" ❌ Refactor tool request failed") return False # Test 4: Validate log patterns self.logger.info(" 1.4: Validating line number processing in logs") # Get logs from server try: log_file_path = "logs/mcp_server.log" with open(log_file_path) as f: lines = f.readlines() logs = "".join(lines[-500:]) except Exception as e: self.logger.error(f"Failed to read server logs: {e}") logs = "" pass # Check for line number formatting patterns line_number_patterns = ["Line numbers for", "enabled", "│", "line number"] # The line number separator found_patterns = 0 for pattern in line_number_patterns: if pattern in logs: found_patterns += 1 self.logger.info(f" Found {found_patterns}/{len(line_number_patterns)} line number patterns in logs") if found_patterns >= 2: self.logger.info(" ✅ Line number processing confirmed in logs") else: self.logger.warning(" ⚠️ Limited line number processing evidence in logs") self.logger.info(" ✅ Line number validation test completed successfully") return True except Exception as e: self.logger.error(f"Line number validation test failed: {type(e).__name__}: {e}") return False ================================================ FILE: simulator_tests/test_logs_validation.py ================================================ #!/usr/bin/env python3 """ Server Logs Validation Test Validates server logs to confirm file deduplication behavior and conversation threading is working properly. """ from .base_test import BaseSimulatorTest class LogsValidationTest(BaseSimulatorTest): """Validate server logs to confirm file deduplication behavior""" @property def test_name(self) -> str: return "logs_validation" @property def test_description(self) -> str: return "Server logs validation" def run_test(self) -> bool: """Validate server logs to confirm file deduplication behavior""" try: self.logger.info("📋 Test: Validating server logs for file deduplication...") # Get server logs from log files import os logs = "" log_files = ["logs/mcp_server.log", "logs/mcp_activity.log"] for log_file in log_files: if os.path.exists(log_file): try: with open(log_file) as f: file_content = f.read() logs += f"\n=== {log_file} ===\n{file_content}\n" self.logger.debug(f"Read {len(file_content)} characters from {log_file}") except Exception as e: self.logger.warning(f"Could not read {log_file}: {e}") else: self.logger.warning(f"Log file not found: {log_file}") if not logs.strip(): self.logger.warning("No log content found - server may not have processed any requests yet") return False # Look for conversation threading patterns that indicate the system is working conversation_patterns = [ "CONVERSATION_RESUME", "CONVERSATION_CONTEXT", "previous turns loaded", "tool embedding", "files included", "files truncated", "already in conversation history", ] conversation_lines = [] for line in logs.split("\n"): for pattern in conversation_patterns: if pattern.lower() in line.lower(): conversation_lines.append(line.strip()) break # Look for evidence of conversation threading and file handling conversation_threading_found = False multi_turn_conversations = False for line in conversation_lines: lower_line = line.lower() if "conversation_resume" in lower_line: conversation_threading_found = True self.logger.debug(f"📄 Conversation threading: {line}") elif "previous turns loaded" in lower_line: multi_turn_conversations = True self.logger.debug(f"📄 Multi-turn conversation: {line}") elif "already in conversation" in lower_line: self.logger.info(f"✅ Found explicit deduplication: {line}") return True # Conversation threading with multiple turns is evidence of file deduplication working if conversation_threading_found and multi_turn_conversations: self.logger.info("✅ Conversation threading with multi-turn context working") self.logger.info( "✅ File deduplication working implicitly (files embedded once in conversation history)" ) return True elif conversation_threading_found: self.logger.info("✅ Conversation threading detected") return True else: self.logger.warning("⚠️ No clear evidence of conversation threading in logs") self.logger.debug(f"Found {len(conversation_lines)} conversation-related log lines") return False except Exception as e: self.logger.error(f"Log validation failed: {e}") return False ================================================ FILE: simulator_tests/test_model_thinking_config.py ================================================ #!/usr/bin/env python3 """ Model Thinking Configuration Test Tests that thinking configuration is properly applied only to models that support it, and that Flash models work correctly without thinking config. """ from .base_test import BaseSimulatorTest class TestModelThinkingConfig(BaseSimulatorTest): """Test model-specific thinking configuration behavior""" @property def test_name(self) -> str: return "model_thinking_config" @property def test_description(self) -> str: return "Model-specific thinking configuration behavior" def test_pro_model_with_thinking_config(self): """Test that Pro model uses thinking configuration""" self.logger.info("Testing Pro model with thinking configuration...") try: # Test with explicit pro model and high thinking mode response, continuation_id = self.call_mcp_tool( "chat", { "prompt": "What is 2 + 2? Please think carefully and explain.", "model": "pro", # Should resolve to gemini-2.5-pro "thinking_mode": "high", # Should use thinking_config }, ) if not response: raise Exception("Pro model test failed: No response received") self.logger.info("✅ Pro model with thinking config works correctly") return True except Exception as e: self.logger.error(f"❌ Pro model test failed: {e}") return False def test_flash_model_without_thinking_config(self): """Test that Flash model works without thinking configuration""" self.logger.info("Testing Flash model without thinking configuration...") try: # Test with explicit flash model and thinking mode (should be ignored) response, continuation_id = self.call_mcp_tool( "chat", { "prompt": "What is 3 + 3? Give a quick answer.", "model": "flash", # Should resolve to gemini-2.5-flash "thinking_mode": "high", # Should be ignored for Flash model }, ) if not response: raise Exception("Flash model test failed: No response received") self.logger.info("✅ Flash model without thinking config works correctly") return True except Exception as e: if "thinking" in str(e).lower() and ("not supported" in str(e).lower() or "invalid" in str(e).lower()): raise Exception(f"Flash model incorrectly tried to use thinking config: {e}") self.logger.error(f"❌ Flash model test failed: {e}") return False def test_model_resolution_logic(self): """Test that model resolution works correctly for both shortcuts and full names""" self.logger.info("Testing model resolution logic...") test_cases = [ ("pro", "should work with Pro model"), ("flash", "should work with Flash model"), ("gemini-2.5-pro", "should work with full Pro model name"), ("gemini-2.5-flash", "should work with full Flash model name"), ] success_count = 0 for model_name, description in test_cases: try: response, continuation_id = self.call_mcp_tool( "chat", { "prompt": f"Test with {model_name}: What is 1 + 1?", "model": model_name, "thinking_mode": "medium", }, ) if not response: raise Exception(f"No response received for model {model_name}") self.logger.info(f"✅ {model_name} {description}") success_count += 1 except Exception as e: self.logger.error(f"❌ {model_name} failed: {e}") return False return success_count == len(test_cases) def test_default_model_behavior(self): """Test behavior with server default model (no explicit model specified)""" self.logger.info("Testing default model behavior...") try: # Test without specifying model (should use server default) response, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Test default model: What is 4 + 4?", # No model specified - should use DEFAULT_MODEL from config "thinking_mode": "medium", }, ) if not response: raise Exception("Default model test failed: No response received") self.logger.info("✅ Default model behavior works correctly") return True except Exception as e: self.logger.error(f"❌ Default model test failed: {e}") return False def run_test(self) -> bool: """Run all model thinking configuration tests""" self.logger.info(f" Test: {self.test_description}") try: # Test Pro model with thinking config if not self.test_pro_model_with_thinking_config(): return False # Test Flash model without thinking config if not self.test_flash_model_without_thinking_config(): return False # Test model resolution logic if not self.test_model_resolution_logic(): return False # Test default model behavior if not self.test_default_model_behavior(): return False self.logger.info(f"✅ All {self.test_name} tests passed!") return True except Exception as e: self.logger.error(f"❌ {self.test_name} test failed: {e}") return False def main(): """Run the model thinking configuration tests""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = TestModelThinkingConfig(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_o3_model_selection.py ================================================ #!/usr/bin/env python3 """ O3 Model Selection Test Tests that O3 models are properly selected and used when explicitly specified, regardless of the default model configuration (even when set to auto). Validates model selection via server logs. """ import datetime from .base_test import BaseSimulatorTest class O3ModelSelectionTest(BaseSimulatorTest): """Test O3 model selection and usage""" @property def test_name(self) -> str: return "o3_model_selection" @property def test_description(self) -> str: return "O3 model selection and usage validation" def run_test(self) -> bool: """Test O3 model selection and usage""" try: self.logger.info(" Test: O3 model selection and usage validation") # Check which API keys are configured import os has_openai = bool(os.environ.get("OPENAI_API_KEY")) has_openrouter = bool(os.environ.get("OPENROUTER_API_KEY")) # If only OpenRouter is configured, adjust test expectations if has_openrouter and not has_openai: self.logger.info(" ℹ️ Only OpenRouter configured - O3 models will be routed through OpenRouter") return self._run_openrouter_o3_test() # If neither OpenAI nor OpenRouter is configured, skip the test if not has_openai and not has_openrouter: self.logger.info(" ⚠️ Neither OpenAI nor OpenRouter API keys configured - skipping test") self.logger.info( " ℹ️ This test requires either OPENAI_API_KEY or OPENROUTER_API_KEY to be set in .env" ) self.logger.info(" ✅ Test skipped (no API keys configured)") return True # Return True to indicate test passed/skipped # Original test for when OpenAI is configured self.logger.info(" ℹ️ OpenAI API configured - expecting direct OpenAI API calls") # Setup test files for later use self.setup_test_files() # Get timestamp for log filtering datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") # Test 1: Explicit O3 model selection self.logger.info(" 1: Testing explicit O3 model selection") response1, _ = self.call_mcp_tool( "chat", { "prompt": "Simple test: What is 2 + 2? Just give a brief answer.", "model": "o3", "temperature": 1.0, # O3 only supports default temperature of 1.0 }, ) if not response1: self.logger.error(" ❌ O3 model test failed") return False self.logger.info(" ✅ O3 model call completed") # Test 2: Explicit O3-mini model selection self.logger.info(" 2: Testing explicit O3-mini model selection") response2, _ = self.call_mcp_tool( "chat", { "prompt": "Simple test: What is 3 + 3? Just give a brief answer.", "model": "o3-mini", "temperature": 1.0, # O3-mini only supports default temperature of 1.0 }, ) if not response2: self.logger.error(" ❌ O3-mini model test failed") return False self.logger.info(" ✅ O3-mini model call completed") # Test 3: Another tool with O3 to ensure it works across tools self.logger.info(" 3: Testing O3 with different tool (codereview)") # Create a simple test file test_code = """def add(a, b): return a + b def multiply(x, y): return x * y """ test_file = self.create_additional_test_file("simple_math.py", test_code) response3, _ = self.call_mcp_tool( "codereview", { "step": "Review this simple code for quality and potential issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting code review analysis", "relevant_files": [test_file], "model": "o3", "temperature": 1.0, # O3 only supports default temperature of 1.0 }, ) if not response3: self.logger.error(" ❌ O3 with codereview tool failed") return False self.logger.info(" ✅ O3 with codereview tool completed") # Validate model usage from server logs self.logger.info(" 4: Validating model usage in logs") logs = self.get_recent_server_logs() # Check for OpenAI API calls (this proves O3 models are being used) openai_api_logs = [line for line in logs.split("\n") if "Sending request to openai API for" in line] # Check for OpenAI model usage logs openai_model_logs = [ line for line in logs.split("\n") if "Using model:" in line and "openai provider" in line ] # Check for successful OpenAI responses openai_response_logs = [ line for line in logs.split("\n") if "openai provider" in line and "Using model:" in line ] # Check that we have both chat and codereview tool calls to OpenAI chat_openai_logs = [line for line in logs.split("\n") if "Sending request to openai API for chat" in line] codereview_openai_logs = [ line for line in logs.split("\n") if "Sending request to openai API for codereview" in line ] # Validation criteria - check for OpenAI usage evidence (more flexible than exact counts) openai_api_called = len(openai_api_logs) >= 1 # Should see at least 1 OpenAI API call openai_model_usage = len(openai_model_logs) >= 1 # Should see at least 1 model usage log openai_responses_received = len(openai_response_logs) >= 1 # Should see at least 1 response some_chat_calls_to_openai = len(chat_openai_logs) >= 1 # Should see at least 1 chat call some_workflow_calls_to_openai = ( len(codereview_openai_logs) >= 1 or len([line for line in logs.split("\n") if "openai" in line and "codereview" in line]) > 0 ) # Should see evidence of workflow tool usage self.logger.info(f" OpenAI API call logs: {len(openai_api_logs)}") self.logger.info(f" OpenAI model usage logs: {len(openai_model_logs)}") self.logger.info(f" OpenAI response logs: {len(openai_response_logs)}") self.logger.info(f" Chat calls to OpenAI: {len(chat_openai_logs)}") self.logger.info(f" Codereview calls to OpenAI: {len(codereview_openai_logs)}") # Log sample evidence for debugging if self.verbose and openai_api_logs: self.logger.debug(" 📋 Sample OpenAI API logs:") for log in openai_api_logs[:5]: self.logger.debug(f" {log}") if self.verbose and chat_openai_logs: self.logger.debug(" 📋 Sample chat OpenAI logs:") for log in chat_openai_logs[:3]: self.logger.debug(f" {log}") # Success criteria success_criteria = [ ("OpenAI API calls made", openai_api_called), ("OpenAI model usage logged", openai_model_usage), ("OpenAI responses received", openai_responses_received), ("Chat tool used OpenAI", some_chat_calls_to_openai), ( "Workflow tool attempted", some_workflow_calls_to_openai or response3 is not None, ), # More flexible check ] passed_criteria = sum(1 for _, passed in success_criteria if passed) self.logger.info(f" Success criteria met: {passed_criteria}/{len(success_criteria)}") for criterion, passed in success_criteria: status = "✅" if passed else "❌" self.logger.info(f" {status} {criterion}") if passed_criteria >= 3: # At least 3 out of 5 criteria self.logger.info(" ✅ O3 model selection validation passed") return True else: self.logger.error(" ❌ O3 model selection validation failed") return False except Exception as e: self.logger.error(f"O3 model selection test failed: {e}") return False finally: self.cleanup_test_files() def _run_openrouter_o3_test(self) -> bool: """Test O3 model selection when using OpenRouter""" try: # Setup test files self.setup_test_files() # Test 1: O3 model via OpenRouter self.logger.info(" 1: Testing O3 model via OpenRouter") response1, _ = self.call_mcp_tool( "chat", { "prompt": "Simple test: What is 2 + 2? Just give a brief answer.", "model": "o3", "temperature": 1.0, }, ) if not response1: self.logger.error(" ❌ O3 model test via OpenRouter failed") return False self.logger.info(" ✅ O3 model call via OpenRouter completed") # Test 2: O3-mini model via OpenRouter self.logger.info(" 2: Testing O3-mini model via OpenRouter") response2, _ = self.call_mcp_tool( "chat", { "prompt": "Simple test: What is 3 + 3? Just give a brief answer.", "model": "o3-mini", "temperature": 1.0, }, ) if not response2: self.logger.error(" ❌ O3-mini model test via OpenRouter failed") return False self.logger.info(" ✅ O3-mini model call via OpenRouter completed") # Test 3: Codereview with O3 via OpenRouter self.logger.info(" 3: Testing O3 with codereview tool via OpenRouter") test_code = """def add(a, b): return a + b def multiply(x, y): return x * y """ test_file = self.create_additional_test_file("simple_math.py", test_code) response3, _ = self.call_mcp_tool( "codereview", { "step": "Review this simple code for quality and potential issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting code review analysis", "relevant_files": [test_file], "model": "o3", "temperature": 1.0, }, ) if not response3: self.logger.error(" ❌ O3 with codereview tool via OpenRouter failed") return False self.logger.info(" ✅ O3 with codereview tool via OpenRouter completed") # Validate OpenRouter usage in logs self.logger.info(" 4: Validating OpenRouter usage in logs") logs = self.get_recent_server_logs() # Check for OpenRouter API calls openrouter_api_logs = [ line for line in logs.split("\n") if "openrouter" in line.lower() and ("API" in line or "request" in line) ] # Check for model resolution through OpenRouter openrouter_model_logs = [ line for line in logs.split("\n") if "openrouter" in line.lower() and ("o3" in line or "model" in line) ] # Check for successful responses openrouter_response_logs = [ line for line in logs.split("\n") if "openrouter" in line.lower() and "response" in line ] self.logger.info(f" OpenRouter API logs: {len(openrouter_api_logs)}") self.logger.info(f" OpenRouter model logs: {len(openrouter_model_logs)}") self.logger.info(f" OpenRouter response logs: {len(openrouter_response_logs)}") # Success criteria for OpenRouter openrouter_used = len(openrouter_api_logs) >= 3 or len(openrouter_model_logs) >= 3 all_calls_succeeded = response1 and response2 and response3 success_criteria = [ ("All O3 model calls succeeded", all_calls_succeeded), ("OpenRouter provider was used", openrouter_used), ] passed_criteria = sum(1 for _, passed in success_criteria if passed) self.logger.info(f" Success criteria met: {passed_criteria}/{len(success_criteria)}") for criterion, passed in success_criteria: status = "✅" if passed else "❌" self.logger.info(f" {status} {criterion}") if passed_criteria == len(success_criteria): self.logger.info(" ✅ O3 model selection via OpenRouter passed") return True else: self.logger.error(" ❌ O3 model selection via OpenRouter failed") return False except Exception as e: self.logger.error(f"OpenRouter O3 test failed: {e}") return False finally: self.cleanup_test_files() def main(): """Run the O3 model selection tests""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = O3ModelSelectionTest(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_o3_pro_expensive.py ================================================ #!/usr/bin/env python3 """ O3-Pro Expensive Model Test ⚠️ WARNING: This test uses o3-pro which is EXTREMELY EXPENSIVE! ⚠️ This test is intentionally NOT added to TEST_REGISTRY to prevent accidental execution. It can only be run manually using: python communication_simulator_test.py --individual o3_pro_expensive Tests that o3-pro model: 1. Uses the correct /v1/responses endpoint (not /v1/chat/completions) 2. Successfully completes a chat call 3. Returns properly formatted response """ from .base_test import BaseSimulatorTest class O3ProExpensiveTest(BaseSimulatorTest): """Test o3-pro model basic functionality - EXPENSIVE, manual only""" @property def test_name(self) -> str: return "o3_pro_expensive" @property def test_description(self) -> str: return "⚠️ EXPENSIVE O3-Pro basic validation (manual only)" def run_test(self) -> bool: """Test o3-pro model with endpoint verification - EXPENSIVE!""" try: self.logger.warning("⚠️ ⚠️ ⚠️ EXPENSIVE TEST - O3-PRO COSTS ~$15-60 PER 1K TOKENS! ⚠️ ⚠️ ⚠️") self.logger.info("Test: O3-Pro endpoint and functionality test") # First, verify we're hitting the right endpoint by checking logs self.logger.info("Step 1: Testing o3-pro with chat tool") # One simple chat call response, tool_result = self.call_mcp_tool( "chat", { "prompt": "What is 2 + 2?", "model": "o3-pro", "temperature": 1.0, }, ) if not response: self.logger.error("❌ O3-Pro chat call failed - no response") if tool_result and "error" in tool_result: error_msg = tool_result["error"] self.logger.error(f"Error details: {error_msg}") # Check if it's the endpoint error we're trying to fix if "v1/responses" in str(error_msg) and "v1/chat/completions" in str(error_msg): self.logger.error( "❌ ENDPOINT BUG DETECTED: o3-pro is trying to use chat/completions instead of responses endpoint!" ) return False # Check the metadata to verify endpoint was used if tool_result and isinstance(tool_result, dict): metadata = tool_result.get("metadata", {}) endpoint_used = metadata.get("endpoint", "unknown") if endpoint_used == "responses": self.logger.info("✅ Correct endpoint used: /v1/responses") else: self.logger.warning(f"⚠️ Endpoint used: {endpoint_used} (expected: responses)") # Verify the response content if response and "4" in str(response): self.logger.info("✅ O3-Pro response is mathematically correct") else: self.logger.warning(f"⚠️ Unexpected response: {response}") self.logger.info("✅ O3-Pro test completed successfully") self.logger.warning("💰 Test completed - check your billing!") return True except Exception as e: self.logger.error(f"O3-Pro test failed with exception: {e}") # Log the full error for debugging endpoint issues import traceback self.logger.error(f"Full traceback: {traceback.format_exc()}") return False def main(): """Run the O3-Pro expensive test""" import sys print("⚠️ ⚠️ ⚠️ WARNING: This test uses O3-PRO which is EXTREMELY EXPENSIVE! ⚠️ ⚠️ ⚠️") print("O3-Pro can cost $15-60 per 1K tokens!") print("This is a MINIMAL test but may still cost $5-15!") print() response = input("Are you absolutely sure you want to run this expensive test? Type 'YES_I_UNDERSTAND_THE_COST': ") if response != "YES_I_UNDERSTAND_THE_COST": print("❌ Test cancelled") sys.exit(1) print("💰 Running minimal O3-Pro test...") verbose = "--verbose" in sys.argv or "-v" in sys.argv test = O3ProExpensiveTest(verbose=verbose) success = test.run_test() if success: print("✅ O3-Pro test completed successfully") print("💰 Don't forget to check your billing!") else: print("❌ O3-Pro test failed") sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_ollama_custom_url.py ================================================ #!/usr/bin/env python3 """ Ollama Custom URL Test Tests custom API endpoint functionality with Ollama-style local models, including: - Basic chat with custom model via local endpoint - File analysis with local model - Conversation continuation with custom provider - Model alias resolution for local models """ from .base_test import BaseSimulatorTest class OllamaCustomUrlTest(BaseSimulatorTest): """Test Ollama custom URL functionality""" @property def test_name(self) -> str: return "ollama_custom_url" @property def test_description(self) -> str: return "Ollama custom URL endpoint functionality" def run_test(self) -> bool: """Test Ollama custom URL functionality""" try: self.logger.info("Test: Ollama custom URL functionality") # Check if custom URL is configured import os custom_url = os.environ.get("CUSTOM_API_URL") if not custom_url: self.logger.warning("CUSTOM_API_URL not set, skipping Ollama test") self.logger.info("To enable this test, add to .env file:") self.logger.info("CUSTOM_API_URL=http://localhost:11434/v1") self.logger.info("CUSTOM_API_KEY=") return True # Skip gracefully self.logger.info(f"Testing with custom URL: {custom_url}") # Setup test files self.setup_test_files() # Test 1: Basic chat with local model self.logger.info(" 1.1: Basic chat with local model") response1, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Hello! Can you introduce yourself and tell me what model you are? Keep your response brief.", "model": "llama3.2", # Use exact Ollama model name }, ) if not self.validate_successful_response(response1, "local model chat"): return False self.logger.info(f" ✅ Local model responded with continuation_id: {continuation_id}") # Test 2: File analysis with local model using a specific Ollama-related file self.logger.info(" 1.2: File analysis with local model") # Create a simple, clear file that shouldn't require clarification ollama_test_content = '''""" Ollama API Client Test A simple test client for connecting to Ollama API endpoints """ import requests import json class OllamaClient: """Simple client for Ollama API""" def __init__(self, base_url="http://localhost:11434"): self.base_url = base_url def list_models(self): """List available models""" response = requests.get(f"{self.base_url}/api/tags") return response.json() def generate(self, model, prompt): """Generate text using a model""" data = { "model": model, "prompt": prompt, "stream": False } response = requests.post(f"{self.base_url}/api/generate", json=data) return response.json() if __name__ == "__main__": client = OllamaClient() models = client.list_models() print(f"Available models: {len(models['models'])}") ''' # Create the test file ollama_test_file = self.create_additional_test_file("ollama_client.py", ollama_test_content) response2, _ = self.call_mcp_tool( "analyze", { "absolute_file_paths": [ollama_test_file], "prompt": "Analyze this Ollama client code. What does this code do and what are its main functions?", "model": "llama3.2", }, ) if not self.validate_successful_response(response2, "local model file analysis", files_provided=True): return False self.logger.info(" ✅ Local model analyzed file successfully") # Test 3: Continue conversation with local model if continuation_id: self.logger.info(" 1.3: Continue conversation with local model") response3, _ = self.call_mcp_tool( "chat", { "prompt": "Thanks for the introduction! I just analyzed an Ollama client Python file. Can you suggest one improvement for writing better API client code in general?", "continuation_id": continuation_id, "model": "llama3.2", }, ) if not self.validate_successful_response(response3, "local model conversation continuation"): return False self.logger.info(" ✅ Conversation continuation with local model working") # Test 4: Test alternative local model aliases self.logger.info(" 1.4: Test alternative local model aliases") response4, _ = self.call_mcp_tool( "chat", { "prompt": "Quick test with alternative alias. Say 'Local model working' if you can respond.", "model": "llama3.2", # Alternative alias }, ) if not self.validate_successful_response(response4, "alternative local model alias"): return False self.logger.info(" ✅ Alternative local model alias working") # Test 5: Test direct model name (if applicable) self.logger.info(" 1.5: Test direct model name") response5, _ = self.call_mcp_tool( "chat", { "prompt": "Final test with direct model name. Respond briefly.", "model": "llama3.2", # Direct model name }, ) if not self.validate_successful_response(response5, "direct model name"): return False self.logger.info(" ✅ Direct model name working") self.logger.info(" ✅ All Ollama custom URL tests passed") return True except Exception as e: self.logger.error(f"Ollama custom URL test failed: {e}") return False finally: self.cleanup_test_files() def validate_successful_response(self, response: str, test_name: str, files_provided: bool = False) -> bool: """Validate that the response indicates success, not an error Args: response: The response text to validate test_name: Name of the test for logging files_provided: Whether actual files were provided to the tool """ if not response: self.logger.error(f"No response received for {test_name}") self._check_server_logs_for_errors() return False # Check for common error indicators error_indicators = [ "OpenRouter API error", "is not a valid model ID", "API key not found", "Connection error", "connection refused", "network is unreachable", "timeout", "error 404", "error 400", "error 401", "error 403", "error 500", "status code 404", "status code 400", "status code 401", "status code 403", "status code 500", "status: error", ] # Special handling for clarification requests from local models if "files_required_to_continue" in response.lower(): if files_provided: # If we provided actual files, clarification request is a FAILURE self.logger.error( f"❌ Local model requested clarification for {test_name} despite being provided with actual files" ) self.logger.debug(f"Clarification response: {response[:200]}...") return False else: # If no files were provided, clarification request is acceptable self.logger.info( f"✅ Local model requested clarification for {test_name} - valid when no files provided" ) self.logger.debug(f"Clarification response: {response[:200]}...") return True # Check for SSRF security restriction - this is expected for local URLs if "restricted IP address" in response and "security risk (SSRF)" in response: self.logger.info( f"✅ Custom URL routing working - {test_name} correctly attempted to connect to custom API" ) self.logger.info(" (Connection blocked by SSRF protection, which is expected for local URLs)") return True response_lower = response.lower() for error in error_indicators: if error.lower() in response_lower: self.logger.error(f"Error detected in {test_name}: {error}") self.logger.debug(f"Full response: {response}") self._check_server_logs_for_errors() return False # Response should be substantial (more than just a few words) if len(response.strip()) < 10: self.logger.error(f"Response too short for {test_name}: {response}") self._check_server_logs_for_errors() return False # Verify this looks like a real AI response, not just an error message if not self._validate_ai_response_content(response): self.logger.error(f"Response doesn't look like valid AI output for {test_name}") self._check_server_logs_for_errors() return False self.logger.debug(f"Successful response for {test_name}: {response[:100]}...") return True def _validate_ai_response_content(self, response: str) -> bool: """Validate that response appears to be legitimate AI output""" if not response: return False response_lower = response.lower() # Check for indicators this is a real AI response positive_indicators = [ "i am", "i'm", "i can", "i'll", "i would", "i think", "this code", "this function", "this file", "this configuration", "hello", "hi", "yes", "sure", "certainly", "of course", "analysis", "analyze", "review", "suggestion", "improvement", "here", "below", "above", "following", "based on", "python", "code", "function", "class", "variable", "llama", "model", "assistant", "ai", ] # Response should contain at least some AI-like language ai_indicators_found = sum(1 for indicator in positive_indicators if indicator in response_lower) if ai_indicators_found < 2: self.logger.warning(f"Response lacks AI-like indicators: {response[:200]}...") return False return True def _check_server_logs_for_errors(self): """Check server logs for any error messages that might explain failures""" try: # Get recent logs from the log file log_file_path = "logs/mcp_server.log" with open(log_file_path) as f: lines = f.readlines() recent_logs = lines[-50:] # Last 50 lines if recent_logs: self.logger.info("Recent server logs:") for line in recent_logs[-10:]: # Last 10 lines if line.strip(): self.logger.info(f" {line.strip()}") except Exception as e: self.logger.debug(f"Failed to check server logs: {e}") def validate_local_model_response(self, response: str) -> bool: """Validate that response appears to come from a local model""" if not response: return False # Basic validation - response should be non-empty and reasonable response_lower = response.lower() # Check for some indicators this might be from a local model # (This is heuristic - local models often mention their nature) local_indicators = ["llama", "local", "assistant", "ai", "model", "help"] # At least response should be meaningful text return len(response.strip()) > 10 and any(indicator in response_lower for indicator in local_indicators) ================================================ FILE: simulator_tests/test_openrouter_fallback.py ================================================ #!/usr/bin/env python3 """ OpenRouter Fallback Test Tests that verify the system correctly falls back to OpenRouter when: - Only OPENROUTER_API_KEY is configured - Native models (flash, pro) are requested but map to OpenRouter equivalents - Auto mode correctly selects OpenRouter models """ from .base_test import BaseSimulatorTest class OpenRouterFallbackTest(BaseSimulatorTest): """Test OpenRouter fallback behavior when it's the only provider""" @property def test_name(self) -> str: return "openrouter_fallback" @property def test_description(self) -> str: return "OpenRouter fallback behavior when only provider" def run_test(self) -> bool: """Test OpenRouter fallback behavior""" try: self.logger.info("Test: OpenRouter fallback behavior when only provider available") # Check if ONLY OpenRouter API key is configured (this is a fallback test) import os has_openrouter = bool(os.environ.get("OPENROUTER_API_KEY")) has_gemini = bool(os.environ.get("GEMINI_API_KEY")) has_openai = bool(os.environ.get("OPENAI_API_KEY")) if not has_openrouter: self.logger.info(" ⚠️ OpenRouter API key not configured - skipping test") self.logger.info(" ℹ️ This test requires OPENROUTER_API_KEY to be set in .env") return True # Return True to indicate test is skipped, not failed if has_gemini or has_openai: self.logger.info(" ⚠️ Other API keys configured - this is not a fallback scenario") self.logger.info(" ℹ️ This test requires ONLY OpenRouter to be configured (no Gemini/OpenAI keys)") self.logger.info(" ℹ️ Current setup has multiple providers, so fallback behavior doesn't apply") return True # Return True to indicate test is skipped, not failed # Setup test files self.setup_test_files() # Test 1: Auto mode should work with OpenRouter self.logger.info(" 1: Testing auto mode with OpenRouter as only provider") response1, continuation_id = self.call_mcp_tool( "chat", { "prompt": "What is 2 + 2? Give a brief answer.", # No model specified - should use auto mode "temperature": 0.1, }, ) if not response1: self.logger.error(" ❌ Auto mode with OpenRouter failed") return False self.logger.info(" ✅ Auto mode call completed with OpenRouter") # Test 2: Flash model should map to OpenRouter equivalent self.logger.info(" 2: Testing flash model mapping to OpenRouter") # Use codereview tool to test a different tool type test_code = """def calculate_sum(numbers): total = 0 for num in numbers: total += num return total""" test_file = self.create_additional_test_file("sum_function.py", test_code) response2, _ = self.call_mcp_tool( "codereview", { "step": "Quick review of this sum function for quality and potential issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting code review of sum function", "relevant_files": [test_file], "model": "flash", "temperature": 0.1, }, ) if not response2: self.logger.error(" ❌ Flash model mapping to OpenRouter failed") return False self.logger.info(" ✅ Flash model successfully mapped to OpenRouter") # Test 3: Pro model should map to OpenRouter equivalent self.logger.info(" 3: Testing pro model mapping to OpenRouter") response3, _ = self.call_mcp_tool( "analyze", { "step": "Analyze the structure of this Python code", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting code structure analysis", "relevant_files": [self.test_files["python"]], "model": "pro", "temperature": 0.1, }, ) if not response3: self.logger.error(" ❌ Pro model mapping to OpenRouter failed") return False self.logger.info(" ✅ Pro model successfully mapped to OpenRouter") # Test 4: Debug tool with OpenRouter self.logger.info(" 4: Testing debug tool with OpenRouter") response4, _ = self.call_mcp_tool( "debug", { "step": "Why might a function return None instead of a value?", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting debug investigation of None return values", "model": "flash", # Should map to OpenRouter "temperature": 0.1, }, ) if not response4: self.logger.error(" ❌ Debug tool with OpenRouter failed") return False self.logger.info(" ✅ Debug tool working with OpenRouter") # Test 5: Validate logs show OpenRouter is being used self.logger.info(" 5: Validating OpenRouter is the active provider") logs = self.get_recent_server_logs() # Check for provider fallback logs fallback_logs = [ line for line in logs.split("\n") if "No Gemini API key found" in line or "No OpenAI API key found" in line or "Only OpenRouter available" in line or "Using OpenRouter" in line ] # Check for OpenRouter provider initialization provider_logs = [ line for line in logs.split("\n") if "OpenRouter provider" in line or "OpenRouterProvider" in line or "openrouter.ai/api/v1" in line ] # Check for model resolution through OpenRouter model_resolution_logs = [ line for line in logs.split("\n") if ("Resolved model" in line and "via OpenRouter" in line) or ("Model alias" in line and "resolved to" in line) or ("flash" in line and "gemini-flash" in line) or ("pro" in line and "gemini-pro" in line) ] # Log findings self.logger.info(f" Fallback indication logs: {len(fallback_logs)}") self.logger.info(f" OpenRouter provider logs: {len(provider_logs)}") self.logger.info(f" Model resolution logs: {len(model_resolution_logs)}") # Sample logs for debugging if self.verbose: if fallback_logs: self.logger.debug(" 📋 Sample fallback logs:") for log in fallback_logs[:3]: self.logger.debug(f" {log}") if provider_logs: self.logger.debug(" 📋 Sample provider logs:") for log in provider_logs[:3]: self.logger.debug(f" {log}") # Success criteria openrouter_active = len(provider_logs) > 0 models_resolved = len(model_resolution_logs) > 0 all_tools_worked = True # We checked this above success_criteria = [ ("OpenRouter provider active", openrouter_active), ("Models resolved through OpenRouter", models_resolved), ("All tools worked with OpenRouter", all_tools_worked), ] passed_criteria = sum(1 for _, passed in success_criteria if passed) self.logger.info(f" Success criteria met: {passed_criteria}/{len(success_criteria)}") for criterion, passed in success_criteria: status = "✅" if passed else "❌" self.logger.info(f" {status} {criterion}") if passed_criteria >= 2: # At least 2 out of 3 criteria self.logger.info(" ✅ OpenRouter fallback test passed") return True else: self.logger.error(" ❌ OpenRouter fallback test failed") return False except Exception as e: self.logger.error(f"OpenRouter fallback test failed: {e}") return False finally: self.cleanup_test_files() def main(): """Run the OpenRouter fallback tests""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = OpenRouterFallbackTest(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_openrouter_models.py ================================================ #!/usr/bin/env python3 """ OpenRouter Model Tests Tests that verify OpenRouter functionality including: - Model alias resolution (flash, pro, o3, etc. map to OpenRouter equivalents) - Multiple OpenRouter models work correctly - Conversation continuity works with OpenRouter models - Error handling when models are not available """ from .base_test import BaseSimulatorTest class OpenRouterModelsTest(BaseSimulatorTest): """Test OpenRouter model functionality and alias mapping""" @property def test_name(self) -> str: return "openrouter_models" @property def test_description(self) -> str: return "OpenRouter model functionality and alias mapping" def run_test(self) -> bool: """Test OpenRouter model functionality""" try: self.logger.info("Test: OpenRouter model functionality and alias mapping") # Check if OpenRouter API key is configured import os has_openrouter = bool(os.environ.get("OPENROUTER_API_KEY")) if not has_openrouter: self.logger.info(" ⚠️ OpenRouter API key not configured - skipping test") self.logger.info(" ℹ️ This test requires OPENROUTER_API_KEY to be set in .env") return True # Return True to indicate test is skipped, not failed # Setup test files for later use self.setup_test_files() # Test 1: Flash alias mapping to OpenRouter self.logger.info(" 1: Testing 'flash' alias (should map to google/gemini-2.5-flash)") response1, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from Flash model!' and nothing else.", "model": "flash", "temperature": 0.1, }, ) if not response1: self.logger.error(" ❌ Flash alias test failed") return False self.logger.info(" ✅ Flash alias call completed") if continuation_id: self.logger.info(f" ✅ Got continuation_id: {continuation_id}") # Test 2: Pro alias mapping to OpenRouter self.logger.info(" 2: Testing 'pro' alias (should map to google/gemini-2.5-pro)") response2, _ = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from Pro model!' and nothing else.", "model": "pro", "temperature": 0.1, }, ) if not response2: self.logger.error(" ❌ Pro alias test failed") return False self.logger.info(" ✅ Pro alias call completed") # Test 3: O3 alias mapping to OpenRouter (should map to openai/gpt-4o) self.logger.info(" 3: Testing 'o3' alias (should map to openai/gpt-4o)") response3, _ = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from O3 model!' and nothing else.", "model": "o3", "temperature": 0.1, }, ) if not response3: self.logger.error(" ❌ O3 alias test failed") return False self.logger.info(" ✅ O3 alias call completed") # Test 4: Direct OpenRouter model name self.logger.info(" 4: Testing direct OpenRouter model name (anthropic/claude-3-haiku)") response4, _ = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from Claude Haiku!' and nothing else.", "model": "anthropic/claude-3-haiku", "temperature": 0.1, }, ) if not response4: self.logger.error(" ❌ Direct OpenRouter model test failed") return False self.logger.info(" ✅ Direct OpenRouter model call completed") # Test 5: OpenRouter alias from config self.logger.info(" 5: Testing OpenRouter alias from config ('opus' -> anthropic/claude-opus-4)") response5, _ = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from Opus!' and nothing else.", "model": "opus", "temperature": 0.1, }, ) if not response5: self.logger.error(" ❌ OpenRouter alias test failed") return False self.logger.info(" ✅ OpenRouter alias call completed") # Test 6: Conversation continuity with OpenRouter models self.logger.info(" 6: Testing conversation continuity with OpenRouter") response6, new_continuation_id = self.call_mcp_tool( "chat", { "prompt": "Remember this number: 42. What number did I just tell you?", "model": "sonnet", # Claude Sonnet via OpenRouter "temperature": 0.1, }, ) if not response6 or not new_continuation_id: self.logger.error(" ❌ Failed to start conversation with continuation_id") return False # Continue the conversation response7, _ = self.call_mcp_tool( "chat", { "prompt": "What was the number I told you earlier?", "model": "sonnet", "continuation_id": new_continuation_id, "temperature": 0.1, }, ) if not response7: self.logger.error(" ❌ Failed to continue conversation") return False # Check if the model remembered the number if "42" in response7: self.logger.info(" ✅ Conversation continuity working with OpenRouter") else: self.logger.warning(" ⚠️ Model may not have remembered the number") # Test 7: Validate OpenRouter API usage from logs self.logger.info(" 7: Validating OpenRouter API usage in logs") logs = self.get_recent_server_logs() # Check for OpenRouter API calls openrouter_logs = [line for line in logs.split("\n") if "openrouter" in line.lower()] openrouter_api_logs = [line for line in logs.split("\n") if "openrouter.ai/api/v1" in line] # Check for specific model mappings flash_mapping_logs = [ line for line in logs.split("\n") if ("flash" in line and "google/gemini-flash" in line) or ("Resolved model" in line and "google/gemini-flash" in line) ] pro_mapping_logs = [ line for line in logs.split("\n") if ("pro" in line and "google/gemini-pro" in line) or ("Resolved model" in line and "google/gemini-pro" in line) ] # Log findings self.logger.info(f" OpenRouter-related logs: {len(openrouter_logs)}") self.logger.info(f" OpenRouter API logs: {len(openrouter_api_logs)}") self.logger.info(f" Flash mapping logs: {len(flash_mapping_logs)}") self.logger.info(f" Pro mapping logs: {len(pro_mapping_logs)}") # Sample log output for debugging if self.verbose and openrouter_logs: self.logger.debug(" 📋 Sample OpenRouter logs:") for log in openrouter_logs[:5]: self.logger.debug(f" {log}") # Success criteria openrouter_api_used = len(openrouter_api_logs) > 0 models_mapped = len(flash_mapping_logs) > 0 or len(pro_mapping_logs) > 0 success_criteria = [ ("OpenRouter API calls made", openrouter_api_used), ("Model aliases mapped correctly", models_mapped), ("All model calls succeeded", True), # We already checked this above ] passed_criteria = sum(1 for _, passed in success_criteria if passed) self.logger.info(f" Success criteria met: {passed_criteria}/{len(success_criteria)}") for criterion, passed in success_criteria: status = "✅" if passed else "❌" self.logger.info(f" {status} {criterion}") if passed_criteria >= 2: # At least 2 out of 3 criteria self.logger.info(" ✅ OpenRouter model tests passed") return True else: self.logger.error(" ❌ OpenRouter model tests failed") return False except Exception as e: self.logger.error(f"OpenRouter model test failed: {e}") return False finally: self.cleanup_test_files() def main(): """Run the OpenRouter model tests""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = OpenRouterModelsTest(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_per_tool_deduplication.py ================================================ #!/usr/bin/env python3 """ Per-Tool File Deduplication Test Tests file deduplication for each individual MCP tool to ensure that files are properly deduplicated within single-tool conversations. Validates that: 1. Files are embedded only once in conversation history 2. Continuation calls don't re-read existing files 3. New files are still properly embedded 4. Server logs show deduplication behavior """ import os from .conversation_base_test import ConversationBaseTest class PerToolDeduplicationTest(ConversationBaseTest): """Test file deduplication for each individual tool""" @property def test_name(self) -> str: return "per_tool_deduplication" @property def test_description(self) -> str: return "File deduplication for individual tools" # create_additional_test_file method now inherited from base class def run_test(self) -> bool: """Test file deduplication with realistic precommit/codereview workflow""" try: self.logger.info("📄 Test: Simplified file deduplication with precommit/codereview workflow") # Setup test environment for conversation testing self.setUp() # Setup test files self.setup_test_files() # Create a short dummy file for quick testing in the current repo dummy_content = """def add(a, b): return a + b # Missing type hints def divide(x, y): return x / y # No zero check """ # Create the file in the current git repo directory to make it show up in git status dummy_file_path = os.path.join(os.getcwd(), "dummy_code.py") with open(dummy_file_path, "w") as f: f.write(dummy_content) # Get timestamp for log filtering import datetime start_time = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S") # Step 1: precommit tool with dummy file (low thinking mode) self.logger.info(" Step 1: precommit tool with dummy file") precommit_params = { "step": "Initial analysis of dummy_code.py for commit readiness. Please give me a quick one line reply.", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Starting pre-commit validation of dummy_code.py", "path": os.getcwd(), # Use current working directory as the git repo path "relevant_files": [dummy_file_path], "thinking_mode": "low", "model": "flash", } response1, continuation_id = self.call_mcp_tool("precommit", precommit_params) if not response1: self.logger.error(" ❌ Step 1: precommit tool failed") return False if not continuation_id: self.logger.error(" ❌ Step 1: precommit tool didn't provide continuation_id") return False # Validate continuation_id format (should be UUID) if len(continuation_id) < 32: self.logger.error(f" ❌ Step 1: Invalid continuation_id format: {continuation_id}") return False self.logger.info(f" ✅ Step 1: precommit completed with continuation_id: {continuation_id[:8]}...") # Step 2: codereview tool with same file (NO continuation - fresh conversation) self.logger.info(" Step 2: codereview tool with same file (fresh conversation)") codereview_params = { "step": "Initial code review of dummy_code.py for quality and best practices. Please give me a quick one line reply.", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting code review of dummy_code.py", "relevant_files": [dummy_file_path], "thinking_mode": "low", "model": "flash", } response2, _ = self.call_mcp_tool("codereview", codereview_params) if not response2: self.logger.error(" ❌ Step 2: codereview tool failed") return False self.logger.info(" ✅ Step 2: codereview completed (fresh conversation)") # Step 3: Create new file and continue with precommit self.logger.info(" Step 3: precommit continuation with old + new file") new_file_content = """def multiply(x, y): return x * y def subtract(a, b): return a - b """ # Create another temp file in the current repo for git changes new_file_path = os.path.join(os.getcwd(), "new_feature.py") with open(new_file_path, "w") as f: f.write(new_file_content) # Continue precommit with both files continue_params = { "continuation_id": continuation_id, "step": "Continue analysis with new_feature.py added. Please give me a quick one line reply about both files.", "step_number": 2, "total_steps": 2, "next_step_required": False, "findings": "Continuing pre-commit validation with both dummy_code.py and new_feature.py", "path": os.getcwd(), # Use current working directory as the git repo path "relevant_files": [dummy_file_path, new_file_path], # Old + new file "thinking_mode": "low", "model": "flash", } response3, _ = self.call_mcp_tool("precommit", continue_params) if not response3: self.logger.error(" ❌ Step 3: precommit continuation failed") return False self.logger.info(" ✅ Step 3: precommit continuation completed") # Validate results in server logs self.logger.info(" 📋 Validating conversation history and file deduplication...") logs = self.get_server_logs_since(start_time) # Check for conversation history building conversation_logs = [ line for line in logs.split("\n") if "conversation" in line.lower() or "history" in line.lower() ] # Check for file embedding/deduplication embedding_logs = [ line for line in logs.split("\n") if "[FILE_PROCESSING]" in line or "embedding" in line.lower() or "[FILES]" in line ] # Check for continuation evidence continuation_logs = [ line for line in logs.split("\n") if "continuation" in line.lower() or continuation_id[:8] in line ] # Check for both files mentioned dummy_file_mentioned = any("dummy_code.py" in line for line in logs.split("\n")) new_file_mentioned = any("new_feature.py" in line for line in logs.split("\n")) # Print diagnostic information self.logger.info(f" Conversation logs found: {len(conversation_logs)}") self.logger.info(f" File embedding logs found: {len(embedding_logs)}") self.logger.info(f" Continuation logs found: {len(continuation_logs)}") self.logger.info(f" Dummy file mentioned: {dummy_file_mentioned}") self.logger.info(f" New file mentioned: {new_file_mentioned}") if self.verbose: self.logger.debug(" 📋 Sample embedding logs:") for log in embedding_logs[:5]: # Show first 5 if log.strip(): self.logger.debug(f" {log.strip()}") self.logger.debug(" 📋 Sample continuation logs:") for log in continuation_logs[:3]: # Show first 3 if log.strip(): self.logger.debug(f" {log.strip()}") # Determine success criteria success_criteria = [ len(embedding_logs) > 0, # File embedding occurred len(continuation_logs) > 0, # Continuation worked dummy_file_mentioned, # Original file processed new_file_mentioned, # New file processed ] passed_criteria = sum(success_criteria) total_criteria = len(success_criteria) self.logger.info(f" Success criteria met: {passed_criteria}/{total_criteria}") if passed_criteria == total_criteria: # All criteria must pass self.logger.info(" ✅ File deduplication workflow test: PASSED") return True else: self.logger.warning(" ⚠️ File deduplication workflow test: FAILED") self.logger.warning(" 💡 Check server logs for detailed file embedding and continuation activity") return False except Exception as e: self.logger.error(f"File deduplication workflow test failed: {e}") return False finally: # Clean up temp files created in current repo temp_files = ["dummy_code.py", "new_feature.py"] for temp_file in temp_files: temp_path = os.path.join(os.getcwd(), temp_file) if os.path.exists(temp_path): os.remove(temp_path) self.logger.debug(f"Removed temp file: {temp_path}") self.cleanup_test_files() ================================================ FILE: simulator_tests/test_planner_continuation_history.py ================================================ #!/usr/bin/env python3 """ Planner Continuation History Test Tests the planner tool's continuation history building across multiple completed planning sessions: - Multiple completed planning sessions in sequence - History context loading for new planning sessions - Proper context building with multiple completed plans - Context accumulation and retrieval """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class PlannerContinuationHistoryTest(ConversationBaseTest): """Test planner tool's continuation history building across multiple completed sessions""" @property def test_name(self) -> str: return "planner_continuation_history" @property def test_description(self) -> str: return "Planner tool continuation history building across multiple completed planning sessions" def run_test(self) -> bool: """Test planner continuation history building across multiple completed sessions""" # Set up the test environment self.setUp() try: self.logger.info("Test: Planner continuation history validation") # Test 1: Complete first planning session (microservices migration) if not self._test_first_planning_session(): return False # Test 2: Complete second planning session with context from first if not self._test_second_planning_session(): return False # Test 3: Complete third planning session with context from both previous if not self._test_third_planning_session(): return False # Test 4: Validate context accumulation across all sessions if not self._test_context_accumulation(): return False self.logger.info(" ✅ All planner continuation history tests passed") return True except Exception as e: self.logger.error(f"Planner continuation history test failed: {e}") return False def _test_first_planning_session(self) -> bool: """Complete first planning session - microservices migration""" try: self.logger.info(" 2.1: First planning session - Microservices Migration") # Step 1: Start migration planning self.logger.info(" 2.1.1: Start migration planning") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "I need to plan a microservices migration for our monolithic e-commerce platform. Let me analyze the current monolith structure.", "step_number": 1, "total_steps": 3, "next_step_required": True, }, ) if not response1 or not continuation_id: self.logger.error("Failed to start first planning session") return False # Step 2: Domain identification self.logger.info(" 2.1.2: Domain identification") response2, _ = self.call_mcp_tool( "planner", { "step": "I've identified key domains: User Management, Product Catalog, Order Processing, Payment, and Inventory. Each will become a separate microservice.", "step_number": 2, "total_steps": 3, "next_step_required": True, "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed step 2 of first planning session") return False # Step 3: Complete migration plan self.logger.info(" 2.1.3: Complete migration plan") response3, _ = self.call_mcp_tool( "planner", { "step": "Migration strategy: Phase 1 - Extract User Management service, Phase 2 - Product Catalog and Inventory services, Phase 3 - Order Processing and Payment services. Use API Gateway for service coordination.", "step_number": 3, "total_steps": 3, "next_step_required": False, # Complete the session "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to complete first planning session") return False # Validate completion response3_data = self._parse_planner_response(response3) if not response3_data.get("planning_complete"): self.logger.error("First planning session not marked as complete") return False if not response3_data.get("plan_summary"): self.logger.error("First planning session missing plan summary") return False self.logger.info(" ✅ First planning session completed successfully") # Store for next test self.first_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"First planning session test failed: {e}") return False def _test_second_planning_session(self) -> bool: """Complete second planning session with context from first""" try: self.logger.info(" 2.2: Second planning session - Database Strategy") # Step 1: Start database planning with previous context self.logger.info(" 2.2.1: Start database strategy with microservices context") response1, new_continuation_id = self.call_mcp_tool( "planner", { "step": "Now I need to plan the database strategy for the microservices architecture. I'll design how each service will manage its data.", "step_number": 1, "total_steps": 2, "next_step_required": True, "continuation_id": self.first_continuation_id, # Use first session's continuation_id }, ) if not response1 or not new_continuation_id: self.logger.error("Failed to start second planning session") return False # Validate context loading response1_data = self._parse_planner_response(response1) if "previous_plan_context" not in response1_data: self.logger.error("Second session should load context from first completed session") return False # Check context contains migration content context = response1_data["previous_plan_context"].lower() if "migration" not in context and "microservices" not in context: self.logger.error("Context should contain migration/microservices content from first session") return False self.logger.info(" ✅ Second session loaded context from first completed session") # Step 2: Complete database plan self.logger.info(" 2.2.2: Complete database strategy") response2, _ = self.call_mcp_tool( "planner", { "step": "Database strategy: Each microservice gets its own database (database-per-service pattern). Use event sourcing for cross-service communication and eventual consistency. Implement CQRS for read/write separation.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Complete the session "continuation_id": new_continuation_id, }, ) if not response2: self.logger.error("Failed to complete second planning session") return False # Validate completion response2_data = self._parse_planner_response(response2) if not response2_data.get("planning_complete"): self.logger.error("Second planning session not marked as complete") return False self.logger.info(" ✅ Second planning session completed successfully") # Store for next test self.second_continuation_id = new_continuation_id return True except Exception as e: self.logger.error(f"Second planning session test failed: {e}") return False def _test_third_planning_session(self) -> bool: """Complete third planning session with context from both previous""" try: self.logger.info(" 2.3: Third planning session - Deployment Strategy") # Step 1: Start deployment planning with accumulated context self.logger.info(" 2.3.1: Start deployment strategy with accumulated context") response1, new_continuation_id = self.call_mcp_tool( "planner", { "step": "Now I need to plan the deployment strategy that supports both the microservices architecture and the database strategy. I'll design the infrastructure and deployment pipeline.", "step_number": 1, "total_steps": 2, "next_step_required": True, "continuation_id": self.second_continuation_id, # Use second session's continuation_id }, ) if not response1 or not new_continuation_id: self.logger.error("Failed to start third planning session") return False # Validate context loading response1_data = self._parse_planner_response(response1) if "previous_plan_context" not in response1_data: self.logger.error("Third session should load context from previous completed sessions") return False # Check context contains content from most recent completed session context = response1_data["previous_plan_context"].lower() expected_terms = ["database", "event sourcing", "cqrs"] found_terms = [term for term in expected_terms if term in context] if len(found_terms) == 0: self.logger.error( f"Context should contain database strategy content from second session. Context: {context[:200]}..." ) return False self.logger.info(" ✅ Third session loaded context from most recent completed session") # Step 2: Complete deployment plan self.logger.info(" 2.3.2: Complete deployment strategy") response2, _ = self.call_mcp_tool( "planner", { "step": "Deployment strategy: Use Kubernetes for orchestration with Helm charts. Implement CI/CD pipeline with GitOps. Use service mesh (Istio) for traffic management, monitoring, and security. Deploy databases in separate namespaces with backup automation.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Complete the session "continuation_id": new_continuation_id, }, ) if not response2: self.logger.error("Failed to complete third planning session") return False # Validate completion response2_data = self._parse_planner_response(response2) if not response2_data.get("planning_complete"): self.logger.error("Third planning session not marked as complete") return False self.logger.info(" ✅ Third planning session completed successfully") # Store for final test self.third_continuation_id = new_continuation_id return True except Exception as e: self.logger.error(f"Third planning session test failed: {e}") return False def _test_context_accumulation(self) -> bool: """Test that context properly accumulates across multiple completed sessions""" try: self.logger.info(" 2.4: Testing context accumulation across all sessions") # Start a new planning session that should load context from the most recent completed session self.logger.info(" 2.4.1: Start monitoring planning with full context history") response1, _ = self.call_mcp_tool( "planner", { "step": "Finally, I need to plan the monitoring and observability strategy that works with the microservices, database, and deployment architecture.", "step_number": 1, "total_steps": 1, "next_step_required": False, "continuation_id": self.third_continuation_id, # Use third session's continuation_id }, ) if not response1: self.logger.error("Failed to start monitoring planning session") return False # Validate context loading response1_data = self._parse_planner_response(response1) if "previous_plan_context" not in response1_data: self.logger.error("Final session should load context from previous completed sessions") return False # Validate context contains most recent completed session content context = response1_data["previous_plan_context"].lower() # Should contain deployment strategy content (most recent) deployment_terms = ["kubernetes", "deployment", "istio", "gitops"] found_deployment_terms = [term for term in deployment_terms if term in context] if len(found_deployment_terms) == 0: self.logger.error(f"Context should contain deployment strategy content. Context: {context[:300]}...") return False self.logger.info(" ✅ Context accumulation working correctly") # Validate this creates a complete planning session if not response1_data.get("planning_complete"): self.logger.error("Final planning session should be marked as complete") return False self.logger.info(" ✅ Context accumulation test completed successfully") return True except Exception as e: self.logger.error(f"Context accumulation test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for planner-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from planner response specifically continuation_id = self._extract_planner_continuation_id(response_text) return response_text, continuation_id def _extract_planner_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from planner response""" try: # Parse the response - it's now direct JSON, not wrapped response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for planner continuation_id: {e}") return None def _parse_planner_response(self, response_text: str) -> dict: """Parse planner tool JSON response""" try: # Parse the response - it's now direct JSON, not wrapped return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse planner response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} ================================================ FILE: simulator_tests/test_planner_validation.py ================================================ #!/usr/bin/env python3 """ PlannerWorkflow Tool Validation Test Tests the planner tool's capabilities using the new workflow architecture. This validates that the new workflow-based implementation maintains all the functionality of the original planner tool while using the workflow pattern like the debug tool. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class PlannerValidationTest(ConversationBaseTest): """Test planner tool with new workflow architecture""" @property def test_name(self) -> str: return "planner_validation" @property def test_description(self) -> str: return "PlannerWorkflow tool validation with new workflow architecture" def run_test(self) -> bool: """Test planner tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: PlannerWorkflow tool validation (new architecture)") # Test 1: Single planning session with workflow architecture if not self._test_single_planning_session(): return False # Test 2: Planning with continuation using workflow if not self._test_planning_with_continuation(): return False # Test 3: Complex plan with deep thinking pauses if not self._test_complex_plan_deep_thinking(): return False # Test 4: Self-contained completion (no expert analysis) if not self._test_self_contained_completion(): return False # Test 5: Branching and revision with workflow if not self._test_branching_and_revision(): return False # Test 6: Workflow file context behavior if not self._test_workflow_file_context(): return False self.logger.info(" ✅ All planner validation tests passed") return True except Exception as e: self.logger.error(f"PlannerWorkflow validation test failed: {e}") return False def _test_single_planning_session(self) -> bool: """Test a complete planning session with workflow architecture""" try: self.logger.info(" 1.1: Testing single planning session with workflow") # Step 1: Start planning self.logger.info(" 1.1.1: Step 1 - Initial planning step") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "I need to plan a comprehensive API redesign for our legacy system. Let me start by analyzing the current state and identifying key requirements for the new API architecture.", "step_number": 1, "total_steps": 4, "next_step_required": True, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial planning response") return False # Parse and validate JSON response response1_data = self._parse_planner_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_planner for next_step_required=True if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_planner"): return False # Debug: Log the actual response structure to see what we're getting self.logger.debug(f"Response structure: {list(response1_data.keys())}") # Check workflow-specific response structure (more flexible) status_key = None for key in response1_data.keys(): if key.endswith("_status"): status_key = key break if not status_key: self.logger.error(f"Missing workflow status field in response: {list(response1_data.keys())}") return False self.logger.debug(f"Found status field: {status_key}") # Check required_actions for workflow guidance if not response1_data.get("required_actions"): self.logger.error("Missing required_actions in workflow response") return False self.logger.info(f" ✅ Step 1 successful with workflow, continuation_id: {continuation_id}") # Step 2: Continue planning self.logger.info(" 1.1.2: Step 2 - API domain analysis") response2, _ = self.call_mcp_tool( "planner", { "step": "After analyzing the current API, I can identify three main domains: User Management, Content Management, and Analytics. Let me design the new API structure with RESTful endpoints and proper versioning.", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "model": "flash", }, ) if not response2: self.logger.error("Failed to continue planning to step 2") return False response2_data = self._parse_planner_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_planner"): return False # Check step history tracking in workflow (more flexible) status_key = None for key in response2_data.keys(): if key.endswith("_status"): status_key = key break if status_key: workflow_status = response2_data.get(status_key, {}) step_history_length = workflow_status.get("step_history_length", 0) if step_history_length < 2: self.logger.error(f"Step history not properly tracked in workflow: {step_history_length}") return False self.logger.debug(f"Step history length: {step_history_length}") else: self.logger.warning("No workflow status found, skipping step history check") self.logger.info(" ✅ Step 2 successful with workflow tracking") # Step 3: Final step - should trigger completion self.logger.info(" 1.1.3: Step 3 - Final planning step") response3, _ = self.call_mcp_tool( "planner", { "step": "API redesign plan complete: Phase 1 - User Management API, Phase 2 - Content Management API, Phase 3 - Analytics API. Each phase includes proper authentication, rate limiting, and comprehensive documentation.", "step_number": 3, "total_steps": 3, # Adjusted total "next_step_required": False, # Final step - should complete without expert analysis "continuation_id": continuation_id, "model": "flash", }, ) if not response3: self.logger.error("Failed to complete planning session") return False response3_data = self._parse_planner_response(response3) if not response3_data: return False # Validate final response structure - should be self-contained completion if response3_data.get("status") != "planner_complete": self.logger.error(f"Expected status 'planner_complete', got '{response3_data.get('status')}'") return False if not response3_data.get("planning_complete"): self.logger.error("Expected planning_complete=true for final step") return False # Should NOT have expert_analysis (self-contained) if "expert_analysis" in response3_data: self.logger.error("PlannerWorkflow should be self-contained without expert analysis") return False # Check plan_summary exists if not response3_data.get("plan_summary"): self.logger.error("Missing plan_summary in final step") return False self.logger.info(" ✅ Planning session completed successfully with workflow architecture") # Store continuation_id for next test self.api_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single planning session test failed: {e}") return False def _test_planning_with_continuation(self) -> bool: """Test planning continuation with workflow architecture""" try: self.logger.info(" 1.2: Testing planning continuation with workflow") # Use continuation from previous test if available continuation_id = getattr(self, "api_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.2.0: Starting fresh planning session") response0, continuation_id = self.call_mcp_tool( "planner", { "step": "Planning API security strategy", "step_number": 1, "total_steps": 2, "next_step_required": True, "model": "flash", }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh planning session") return False # Test continuation step self.logger.info(" 1.2.1: Continue planning session") response1, _ = self.call_mcp_tool( "planner", { "step": "Building on the API redesign, let me now plan the security implementation with OAuth 2.0, API keys, and rate limiting strategies.", "step_number": 2, "total_steps": 2, "next_step_required": True, "continuation_id": continuation_id, "model": "flash", }, ) if not response1: self.logger.error("Failed to continue planning") return False response1_data = self._parse_planner_response(response1) if not response1_data: return False # Validate continuation behavior if not self._validate_step_response(response1_data, 2, 2, True, "pause_for_planner"): return False # Check that continuation_id is preserved if response1_data.get("continuation_id") != continuation_id: self.logger.error("Continuation ID not preserved in workflow") return False self.logger.info(" ✅ Planning continuation working with workflow") return True except Exception as e: self.logger.error(f"Planning continuation test failed: {e}") return False def _test_complex_plan_deep_thinking(self) -> bool: """Test complex plan with deep thinking pauses""" try: self.logger.info(" 1.3: Testing complex plan with deep thinking pauses") # Start complex plan (≥5 steps) - should trigger deep thinking self.logger.info(" 1.3.1: Step 1 of complex plan (should trigger deep thinking)") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "I need to plan a complete digital transformation for our enterprise organization, including cloud migration, process automation, and cultural change management.", "step_number": 1, "total_steps": 8, # Complex plan ≥5 steps "next_step_required": True, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start complex planning") return False response1_data = self._parse_planner_response(response1) if not response1_data: return False # Should trigger deep thinking pause for complex plan if response1_data.get("status") != "pause_for_deep_thinking": self.logger.error("Expected deep thinking pause for complex plan step 1") return False if not response1_data.get("thinking_required"): self.logger.error("Expected thinking_required=true for complex plan") return False # Check required thinking actions required_thinking = response1_data.get("required_thinking", []) if len(required_thinking) < 4: self.logger.error("Expected comprehensive thinking requirements for complex plan") return False # Check for deep thinking guidance in next_steps next_steps = response1_data.get("next_steps", "") if "MANDATORY" not in next_steps or "deep thinking" not in next_steps.lower(): self.logger.error("Expected mandatory deep thinking guidance") return False self.logger.info(" ✅ Complex plan step 1 correctly triggered deep thinking pause") # Step 2 of complex plan - should also trigger deep thinking self.logger.info(" 1.3.2: Step 2 of complex plan (should trigger deep thinking)") response2, _ = self.call_mcp_tool( "planner", { "step": "After deep analysis, I can see this transformation requires three parallel tracks: Technical Infrastructure, Business Process, and Human Capital. Let me design the coordination strategy.", "step_number": 2, "total_steps": 8, "next_step_required": True, "continuation_id": continuation_id, "model": "flash", }, ) if not response2: self.logger.error("Failed to continue complex planning") return False response2_data = self._parse_planner_response(response2) if not response2_data: return False # Step 2 should also trigger deep thinking for complex plans if response2_data.get("status") != "pause_for_deep_thinking": self.logger.error("Expected deep thinking pause for complex plan step 2") return False self.logger.info(" ✅ Complex plan step 2 correctly triggered deep thinking pause") # Step 4 of complex plan - should use normal flow (after step 3) self.logger.info(" 1.3.3: Step 4 of complex plan (should use normal flow)") response4, _ = self.call_mcp_tool( "planner", { "step": "Now moving to tactical planning: Phase 1 execution details with specific timelines and resource allocation for the technical infrastructure track.", "step_number": 4, "total_steps": 8, "next_step_required": True, "continuation_id": continuation_id, "model": "flash", }, ) if not response4: self.logger.error("Failed to continue to step 4") return False response4_data = self._parse_planner_response(response4) if not response4_data: return False # Step 4 should use normal flow (no more deep thinking pauses) if response4_data.get("status") != "pause_for_planner": self.logger.error("Expected normal planning flow for step 4") return False if response4_data.get("thinking_required"): self.logger.error("Step 4 should not require special thinking pause") return False self.logger.info(" ✅ Complex plan transitions to normal flow after step 3") return True except Exception as e: self.logger.error(f"Complex plan deep thinking test failed: {e}") return False def _test_self_contained_completion(self) -> bool: """Test self-contained completion without expert analysis""" try: self.logger.info(" 1.4: Testing self-contained completion") # Simple planning session that should complete without expert analysis self.logger.info(" 1.4.1: Simple planning session") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "Planning a simple website redesign with new color scheme and improved navigation.", "step_number": 1, "total_steps": 2, "next_step_required": True, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start simple planning") return False # Final step - should complete without expert analysis self.logger.info(" 1.4.2: Final step - self-contained completion") response2, _ = self.call_mcp_tool( "planner", { "step": "Website redesign plan complete: Phase 1 - Update color palette and typography, Phase 2 - Redesign navigation structure and user flows.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step "continuation_id": continuation_id, "model": "flash", }, ) if not response2: self.logger.error("Failed to complete simple planning") return False response2_data = self._parse_planner_response(response2) if not response2_data: return False # Validate self-contained completion if response2_data.get("status") != "planner_complete": self.logger.error("Expected self-contained completion status") return False # Should NOT call expert analysis if "expert_analysis" in response2_data: self.logger.error("PlannerWorkflow should not call expert analysis") return False # Should have planning_complete flag if not response2_data.get("planning_complete"): self.logger.error("Expected planning_complete=true") return False # Should have plan_summary if not response2_data.get("plan_summary"): self.logger.error("Expected plan_summary in completion") return False # Check completion instructions output = response2_data.get("output", {}) if not output.get("instructions"): self.logger.error("Missing output instructions for plan presentation") return False self.logger.info(" ✅ Self-contained completion working correctly") return True except Exception as e: self.logger.error(f"Self-contained completion test failed: {e}") return False def _test_branching_and_revision(self) -> bool: """Test branching and revision with workflow architecture""" try: self.logger.info(" 1.5: Testing branching and revision with workflow") # Start planning session for branching test self.logger.info(" 1.5.1: Start planning for branching test") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "Planning mobile app development strategy with different technology options to evaluate.", "step_number": 1, "total_steps": 4, "next_step_required": True, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start branching test") return False # Create branch self.logger.info(" 1.5.2: Create branch for React Native approach") response2, _ = self.call_mcp_tool( "planner", { "step": "Branch A: React Native approach - cross-platform development with shared codebase, faster development cycle, and consistent UI across platforms.", "step_number": 2, "total_steps": 4, "next_step_required": True, "is_branch_point": True, "branch_from_step": 1, "branch_id": "react-native", "continuation_id": continuation_id, "model": "flash", }, ) if not response2: self.logger.error("Failed to create branch") return False response2_data = self._parse_planner_response(response2) if not response2_data: return False # Validate branching in workflow metadata = response2_data.get("metadata", {}) if not metadata.get("is_branch_point"): self.logger.error("Branch point not recorded in workflow") return False if metadata.get("branch_id") != "react-native": self.logger.error("Branch ID not properly recorded") return False if "react-native" not in metadata.get("branches", []): self.logger.error("Branch not added to branches list") return False self.logger.info(" ✅ Branching working with workflow architecture") # Test revision self.logger.info(" 1.5.3: Test revision capability") response3, _ = self.call_mcp_tool( "planner", { "step": "Revision of step 2: After consideration, let me revise the React Native approach to include performance optimizations and native module integration for critical features.", "step_number": 3, "total_steps": 4, "next_step_required": True, "is_step_revision": True, "revises_step_number": 2, "continuation_id": continuation_id, "model": "flash", }, ) if not response3: self.logger.error("Failed to create revision") return False response3_data = self._parse_planner_response(response3) if not response3_data: return False # Validate revision in workflow metadata = response3_data.get("metadata", {}) if not metadata.get("is_step_revision"): self.logger.error("Step revision not recorded in workflow") return False if metadata.get("revises_step_number") != 2: self.logger.error("Revised step number not properly recorded") return False self.logger.info(" ✅ Revision working with workflow architecture") return True except Exception as e: self.logger.error(f"Branching and revision test failed: {e}") return False def _test_workflow_file_context(self) -> bool: """Test workflow file context behavior (should be minimal for planner)""" try: self.logger.info(" 1.6: Testing workflow file context behavior") # Planner typically doesn't use files, but test the workflow handles this correctly self.logger.info(" 1.6.1: Planning step with no files (normal case)") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "Planning data architecture for analytics platform.", "step_number": 1, "total_steps": 2, "next_step_required": True, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start workflow file context test") return False response1_data = self._parse_planner_response(response1) if not response1_data: return False # Planner workflow should not have file_context since it doesn't use files if "file_context" in response1_data: self.logger.info(" ℹ️ Workflow file context present but should be minimal for planner") # Final step self.logger.info(" 1.6.2: Final step (should complete without file embedding)") response2, _ = self.call_mcp_tool( "planner", { "step": "Data architecture plan complete with data lakes, processing pipelines, and analytics layers.", "step_number": 2, "total_steps": 2, "next_step_required": False, "continuation_id": continuation_id, "model": "flash", }, ) if not response2: self.logger.error("Failed to complete workflow file context test") return False response2_data = self._parse_planner_response(response2) if not response2_data: return False # Final step should complete self-contained if response2_data.get("status") != "planner_complete": self.logger.error("Expected self-contained completion for planner workflow") return False self.logger.info(" ✅ Workflow file context behavior appropriate for planner") return True except Exception as e: self.logger.error(f"Workflow file context test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for planner-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from planner response specifically continuation_id = self._extract_planner_continuation_id(response_text) return response_text, continuation_id def _extract_planner_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from planner response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for planner continuation_id: {e}") return None def _parse_planner_response(self, response_text: str) -> dict: """Parse planner tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse planner response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a planner step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check step_content exists if not response_data.get("step_content"): self.logger.error("Missing step_content in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False ================================================ FILE: simulator_tests/test_planner_validation_old.py ================================================ #!/usr/bin/env python3 """ Planner Tool Validation Test Tests the planner tool's sequential planning capabilities including: - Step-by-step planning with proper JSON responses - Continuation logic across planning sessions - Branching and revision capabilities - Previous plan context loading - Plan completion and summary storage """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class PlannerValidationTest(ConversationBaseTest): """Test planner tool's sequential planning and continuation features""" @property def test_name(self) -> str: return "planner_validation" @property def test_description(self) -> str: return "Planner tool sequential planning and continuation validation" def run_test(self) -> bool: """Test planner tool sequential planning capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: Planner tool validation") # Test 1: Single planning session with multiple steps if not self._test_single_planning_session(): return False # Test 2: Plan completion and continuation to new planning session if not self._test_plan_continuation(): return False # Test 3: Branching and revision capabilities if not self._test_branching_and_revision(): return False self.logger.info(" ✅ All planner validation tests passed") return True except Exception as e: self.logger.error(f"Planner validation test failed: {e}") return False def _test_single_planning_session(self) -> bool: """Test a complete planning session with multiple steps""" try: self.logger.info(" 1.1: Testing single planning session") # Step 1: Start planning self.logger.info(" 1.1.1: Step 1 - Initial planning step") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "I need to plan a microservices migration for our monolithic e-commerce platform. Let me start by understanding the current architecture and identifying the key business domains.", "step_number": 1, "total_steps": 5, "next_step_required": True, }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial planning response") return False # Parse and validate JSON response response1_data = self._parse_planner_response(response1) if not response1_data: return False # Validate step 1 response structure if not self._validate_step_response(response1_data, 1, 5, True, "planning_success"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Continue planning self.logger.info(" 1.1.2: Step 2 - Domain identification") response2, _ = self.call_mcp_tool( "planner", { "step": "Based on my analysis, I can identify the main business domains: User Management, Product Catalog, Order Processing, Payment, and Inventory. Let me plan how to extract these into separate services.", "step_number": 2, "total_steps": 5, "next_step_required": True, "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue planning to step 2") return False response2_data = self._parse_planner_response(response2) if not self._validate_step_response(response2_data, 2, 5, True, "planning_success"): return False self.logger.info(" ✅ Step 2 successful") # Step 3: Final step self.logger.info(" 1.1.3: Step 3 - Final planning step") response3, _ = self.call_mcp_tool( "planner", { "step": "Now I'll create a phased migration strategy: Phase 1 - Extract User Management, Phase 2 - Product Catalog and Inventory, Phase 3 - Order Processing and Payment services. This completes the initial migration plan.", "step_number": 3, "total_steps": 3, # Adjusted total "next_step_required": False, # Final step "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to complete planning session") return False response3_data = self._parse_planner_response(response3) if not self._validate_final_step_response(response3_data, 3, 3): return False self.logger.info(" ✅ Planning session completed successfully") # Store continuation_id for next test self.migration_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single planning session test failed: {e}") return False def _test_plan_continuation(self) -> bool: """Test continuing from a previous completed plan""" try: self.logger.info(" 1.2: Testing plan continuation with previous context") # Start a new planning session using the continuation_id from previous completed plan self.logger.info(" 1.2.1: New planning session with previous plan context") response1, new_continuation_id = self.call_mcp_tool( "planner", { "step": "Now that I have the microservices migration plan, let me plan the database strategy. I need to decide how to handle data consistency across the new services.", "step_number": 1, # New planning session starts at step 1 "total_steps": 4, "next_step_required": True, "continuation_id": self.migration_continuation_id, # Use previous plan's continuation_id }, ) if not response1 or not new_continuation_id: self.logger.error("Failed to start new planning session with context") return False response1_data = self._parse_planner_response(response1) if not response1_data: return False # Should have previous plan context if "previous_plan_context" not in response1_data: self.logger.error("Expected previous_plan_context in new planning session") return False # Check for key terms from the previous plan context = response1_data["previous_plan_context"].lower() if "migration" not in context and "plan" not in context: self.logger.error("Previous plan context doesn't contain expected content") return False self.logger.info(" ✅ New planning session loaded previous plan context") # Continue the new planning session (step 2+ should NOT load context) self.logger.info(" 1.2.2: Continue new planning session (no context loading)") response2, _ = self.call_mcp_tool( "planner", { "step": "I'll implement a database-per-service pattern with eventual consistency using event sourcing for cross-service communication.", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": new_continuation_id, # Same continuation, step 2 }, ) if not response2: self.logger.error("Failed to continue new planning session") return False response2_data = self._parse_planner_response(response2) if not response2_data: return False # Step 2+ should NOT have previous_plan_context (only step 1 with continuation_id gets context) if "previous_plan_context" in response2_data: self.logger.error("Step 2 should NOT have previous_plan_context") return False self.logger.info(" ✅ Step 2 correctly has no previous context (as expected)") return True except Exception as e: self.logger.error(f"Plan continuation test failed: {e}") return False def _test_branching_and_revision(self) -> bool: """Test branching and revision capabilities""" try: self.logger.info(" 1.3: Testing branching and revision capabilities") # Start a new planning session for testing branching self.logger.info(" 1.3.1: Start planning session for branching test") response1, continuation_id = self.call_mcp_tool( "planner", { "step": "Let me plan the deployment strategy for the microservices. I'll consider different deployment options.", "step_number": 1, "total_steps": 4, "next_step_required": True, }, ) if not response1 or not continuation_id: self.logger.error("Failed to start branching test planning session") return False # Test branching self.logger.info(" 1.3.2: Create a branch from step 1") response2, _ = self.call_mcp_tool( "planner", { "step": "Branch A: I'll explore Kubernetes deployment with service mesh (Istio) for advanced traffic management and observability.", "step_number": 2, "total_steps": 4, "next_step_required": True, "is_branch_point": True, "branch_from_step": 1, "branch_id": "kubernetes-istio", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to create branch") return False response2_data = self._parse_planner_response(response2) if not response2_data: return False # Validate branching metadata metadata = response2_data.get("metadata", {}) if not metadata.get("is_branch_point"): self.logger.error("Branch point not properly recorded in metadata") return False if metadata.get("branch_id") != "kubernetes-istio": self.logger.error("Branch ID not properly recorded") return False if "kubernetes-istio" not in metadata.get("branches", []): self.logger.error("Branch not recorded in branches list") return False self.logger.info(" ✅ Branching working correctly") # Test revision self.logger.info(" 1.3.3: Revise step 2") response3, _ = self.call_mcp_tool( "planner", { "step": "Revision: Actually, let me revise the Kubernetes approach. I'll use a simpler deployment initially, then migrate to Kubernetes later.", "step_number": 3, "total_steps": 4, "next_step_required": True, "is_step_revision": True, "revises_step_number": 2, "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to create revision") return False response3_data = self._parse_planner_response(response3) if not response3_data: return False # Validate revision metadata metadata = response3_data.get("metadata", {}) if not metadata.get("is_step_revision"): self.logger.error("Step revision not properly recorded in metadata") return False if metadata.get("revises_step_number") != 2: self.logger.error("Revised step number not properly recorded") return False self.logger.info(" ✅ Revision working correctly") return True except Exception as e: self.logger.error(f"Branching and revision test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for planner-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from planner response specifically continuation_id = self._extract_planner_continuation_id(response_text) return response_text, continuation_id def _extract_planner_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from planner response""" try: # Parse the response - it's now direct JSON, not wrapped response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for planner continuation_id: {e}") return None def _parse_planner_response(self, response_text: str) -> dict: """Parse planner tool JSON response""" try: # Parse the response - it's now direct JSON, not wrapped return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse planner response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a planning step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check that step_content exists if not response_data.get("step_content"): self.logger.error("Missing step_content in response") return False # Check metadata exists if "metadata" not in response_data: self.logger.error("Missing metadata in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False def _validate_final_step_response(self, response_data: dict, expected_step: int, expected_total: int) -> bool: """Validate a final planning step response""" try: # Basic step validation if not self._validate_step_response( response_data, expected_step, expected_total, False, "planning_success" ): return False # Check planning_complete flag if not response_data.get("planning_complete"): self.logger.error("Expected planning_complete=true for final step") return False # Check plan_summary exists if not response_data.get("plan_summary"): self.logger.error("Missing plan_summary in final step") return False # Check plan_summary contains expected content plan_summary = response_data.get("plan_summary", "") if "COMPLETE PLAN:" not in plan_summary: self.logger.error("plan_summary doesn't contain 'COMPLETE PLAN:' marker") return False # Check next_steps mentions completion next_steps = response_data.get("next_steps", "") if "complete" not in next_steps.lower(): self.logger.error("next_steps doesn't indicate planning completion") return False return True except Exception as e: self.logger.error(f"Error validating final step response: {e}") return False ================================================ FILE: simulator_tests/test_precommitworkflow_validation.py ================================================ #!/usr/bin/env python3 """ PrecommitWorkflow Tool Validation Test Tests the precommit tool's capabilities using the new workflow architecture. This validates that the workflow-based pre-commit validation provides step-by-step analysis with proper investigation guidance and expert analysis integration. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class PrecommitWorkflowValidationTest(ConversationBaseTest): """Test precommit tool with new workflow architecture""" @property def test_name(self) -> str: return "precommit_validation" @property def test_description(self) -> str: return "PrecommitWorkflow tool validation with new workflow architecture" def run_test(self) -> bool: """Test precommit tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: PrecommitWorkflow tool validation (new architecture)") # Create test git repository structure with changes self._create_test_git_changes() # Test 1: Single validation session with multiple steps if not self._test_single_validation_session(): return False # Test 2: Validation flow that requires refocusing if not self._test_validation_refocus_flow(): return False # Test 3: Complete validation with expert analysis if not self._test_complete_validation_with_analysis(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Context-aware file embedding if not self._test_context_aware_file_embedding(): return False # Test 6: Multi-step file context optimization if not self._test_multi_step_file_context(): return False self.logger.info(" ✅ All precommit validation tests passed") return True except Exception as e: self.logger.error(f"PrecommitWorkflow validation test failed: {e}") return False def _create_test_git_changes(self): """Create test files simulating git changes for pre-commit validation""" # Create a new API endpoint with potential security issues new_api_code = """#!/usr/bin/env python3 from flask import Flask, request, jsonify import sqlite3 import os app = Flask(__name__) @app.route('/api/user/', methods=['GET']) def get_user(user_id): \"\"\"Get user information by ID\"\"\" # Potential SQL injection vulnerability conn = sqlite3.connect('users.db') cursor = conn.cursor() # BUG: Direct string interpolation creates SQL injection risk query = f"SELECT * FROM users WHERE id = {user_id}" cursor.execute(query) result = cursor.fetchone() conn.close() if result: return jsonify({ 'id': result[0], 'username': result[1], 'email': result[2], 'password_hash': result[3] # Security issue: exposing password hash }) else: return jsonify({'error': 'User not found'}), 404 @app.route('/api/admin/users', methods=['GET']) def list_all_users(): \"\"\"Admin endpoint to list all users\"\"\" # Missing authentication check conn = sqlite3.connect('users.db') cursor = conn.cursor() cursor.execute("SELECT id, username, email FROM users") users = [] for row in cursor.fetchall(): users.append({ 'id': row[0], 'username': row[1], 'email': row[2] }) conn.close() return jsonify(users) if __name__ == '__main__': # Debug mode in production is a security risk app.run(debug=True, host='0.0.0.0') """ # Create configuration file with issues config_code = """#!/usr/bin/env python3 import os # Database configuration DATABASE_URL = os.getenv('DATABASE_URL', 'sqlite:///users.db') # Security settings SECRET_KEY = "hardcoded-secret-key-123" # Security issue: hardcoded secret DEBUG_MODE = True # Should be environment-based # API settings API_RATE_LIMIT = 1000 # Very high, no rate limiting effectively MAX_FILE_UPLOAD = 50 * 1024 * 1024 # 50MB - quite large # Missing important security headers configuration CORS_ORIGINS = "*" # Security issue: allows all origins """ # Create test files self.api_file = self.create_additional_test_file("api_endpoints.py", new_api_code) self.config_file = self.create_additional_test_file("config.py", config_code) self.logger.info(f" ✅ Created test files: {self.api_file}, {self.config_file}") # Create change description change_description = """COMMIT DESCRIPTION: Added new user API endpoints and configuration for user management system. CHANGES MADE: - Added GET /api/user/ endpoint to retrieve user information - Added GET /api/admin/users endpoint for admin user listing - Added configuration file with database and security settings - Set up Flask application with basic routing REQUIREMENTS: - User data should be retrievable by ID - Admin should be able to list all users - System should be configurable via environment variables - Security should be properly implemented """ self.changes_file = self.create_additional_test_file("commit_description.txt", change_description) self.logger.info(f" ✅ Created change description: {self.changes_file}") def _test_single_validation_session(self) -> bool: """Test a complete validation session with multiple steps""" try: self.logger.info(" 1.1: Testing single validation session") # Step 1: Start validation self.logger.info(" 1.1.1: Step 1 - Initial validation plan") response1, continuation_id = self.call_mcp_tool( "precommit", { "step": "I need to perform comprehensive pre-commit validation for new API endpoints. Let me start by analyzing the changes and identifying potential issues.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "New user API endpoints and configuration added. Need to examine for security, performance, and best practices.", "files_checked": [self.changes_file], "relevant_files": [self.changes_file], "path": self.test_dir, # Required for step 1 "review_type": "full", "severity_filter": "all", }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial validation response") return False # Parse and validate JSON response response1_data = self._parse_precommit_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_validation for next_step_required=True if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_validation"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Examine the code for issues self.logger.info(" 1.1.2: Step 2 - Code examination") response2, _ = self.call_mcp_tool( "precommit", { "step": "Now examining the API endpoint implementation and configuration for security vulnerabilities and best practices violations.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Found multiple critical security issues: SQL injection vulnerability in get_user(), hardcoded secrets in config, missing authentication, and password hash exposure.", "files_checked": [self.changes_file, self.api_file, self.config_file], "relevant_files": [self.api_file, self.config_file], "relevant_context": ["get_user", "list_all_users"], "issues_found": [ {"severity": "critical", "description": "SQL injection vulnerability in user lookup"}, {"severity": "high", "description": "Hardcoded secret key in configuration"}, {"severity": "high", "description": "Password hash exposed in API response"}, {"severity": "medium", "description": "Missing authentication on admin endpoint"}, ], # Assessment field removed - using precommit_type instead # Confidence field removed - using precommit_type instead "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue validation to step 2") return False response2_data = self._parse_precommit_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_validation"): return False # Check validation status tracking validation_status = response2_data.get("validation_status", {}) if validation_status.get("files_checked", 0) < 3: self.logger.error("Files checked count not properly tracked") return False if validation_status.get("issues_identified", 0) != 4: self.logger.error("Issues found not properly tracked") return False if validation_status.get("precommit_type") != "external": self.logger.error("Precommit type not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper tracking") # Store continuation_id for next test self.validation_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single validation session test failed: {e}") return False def _test_validation_refocus_flow(self) -> bool: """Test validation workflow that requires refocusing to revise findings""" try: self.logger.info(" 1.2: Testing validation refocus workflow") # Start a new validation for testing refocus behaviour self.logger.info(" 1.2.1: Start validation for refocus test") response1, continuation_id = self.call_mcp_tool( "precommit", { "step": "Validating database connection optimization changes", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial analysis shows database connection pooling implementation", "files_checked": ["/db/connection.py"], "relevant_files": ["/db/connection.py"], "path": self.test_dir, }, ) if not response1 or not continuation_id: self.logger.error("Failed to start refocus test validation") return False # Step 2: Wrong direction self.logger.info(" 1.2.2: Step 2 - Wrong validation focus") response2, _ = self.call_mcp_tool( "precommit", { "step": "Focusing on connection pool size optimization", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Connection pool configuration seems reasonable, might be looking in wrong place", "files_checked": ["/db/connection.py", "/config/database.py"], "relevant_files": [], # Assessment fields removed - using precommit_type instead "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Shift investigation focus self.logger.info(" 1.2.3: Step 3 - Refocus and revise approach") response3, _ = self.call_mcp_tool( "precommit", { "step": "Refocusing - the issue might not be database configuration. Let me examine the actual SQL queries and data access patterns instead.", "step_number": 3, "total_steps": 4, "next_step_required": True, "findings": "Found inefficient N+1 query pattern in user data loading causing performance issues", "files_checked": ["/models/user.py"], "relevant_files": ["/models/user.py"], "relevant_context": ["User.load_profile"], "issues_found": [ {"severity": "medium", "description": "N+1 query pattern in user profile loading"} ], # Assessment fields removed - using precommit_type instead "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to refocus") return False response3_data = self._parse_precommit_response(response3) if not self._validate_step_response(response3_data, 3, 4, True, "pause_for_validation"): return False self.logger.info(" ✅ Refocus flow working correctly") return True except Exception as e: self.logger.error(f"Refocus test failed: {e}") return False def _test_complete_validation_with_analysis(self) -> bool: """Test complete validation ending with expert analysis""" try: self.logger.info(" 1.3: Testing complete validation with expert analysis") # Use the continuation from first test continuation_id = getattr(self, "validation_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh validation") response0, continuation_id = self.call_mcp_tool( "precommit", { "step": "Validating the security fixes for API endpoints", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found critical security vulnerabilities in API implementation", "files_checked": [self.api_file], "relevant_files": [self.api_file], "relevant_context": ["get_user", "list_all_users"], "issues_found": [{"severity": "critical", "description": "SQL injection vulnerability"}], "path": self.test_dir, }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh validation") return False # Final step - trigger expert analysis self.logger.info(" 1.3.1: Final step - complete validation") response_final, _ = self.call_mcp_tool( "precommit", { "step": "Validation complete. I have identified all critical security issues and missing safeguards in the new API endpoints.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert analysis "findings": "Comprehensive analysis complete: SQL injection, hardcoded secrets, missing authentication, password exposure, and insecure defaults all identified with specific fixes needed.", "files_checked": [self.api_file, self.config_file], "relevant_files": [self.api_file, self.config_file], "relevant_context": ["get_user", "list_all_users", "SECRET_KEY", "DEBUG_MODE"], "issues_found": [ {"severity": "critical", "description": "SQL injection vulnerability in user lookup query"}, {"severity": "high", "description": "Hardcoded secret key exposes application security"}, {"severity": "high", "description": "Password hash exposed in API response"}, {"severity": "medium", "description": "Missing authentication on admin endpoint"}, {"severity": "medium", "description": "Debug mode enabled in production configuration"}, ], # Confidence field removed - using precommit_type instead "continuation_id": continuation_id, "model": "flash", # Use flash for expert analysis }, ) if not response_final: self.logger.error("Failed to complete validation") return False response_final_data = self._parse_precommit_response(response_final) if not response_final_data: return False # Validate final response structure - expect calling_expert_analysis for next_step_required=False if response_final_data.get("status") != "calling_expert_analysis": self.logger.error( f"Expected status 'calling_expert_analysis', got '{response_final_data.get('status')}'" ) return False if not response_final_data.get("validation_complete"): self.logger.error("Expected validation_complete=true for final step") return False # Check for expert analysis if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) # Check for expected analysis content (checking common patterns) analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower() # Look for security issue identification security_indicators = ["sql", "injection", "security", "hardcoded", "secret", "authentication"] found_indicators = sum(1 for indicator in security_indicators if indicator in analysis_text) if found_indicators >= 3: self.logger.info(" ✅ Expert analysis identified security issues correctly") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully identified security issues (found {found_indicators}/6 indicators)" ) # Check complete validation summary if "complete_validation" not in response_final_data: self.logger.error("Missing complete_validation in final response") return False complete_validation = response_final_data["complete_validation"] if not complete_validation.get("relevant_context"): self.logger.error("Missing relevant context in complete validation") return False if "get_user" not in complete_validation["relevant_context"]: self.logger.error("Expected function not found in validation summary") return False self.logger.info(" ✅ Complete validation with expert analysis successful") return True except Exception as e: self.logger.error(f"Complete validation test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test certain confidence behavior - should skip expert analysis""" try: self.logger.info(" 1.4: Testing certain confidence behavior") # Test certain confidence - should skip expert analysis self.logger.info(" 1.4.1: Certain confidence validation") response_certain, _ = self.call_mcp_tool( "precommit", { "step": "I have confirmed all security issues with 100% certainty: SQL injection, hardcoded secrets, and missing authentication.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "All critical issues identified: parameterized queries needed, environment variables for secrets, authentication middleware required, and debug mode must be disabled for production.", "files_checked": [self.api_file, self.config_file], "relevant_files": [self.api_file, self.config_file], "relevant_context": ["get_user", "list_all_users"], "issues_found": [ { "severity": "critical", "description": "SQL injection vulnerability - fix with parameterized queries", }, {"severity": "high", "description": "Hardcoded secret - use environment variables"}, {"severity": "medium", "description": "Missing authentication - add middleware"}, ], "precommit_type": "internal", # This should skip expert analysis "path": self.test_dir, "model": "flash", }, ) if not response_certain: self.logger.error("Failed to test certain confidence") return False response_certain_data = self._parse_precommit_response(response_certain) if not response_certain_data: return False # Validate certain confidence response - should skip expert analysis if response_certain_data.get("status") != "validation_complete_ready_for_commit": self.logger.error( f"Expected status 'validation_complete_ready_for_commit', got '{response_certain_data.get('status')}'" ) return False if not response_certain_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for certain confidence") return False expert_analysis = response_certain_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_internal_analysis_type": self.logger.error("Expert analysis should be skipped for certain confidence") return False self.logger.info(" ✅ Certain confidence behavior working correctly") return True except Exception as e: self.logger.error(f"Certain confidence test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for precommit-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from precommit response specifically continuation_id = self._extract_precommit_continuation_id(response_text) return response_text, continuation_id def _extract_precommit_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from precommit response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for precommit continuation_id: {e}") return None def _parse_precommit_response(self, response_text: str) -> dict: """Parse precommit tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse precommit response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a precommit validation step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check validation_status exists if "validation_status" not in response_data: self.logger.error("Missing validation_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False def _test_context_aware_file_embedding(self) -> bool: """Test context-aware file embedding optimization""" try: self.logger.info(" 1.5: Testing context-aware file embedding") # Create multiple test files for context testing auth_file_content = """#!/usr/bin/env python3 from functools import wraps from flask import request, jsonify def require_auth(f): \"\"\"Authentication decorator\"\"\" @wraps(f) def decorated_function(*args, **kwargs): token = request.headers.get('Authorization') if not token: return jsonify({'error': 'No token provided'}), 401 # Validate token here if not validate_token(token): return jsonify({'error': 'Invalid token'}), 401 return f(*args, **kwargs) return decorated_function def validate_token(token): \"\"\"Validate authentication token\"\"\" # Token validation logic return token.startswith('Bearer ') """ middleware_file_content = """#!/usr/bin/env python3 from flask import Flask, request, g import time def add_security_headers(app): \"\"\"Add security headers to all responses\"\"\" @app.after_request def security_headers(response): response.headers['X-Content-Type-Options'] = 'nosniff' response.headers['X-Frame-Options'] = 'DENY' response.headers['X-XSS-Protection'] = '1; mode=block' return response def rate_limiting_middleware(app): \"\"\"Basic rate limiting\"\"\" @app.before_request def limit_remote_addr(): # Simple rate limiting logic pass """ # Create test files auth_file = self.create_additional_test_file("auth.py", auth_file_content) middleware_file = self.create_additional_test_file("middleware.py", middleware_file_content) # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "precommit", { "step": "Starting validation of new authentication and security middleware", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of authentication and middleware components", "files_checked": [auth_file, middleware_file], "relevant_files": [auth_file], # This should be referenced, not embedded "relevant_context": ["require_auth"], # Assessment fields removed - using precommit_type instead "path": self.test_dir, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_precommit_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False if "Files referenced but not embedded" not in file_context.get("context_optimization", ""): self.logger.error("Expected context optimization message for reference_only") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Intermediate step with continuation - should still only reference self.logger.info(" 1.5.2: Intermediate step with continuation (should reference only)") response2, _ = self.call_mcp_tool( "precommit", { "step": "Continuing validation with detailed security analysis", "step_number": 2, "total_steps": 3, "next_step_required": True, # Still intermediate "continuation_id": continuation_id, "findings": "Found potential issues in token validation and missing security headers", "files_checked": [auth_file, middleware_file], "relevant_files": [auth_file, middleware_file], # Both files referenced "relevant_context": ["require_auth", "validate_token", "add_security_headers"], "issues_found": [ {"severity": "medium", "description": "Basic token validation might be insufficient"} ], # Assessment fields removed - using precommit_type instead "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False response2_data = self._parse_precommit_response(response2) if not response2_data: return False # Check file context - should still be reference_only file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context for step 2, got: {file_context2.get('type')}") return False # Should include reference note if not file_context2.get("note"): self.logger.error("Expected file reference note for intermediate step") return False reference_note = file_context2.get("note", "") if "auth.py" not in reference_note or "middleware.py" not in reference_note: self.logger.error("File reference note should mention both files") return False self.logger.info(" ✅ Intermediate step with continuation correctly uses reference_only") # Test 3: Final step - should embed files for expert analysis self.logger.info(" 1.5.3: Final step (should embed files)") response3, _ = self.call_mcp_tool( "precommit", { "step": "Validation complete - identified security gaps and improvement areas", "step_number": 3, "total_steps": 3, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Security implementation has several gaps: token validation is basic, missing CSRF protection, and rate limiting is not implemented", "files_checked": [auth_file, middleware_file], "relevant_files": [auth_file, middleware_file], # Should be fully embedded "relevant_context": ["require_auth", "validate_token", "add_security_headers"], "issues_found": [ {"severity": "medium", "description": "Token validation needs strengthening"}, {"severity": "low", "description": "Missing CSRF protection"}, {"severity": "low", "description": "Rate limiting not implemented"}, ], # Assessment field removed - using precommit_type instead # Confidence field removed - using precommit_type instead "model": "flash", }, ) if not response3: self.logger.error("Failed to complete to final step") return False response3_data = self._parse_precommit_response(response3) if not response3_data: return False # Check file context - should be fully_embedded for final step file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context3.get('type')}" ) return False if "Full file content embedded for expert analysis" not in file_context3.get("context_optimization", ""): self.logger.error("Expected expert analysis optimization message for fully_embedded") return False # Should show files embedded count files_embedded = file_context3.get("files_embedded", 0) if files_embedded == 0: # This is OK - files might already be in conversation history self.logger.info( " ℹ️ Files embedded count is 0 - files already in conversation history (smart deduplication)" ) else: self.logger.info(f" ✅ Files embedded count: {files_embedded}") self.logger.info(" ✅ Final step correctly uses fully_embedded file context") # Verify expert analysis was called for final step if response3_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False if "expert_analysis" not in response3_data: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Context-aware file embedding test completed successfully") return True except Exception as e: self.logger.error(f"Context-aware file embedding test failed: {e}") return False def _test_multi_step_file_context(self) -> bool: """Test multi-step workflow with proper file context transitions""" try: self.logger.info(" 1.6: Testing multi-step file context optimization") # Create a complex scenario with multiple files for pre-commit validation database_content = """#!/usr/bin/env python3 import sqlite3 import os from contextlib import contextmanager class DatabaseManager: def __init__(self): self.db_path = os.getenv('DATABASE_PATH', 'app.db') @contextmanager def get_connection(self): \"\"\"Get database connection with proper cleanup\"\"\" conn = None try: conn = sqlite3.connect(self.db_path) yield conn finally: if conn: conn.close() def create_user(self, username, email, password_hash): \"\"\"Create a new user\"\"\" with self.get_connection() as conn: cursor = conn.cursor() # Proper parameterized query cursor.execute( "INSERT INTO users (username, email, password_hash) VALUES (?, ?, ?)", (username, email, password_hash) ) conn.commit() return cursor.lastrowid """ tests_content = """#!/usr/bin/env python3 import unittest from unittest.mock import patch, MagicMock from database_manager import DatabaseManager class TestDatabaseManager(unittest.TestCase): def setUp(self): self.db_manager = DatabaseManager() @patch('sqlite3.connect') def test_create_user(self, mock_connect): \"\"\"Test user creation\"\"\" mock_conn = MagicMock() mock_cursor = MagicMock() mock_cursor.lastrowid = 123 mock_conn.cursor.return_value = mock_cursor mock_connect.return_value = mock_conn user_id = self.db_manager.create_user('testuser', 'test@example.com', 'hashed_password') self.assertEqual(user_id, 123) mock_cursor.execute.assert_called_once_with( "INSERT INTO users (username, email, password_hash) VALUES (?, ?, ?)", ('testuser', 'test@example.com', 'hashed_password') ) if __name__ == '__main__': unittest.main() """ # Create test files db_file = self.create_additional_test_file("database_manager.py", database_content) test_file = self.create_additional_test_file("test_database.py", tests_content) # Step 1: Start validation (new conversation) self.logger.info(" 1.6.1: Step 1 - Start validation") response1, continuation_id = self.call_mcp_tool( "precommit", { "step": "Validating new database manager implementation and corresponding tests", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "New database manager with connection handling and user creation functionality", "files_checked": [db_file], "relevant_files": [db_file], "relevant_context": [], # Assessment fields removed - using precommit_type instead "path": self.test_dir, "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start multi-step file context test") return False response1_data = self._parse_precommit_response(response1) # Validate step 1 - should use reference_only file_context1 = response1_data.get("file_context", {}) if file_context1.get("type") != "reference_only": self.logger.error("Step 1 should use reference_only file context") return False self.logger.info(" ✅ Step 1: reference_only file context") # Step 2: Expand validation self.logger.info(" 1.6.2: Step 2 - Expand validation") response2, _ = self.call_mcp_tool( "precommit", { "step": "Found good database implementation - now examining test coverage", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Database manager uses proper parameterized queries and context managers. Test file provides good coverage with mocking.", "files_checked": [db_file, test_file], "relevant_files": [db_file, test_file], "relevant_context": ["DatabaseManager.create_user", "TestDatabaseManager.test_create_user"], # Assessment fields removed - using precommit_type instead "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False response2_data = self._parse_precommit_response(response2) # Validate step 2 - should still use reference_only file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "reference_only": self.logger.error("Step 2 should use reference_only file context") return False # Should reference both files reference_note = file_context2.get("note", "") if "database_manager.py" not in reference_note or "test_database.py" not in reference_note: self.logger.error("Step 2 should reference both files in note") return False self.logger.info(" ✅ Step 2: reference_only file context with multiple files") # Step 3: Deep analysis self.logger.info(" 1.6.3: Step 3 - Deep analysis") response3, _ = self.call_mcp_tool( "precommit", { "step": "Performing comprehensive security and best practices analysis", "step_number": 3, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Code follows security best practices: parameterized queries prevent SQL injection, proper resource cleanup with context managers, environment-based configuration.", "files_checked": [db_file, test_file], "relevant_files": [db_file, test_file], "relevant_context": ["DatabaseManager.get_connection", "DatabaseManager.create_user"], "issues_found": [], # No issues found # Assessment field removed - using precommit_type instead # Confidence field removed - using precommit_type instead "model": "flash", }, ) if not response3: self.logger.error("Failed to continue to step 3") return False response3_data = self._parse_precommit_response(response3) # Validate step 3 - should still use reference_only file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "reference_only": self.logger.error("Step 3 should use reference_only file context") return False self.logger.info(" ✅ Step 3: reference_only file context") # Step 4: Final validation with expert consultation self.logger.info(" 1.6.4: Step 4 - Final step with expert analysis") response4, _ = self.call_mcp_tool( "precommit", { "step": "Validation complete - code is ready for commit", "step_number": 4, "total_steps": 4, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Comprehensive validation complete: secure implementation with parameterized queries, proper resource management, good test coverage, and no security vulnerabilities identified.", "files_checked": [db_file, test_file], "relevant_files": [db_file, test_file], "relevant_context": ["DatabaseManager", "TestDatabaseManager"], "issues_found": [], # Assessment field removed - using precommit_type instead # Confidence field removed - using precommit_type instead "model": "flash", }, ) if not response4: self.logger.error("Failed to complete to final step") return False response4_data = self._parse_precommit_response(response4) # Validate step 4 - should use fully_embedded for expert analysis file_context4 = response4_data.get("file_context", {}) if file_context4.get("type") != "fully_embedded": self.logger.error("Step 4 (final) should use fully_embedded file context") return False if "expert analysis" not in file_context4.get("context_optimization", "").lower(): self.logger.error("Final step should mention expert analysis in context optimization") return False # Verify expert analysis was triggered if response4_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False # Check that expert analysis has file context expert_analysis = response4_data.get("expert_analysis", {}) if not expert_analysis: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Step 4: fully_embedded file context with expert analysis") # Validate the complete workflow progression progression_summary = { "step_1": "reference_only (new conversation, intermediate)", "step_2": "reference_only (continuation, intermediate)", "step_3": "reference_only (continuation, intermediate)", "step_4": "fully_embedded (continuation, final)", } self.logger.info(" 📋 File context progression:") for step, context_type in progression_summary.items(): self.logger.info(f" {step}: {context_type}") self.logger.info(" ✅ Multi-step file context optimization test completed successfully") return True except Exception as e: self.logger.error(f"Multi-step file context test failed: {e}") return False ================================================ FILE: simulator_tests/test_prompt_size_limit_bug.py ================================================ #!/usr/bin/env python3 """ Prompt Size Limit Bug Test This test reproduces a critical bug where the prompt size limit check incorrectly includes conversation history when validating incoming prompts from Claude to MCP. The limit should ONLY apply to the actual prompt text sent by the user, not the entire conversation context. Bug Scenario: - User starts a conversation with chat tool - Continues conversation multiple times (building up history) - On subsequent continuation, a short prompt (150 chars) triggers "resend_prompt" error claiming >50k characters Expected Behavior: - Only count the actual prompt parameter for size limit - Conversation history should NOT count toward prompt size limit - Only the user's actual input should be validated against 50k limit """ from .conversation_base_test import ConversationBaseTest class PromptSizeLimitBugTest(ConversationBaseTest): """Test to reproduce and verify fix for prompt size limit bug""" @property def test_name(self) -> str: return "prompt_size_limit_bug" @property def test_description(self) -> str: return "Reproduce prompt size limit bug with conversation continuation" def run_test(self) -> bool: """Test prompt size limit bug reproduction using in-process calls""" try: self.logger.info("🐛 Test: Prompt size limit bug reproduction (in-process)") # Setup test environment self.setUp() # Create a test file to provide context test_file_content = """ # Test SwiftUI-like Framework Implementation struct ContentView: View { @State private var counter = 0 var body: some View { VStack { Text("Count: \\(counter)") Button("Increment") { counter += 1 } } } } class Renderer { static let shared = Renderer() func render(view: View) { // Implementation details for UIKit/AppKit rendering } } protocol View { var body: some View { get } } """ test_file_path = self.create_additional_test_file("SwiftFramework.swift", test_file_content) # Step 1: Start initial conversation self.logger.info(" Step 1: Start conversation with initial context") initial_prompt = "I'm building a SwiftUI-like framework. Can you help me design the architecture?" response1, continuation_id = self.call_mcp_tool_direct( "chat", { "prompt": initial_prompt, "absolute_file_paths": [test_file_path], "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error(" ❌ Failed to start initial conversation") return False self.logger.info(f" ✅ Initial conversation started: {continuation_id[:8]}...") # Step 2: Continue conversation multiple times to build substantial history conversation_prompts = [ "That's helpful! Can you elaborate on the View protocol design?", "How should I implement the State property wrapper?", "What's the best approach for the VStack layout implementation?", "Should I use UIKit directly or create an abstraction layer?", "Smart approach! For the rendering layer, would you suggest UIKit/AppKit directly?", ] for i, prompt in enumerate(conversation_prompts, 2): self.logger.info(f" Step {i}: Continue conversation (exchange {i})") response, _ = self.call_mcp_tool_direct( "chat", { "prompt": prompt, "continuation_id": continuation_id, "model": "flash", }, ) if not response: self.logger.error(f" ❌ Failed at exchange {i}") return False self.logger.info(f" ✅ Exchange {i} completed") # Step 3: Send short prompt that should NOT trigger size limit self.logger.info(" Step 7: Send short prompt (should NOT trigger size limit)") # This is a very short prompt - should not trigger the bug after fix short_prompt = "Thanks! This gives me a solid foundation to start prototyping." self.logger.info(f" Short prompt length: {len(short_prompt)} characters") response_final, _ = self.call_mcp_tool_direct( "chat", { "prompt": short_prompt, "continuation_id": continuation_id, "model": "flash", }, ) if not response_final: self.logger.error(" ❌ Final short prompt failed") return False # Parse the response to check for the bug import json try: response_data = json.loads(response_final) status = response_data.get("status", "") if status == "resend_prompt": # This is the bug! Short prompt incorrectly triggering size limit metadata = response_data.get("metadata", {}) prompt_size = metadata.get("prompt_size", 0) self.logger.error( f" 🐛 BUG STILL EXISTS: Short prompt ({len(short_prompt)} chars) triggered resend_prompt" ) self.logger.error(f" Reported prompt_size: {prompt_size} (should be ~{len(short_prompt)})") self.logger.error(" This indicates conversation history is still being counted") return False # Bug still exists elif status in ["success", "continuation_available"]: self.logger.info(" ✅ Short prompt processed correctly - bug appears to be FIXED!") self.logger.info(f" Prompt length: {len(short_prompt)} chars, Status: {status}") return True else: self.logger.warning(f" ⚠️ Unexpected status: {status}") # Check if this might be a non-JSON response (successful execution) if len(response_final) > 0 and not response_final.startswith('{"'): self.logger.info(" ✅ Non-JSON response suggests successful tool execution") return True return False except json.JSONDecodeError: # Non-JSON response often means successful tool execution self.logger.info(" ✅ Non-JSON response suggests successful tool execution (bug likely fixed)") self.logger.debug(f" Response preview: {response_final[:200]}...") return True except Exception as e: self.logger.error(f"Prompt size limit bug test failed: {e}") import traceback self.logger.debug(f"Full traceback: {traceback.format_exc()}") return False def main(): """Run the prompt size limit bug test""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = PromptSizeLimitBugTest(verbose=verbose) success = test.run_test() if success: print("Bug reproduction test completed - check logs for details") else: print("Test failed to complete") sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_refactor_validation.py ================================================ #!/usr/bin/env python3 """ Refactor Tool Validation Test Tests the refactor tool's capabilities using the new workflow architecture. This validates the step-by-step refactoring analysis pattern with expert validation. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class RefactorValidationTest(ConversationBaseTest): """Test refactor tool with new workflow architecture""" @property def test_name(self) -> str: return "refactor_validation" @property def test_description(self) -> str: return "Refactor tool validation with new workflow architecture" def run_test(self) -> bool: """Test refactor tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: Refactor tool validation (new architecture)") # Create test files with refactoring opportunities self._create_refactoring_test_code() # Test 1: Single refactoring analysis session with multiple steps if not self._test_single_refactoring_session(): return False # Test 2: Refactoring analysis requiring refocus if not self._test_refactoring_refocus_flow(): return False # Test 3: Complete refactoring analysis with expert analysis if not self._test_complete_refactoring_with_analysis(): return False # Test 4: Certain confidence with complete refactor_result_confidence if not self._test_certain_confidence_complete_refactoring(): return False # Test 5: Context-aware file embedding for refactoring if not self._test_context_aware_refactoring_file_embedding(): return False # Test 6: Different refactor types if not self._test_different_refactor_types(): return False self.logger.info(" ✅ All refactor validation tests passed") return True except Exception as e: self.logger.error(f"Refactor validation test failed: {e}") return False def _create_refactoring_test_code(self): """Create test files with various refactoring opportunities""" # Create a Python file with obvious code smells and decomposition opportunities refactor_code = """#!/usr/bin/env python3 import json import os from datetime import datetime # Code smell: Large class with multiple responsibilities class DataProcessorManager: def __init__(self, config_file): self.config = self._load_config(config_file) self.processed_count = 0 self.error_count = 0 self.log_file = "processing.log" def _load_config(self, config_file): \"\"\"Load configuration from file\"\"\" with open(config_file, 'r') as f: return json.load(f) # Code smell: Long method doing too many things (decompose opportunity) def process_user_data(self, user_data, validation_rules, output_format): \"\"\"Process user data with validation and formatting\"\"\" # Validation logic if not user_data: print("Error: No user data") # Code smell: print instead of logging return None if not isinstance(user_data, dict): print("Error: Invalid data format") return None # Check required fields required_fields = ['name', 'email', 'age'] for field in required_fields: if field not in user_data: print(f"Error: Missing field {field}") return None # Apply validation rules for rule in validation_rules: if rule['field'] == 'email': if '@' not in user_data['email']: # Code smell: simple validation print("Error: Invalid email") return None elif rule['field'] == 'age': if user_data['age'] < 18: # Code smell: magic number print("Error: Age too young") return None # Data processing processed_data = {} processed_data['full_name'] = user_data['name'].title() processed_data['email_domain'] = user_data['email'].split('@')[1] processed_data['age_category'] = 'adult' if user_data['age'] >= 18 else 'minor' # Code smell: Duplicate date formatting logic if output_format == 'json': processed_data['processed_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') result = json.dumps(processed_data, ensure_ascii=False) elif output_format == 'csv': processed_data['processed_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') result = f"{processed_data['full_name']},{processed_data['email_domain']},{processed_data['age_category']}" else: processed_data['processed_at'] = datetime.now().strftime('%Y-%m-%d %H:%M:%S') result = str(processed_data) # Logging and statistics self.processed_count += 1 with open(self.log_file, 'a') as f: # Code smell: file handling without context f.write(f"Processed: {user_data['name']} at {datetime.now()}\\n") return result # Code smell: Another long method (decompose opportunity) def batch_process_files(self, file_list, output_dir): \"\"\"Process multiple files in batch\"\"\" results = [] for file_path in file_list: # File validation if not os.path.exists(file_path): print(f"Error: File {file_path} not found") continue if not file_path.endswith('.json'): print(f"Error: File {file_path} is not JSON") continue # Read and process file try: with open(file_path, 'r') as f: data = json.load(f) # Code smell: Nested loops and complex logic for user_id, user_data in data.items(): if isinstance(user_data, dict): # Duplicate validation logic from process_user_data if 'name' in user_data and 'email' in user_data: if '@' in user_data['email']: # More processing... processed = { 'id': user_id, 'name': user_data['name'].title(), 'email': user_data['email'].lower() } results.append(processed) # Write output file output_file = os.path.join(output_dir, f"processed_{os.path.basename(file_path)}") with open(output_file, 'w') as f: json.dump(results, f, indent=2) except Exception as e: print(f"Error processing file {file_path}: {e}") self.error_count += 1 return results # Code smell: Method doing file I/O and business logic def generate_report(self): \"\"\"Generate processing report\"\"\" report_data = { 'total_processed': self.processed_count, 'total_errors': self.error_count, 'success_rate': (self.processed_count / (self.processed_count + self.error_count)) * 100 if (self.processed_count + self.error_count) > 0 else 0, 'generated_at': datetime.now().strftime('%Y-%m-%d %H:%M:%S') } # Write to multiple formats (code smell: duplicate logic) with open('report.json', 'w') as f: json.dump(report_data, f, indent=2) with open('report.txt', 'w') as f: f.write(f"Processing Report\\n") f.write(f"================\\n") f.write(f"Total Processed: {report_data['total_processed']}\\n") f.write(f"Total Errors: {report_data['total_errors']}\\n") f.write(f"Success Rate: {report_data['success_rate']:.2f}%\\n") f.write(f"Generated: {report_data['generated_at']}\\n") return report_data # Code smell: Utility functions that could be in a separate module def validate_email(email): \"\"\"Simple email validation\"\"\" return '@' in email and '.' in email def format_name(name): \"\"\"Format name to title case\"\"\" return name.title() if name else "" def calculate_age_category(age): \"\"\"Calculate age category\"\"\" if age < 18: return 'minor' elif age < 65: return 'adult' else: return 'senior' """ # Create test file with refactoring opportunities self.refactor_file = self.create_additional_test_file("data_processor_manager.py", refactor_code) self.logger.info(f" ✅ Created test file with refactoring opportunities: {self.refactor_file}") # Create a smaller file for focused testing small_refactor_code = """#!/usr/bin/env python3 # Code smell: God function def process_everything(data, config, logger): \"\"\"Function that does too many things\"\"\" # Validation if not data: print("No data") # Should use logger return None # Processing result = [] for item in data: if item > 5: # Magic number result.append(item * 2) # Magic number # Logging print(f"Processed {len(result)} items") # File I/O with open("output.txt", "w") as f: f.write(str(result)) return result # Modernization opportunity: Could use dataclass class UserData: def __init__(self, name, email, age): self.name = name self.email = email self.age = age def to_dict(self): return { 'name': self.name, 'email': self.email, 'age': self.age } """ self.small_refactor_file = self.create_additional_test_file("simple_processor.py", small_refactor_code) self.logger.info(f" ✅ Created small test file: {self.small_refactor_file}") def _test_single_refactoring_session(self) -> bool: """Test a complete refactoring analysis session with multiple steps""" try: self.logger.info(" 1.1: Testing single refactoring analysis session") # Step 1: Start refactoring analysis self.logger.info(" 1.1.1: Step 1 - Initial refactoring investigation") response1, continuation_id = self.call_mcp_tool( "refactor", { "step": "Starting refactoring analysis of the data processor code. Let me examine the code structure and identify opportunities for decomposition, code smell fixes, and modernization.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial scan shows a large DataProcessorManager class with multiple responsibilities. The class handles configuration, data processing, file I/O, and logging - violating single responsibility principle.", "files_checked": [self.refactor_file], "relevant_files": [self.refactor_file], "confidence": "incomplete", "refactor_type": "codesmells", "focus_areas": ["maintainability", "readability"], }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial refactoring response") return False # Parse and validate JSON response response1_data = self._parse_refactor_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_refactoring_analysis for next_step_required=True if not self._validate_refactoring_step_response( response1_data, 1, 4, True, "pause_for_refactoring_analysis" ): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Deeper analysis self.logger.info(" 1.1.2: Step 2 - Detailed code analysis") response2, _ = self.call_mcp_tool( "refactor", { "step": "Now examining the specific methods and identifying concrete refactoring opportunities. Found multiple code smells and decomposition needs.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Identified several major issues: 1) process_user_data method is 50+ lines doing validation, processing, and I/O. 2) Duplicate validation logic. 3) Magic numbers (18 for age). 4) print statements instead of proper logging. 5) File handling without proper context management.", "files_checked": [self.refactor_file], "relevant_files": [self.refactor_file], "relevant_context": [ "DataProcessorManager.process_user_data", "DataProcessorManager.batch_process_files", ], "issues_found": [ { "type": "codesmells", "severity": "high", "description": "Long method: process_user_data does too many things", }, { "type": "codesmells", "severity": "medium", "description": "Magic numbers: age validation uses hardcoded 18", }, { "type": "codesmells", "severity": "medium", "description": "Duplicate validation logic in multiple places", }, ], "confidence": "partial", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue refactoring analysis to step 2") return False response2_data = self._parse_refactor_response(response2) if not self._validate_refactoring_step_response( response2_data, 2, 4, True, "pause_for_refactoring_analysis" ): return False # Check refactoring status tracking refactoring_status = response2_data.get("refactoring_status", {}) if refactoring_status.get("files_checked", 0) < 1: self.logger.error("Files checked count not properly tracked") return False opportunities_by_type = refactoring_status.get("opportunities_by_type", {}) if "codesmells" not in opportunities_by_type: self.logger.error("Code smells not properly tracked in opportunities") return False if refactoring_status.get("refactor_confidence") != "partial": self.logger.error("Refactor confidence not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper refactoring tracking") # Store continuation_id for next test self.refactoring_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single refactoring session test failed: {e}") return False def _test_refactoring_refocus_flow(self) -> bool: """Test refactoring analysis that shifts focus mid-investigation""" try: self.logger.info(" 1.2: Testing refactoring analysis refocus workflow") # Start a new refactoring analysis for testing refocus behaviour self.logger.info(" 1.2.1: Start refactoring analysis for refocus test") response1, continuation_id = self.call_mcp_tool( "refactor", { "step": "Analyzing code for decomposition opportunities", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial focus on class-level decomposition", "files_checked": [self.small_refactor_file], "relevant_files": [self.small_refactor_file], "confidence": "incomplete", "refactor_type": "decompose", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start refocus test refactoring analysis") return False # Step 2: Wrong direction self.logger.info(" 1.2.2: Step 2 - Wrong refactoring focus") response2, _ = self.call_mcp_tool( "refactor", { "step": "Focusing on class decomposition strategies", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Class structure seems reasonable, might be looking in wrong direction", "files_checked": [self.small_refactor_file], "relevant_files": [], "confidence": "incomplete", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Backtrack from step 2 self.logger.info(" 1.2.3: Step 3 - Refocus on function decomposition") response3, _ = self.call_mcp_tool( "refactor", { "step": "Refocusing - the real decomposition opportunity is the god function process_everything. Let me analyze function-level refactoring instead.", "step_number": 3, "total_steps": 4, "next_step_required": True, "findings": "Found the main decomposition opportunity: process_everything function does validation, processing, logging, and file I/O. Should be split into separate functions with single responsibilities.", "files_checked": [self.small_refactor_file], "relevant_files": [self.small_refactor_file], "relevant_context": ["process_everything"], "issues_found": [ { "type": "decompose", "severity": "high", "description": "God function: process_everything has multiple responsibilities", }, { "type": "codesmells", "severity": "medium", "description": "Magic numbers in processing logic", }, ], "confidence": "partial", "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to refocus") return False response3_data = self._parse_refactor_response(response3) if not self._validate_refactoring_step_response( response3_data, 3, 4, True, "pause_for_refactoring_analysis" ): return False self.logger.info(" ✅ Refocus working correctly for refactoring analysis") return True except Exception as e: self.logger.error(f"Refocusing test failed: {e}") return False def _test_complete_refactoring_with_analysis(self) -> bool: """Test complete refactoring analysis ending with expert analysis""" try: self.logger.info(" 1.3: Testing complete refactoring analysis with expert analysis") # Use the continuation from first test continuation_id = getattr(self, "refactoring_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh refactoring analysis") response0, continuation_id = self.call_mcp_tool( "refactor", { "step": "Analyzing the data processor for comprehensive refactoring opportunities", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found multiple refactoring opportunities in DataProcessorManager", "files_checked": [self.refactor_file], "relevant_files": [self.refactor_file], "relevant_context": ["DataProcessorManager.process_user_data"], "confidence": "partial", "refactor_type": "codesmells", }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh refactoring analysis") return False # Final step - trigger expert analysis self.logger.info(" 1.3.1: Final step - complete refactoring analysis") response_final, _ = self.call_mcp_tool( "refactor", { "step": "Refactoring analysis complete. Identified comprehensive opportunities for code smell fixes, decomposition, and modernization across the DataProcessorManager class.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert analysis "findings": "Complete analysis shows: 1) Large class violating SRP, 2) Long methods needing decomposition, 3) Duplicate validation logic, 4) Magic numbers, 5) Poor error handling with print statements, 6) File I/O mixed with business logic. All major refactoring opportunities identified with specific line locations.", "files_checked": [self.refactor_file], "relevant_files": [self.refactor_file], "relevant_context": [ "DataProcessorManager.process_user_data", "DataProcessorManager.batch_process_files", "DataProcessorManager.generate_report", ], "issues_found": [ { "type": "decompose", "severity": "critical", "description": "Large class with multiple responsibilities", }, { "type": "codesmells", "severity": "high", "description": "Long method: process_user_data (50+ lines)", }, {"type": "codesmells", "severity": "high", "description": "Duplicate validation logic"}, {"type": "codesmells", "severity": "medium", "description": "Magic numbers in age validation"}, { "type": "modernize", "severity": "medium", "description": "Use proper logging instead of print statements", }, ], "confidence": "partial", # Use partial to trigger expert analysis "continuation_id": continuation_id, "model": "flash", # Use flash for expert analysis }, ) if not response_final: self.logger.error("Failed to complete refactoring analysis") return False response_final_data = self._parse_refactor_response(response_final) if not response_final_data: return False # Validate final response structure - expect calling_expert_analysis or files_required_to_continue expected_statuses = ["calling_expert_analysis", "files_required_to_continue"] actual_status = response_final_data.get("status") if actual_status not in expected_statuses: self.logger.error(f"Expected status to be one of {expected_statuses}, got '{actual_status}'") return False if not response_final_data.get("refactoring_complete"): self.logger.error("Expected refactoring_complete=true for final step") return False # Check for expert analysis or content (depending on status) if actual_status == "calling_expert_analysis": if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) analysis_content = json.dumps(expert_analysis, ensure_ascii=False).lower() elif actual_status == "files_required_to_continue": # For files_required_to_continue, analysis is in content field if "content" not in response_final_data: self.logger.error("Missing content in files_required_to_continue response") return False expert_analysis = {"content": response_final_data.get("content", "")} analysis_content = response_final_data.get("content", "").lower() else: self.logger.error(f"Unexpected status: {actual_status}") return False # Check for expected analysis content (checking common patterns) analysis_text = analysis_content # Look for refactoring identification refactor_indicators = ["refactor", "decompose", "code smell", "method", "class", "responsibility"] found_indicators = sum(1 for indicator in refactor_indicators if indicator in analysis_text) if found_indicators >= 3: self.logger.info(" ✅ Expert analysis identified refactoring opportunities correctly") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully identified refactoring opportunities (found {found_indicators}/6 indicators)" ) # Check complete refactoring summary if "complete_refactoring" not in response_final_data: self.logger.error("Missing complete_refactoring in final response") return False complete_refactoring = response_final_data["complete_refactoring"] if not complete_refactoring.get("relevant_context"): self.logger.error("Missing relevant context in complete refactoring") return False if "DataProcessorManager.process_user_data" not in complete_refactoring["relevant_context"]: self.logger.error("Expected method not found in refactoring summary") return False self.logger.info(" ✅ Complete refactoring analysis with expert analysis successful") return True except Exception as e: self.logger.error(f"Complete refactoring analysis test failed: {e}") return False def _test_certain_confidence_complete_refactoring(self) -> bool: """Test complete confidence - should skip expert analysis""" try: self.logger.info(" 1.4: Testing complete confidence behavior") # Test complete confidence - should skip expert analysis self.logger.info(" 1.4.1: Complete confidence refactoring") response_certain, _ = self.call_mcp_tool( "refactor", { "step": "I have completed comprehensive refactoring analysis with 100% certainty: identified all major opportunities including decomposition, code smells, and modernization.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "Complete refactoring analysis: 1) DataProcessorManager class needs decomposition into separate responsibilities, 2) process_user_data method needs breaking into validation, processing, and formatting functions, 3) Replace print statements with proper logging, 4) Extract magic numbers to constants, 5) Use dataclasses for modern Python patterns.", "files_checked": [self.small_refactor_file], "relevant_files": [self.small_refactor_file], "relevant_context": ["process_everything", "UserData"], "issues_found": [ {"type": "decompose", "severity": "high", "description": "God function needs decomposition"}, {"type": "modernize", "severity": "medium", "description": "Use dataclass for UserData"}, {"type": "codesmells", "severity": "medium", "description": "Replace print with logging"}, ], "confidence": "complete", # Complete confidence should skip expert analysis "refactor_type": "codesmells", "model": "flash", }, ) if not response_certain: self.logger.error("Failed to test certain confidence with complete refactoring") return False response_certain_data = self._parse_refactor_response(response_certain) if not response_certain_data: return False # Validate certain confidence response - should skip expert analysis if response_certain_data.get("status") != "refactoring_analysis_complete_ready_for_implementation": self.logger.error( f"Expected status 'refactoring_analysis_complete_ready_for_implementation', got '{response_certain_data.get('status')}'" ) return False if not response_certain_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for complete confidence") return False expert_analysis = response_certain_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_complete_refactoring_confidence": self.logger.error("Expert analysis should be skipped for complete confidence") return False self.logger.info(" ✅ Complete confidence behavior working correctly") return True except Exception as e: self.logger.error(f"Complete confidence test failed: {e}") return False def _test_context_aware_refactoring_file_embedding(self) -> bool: """Test context-aware file embedding optimization for refactoring workflow""" try: self.logger.info(" 1.5: Testing context-aware file embedding for refactoring") # Create multiple test files for context testing utils_content = """#!/usr/bin/env python3 # Utility functions with refactoring opportunities def calculate_total(items): \"\"\"Calculate total with magic numbers\"\"\" total = 0 for item in items: if item > 10: # Magic number total += item * 1.1 # Magic number for tax return total def format_output(data, format_type): \"\"\"Format output - duplicate logic\"\"\" if format_type == 'json': import json return json.dumps(data, ensure_ascii=False) elif format_type == 'csv': return ','.join(str(v) for v in data.values()) else: return str(data) """ helpers_content = """#!/usr/bin/env python3 # Helper functions that could be modernized class DataContainer: \"\"\"Simple data container - could use dataclass\"\"\" def __init__(self, name, value, category): self.name = name self.value = value self.category = category def to_dict(self): return { 'name': self.name, 'value': self.value, 'category': self.category } """ # Create test files utils_file = self.create_additional_test_file("utils.py", utils_content) helpers_file = self.create_additional_test_file("helpers.py", helpers_content) # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "refactor", { "step": "Starting refactoring analysis of utility modules", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of utility and helper modules for refactoring opportunities", "files_checked": [utils_file, helpers_file], "relevant_files": [utils_file], # This should be referenced, not embedded "relevant_context": ["calculate_total"], "confidence": "incomplete", "refactor_type": "codesmells", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_refactor_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False if "Files referenced but not embedded" not in file_context.get("context_optimization", ""): self.logger.error("Expected context optimization message for reference_only") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Final step - should embed files for expert analysis self.logger.info(" 1.5.2: Final step (should embed files)") response2, _ = self.call_mcp_tool( "refactor", { "step": "Refactoring analysis complete - identified all opportunities", "step_number": 3, "total_steps": 3, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete analysis: Found magic numbers in calculate_total, duplicate formatting logic, and modernization opportunity with DataContainer class that could use dataclass.", "files_checked": [utils_file, helpers_file], "relevant_files": [utils_file, helpers_file], # Should be fully embedded "relevant_context": ["calculate_total", "format_output", "DataContainer"], "issues_found": [ {"type": "codesmells", "severity": "medium", "description": "Magic numbers in calculate_total"}, {"type": "modernize", "severity": "low", "description": "DataContainer could use dataclass"}, {"type": "codesmells", "severity": "low", "description": "Duplicate formatting logic"}, ], "confidence": "partial", # Use partial to trigger expert analysis "model": "flash", }, ) if not response2: self.logger.error("Failed to complete to final step") return False response2_data = self._parse_refactor_response(response2) if not response2_data: return False # Check file context - should be fully_embedded for final step file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context2.get('type')}" ) return False if "Full file content embedded for expert analysis" not in file_context2.get("context_optimization", ""): self.logger.error("Expected expert analysis optimization message for fully_embedded") return False self.logger.info(" ✅ Final step correctly uses fully_embedded file context") # Verify expert analysis was called for final step (or files_required_to_continue) expected_statuses = ["calling_expert_analysis", "files_required_to_continue"] actual_status = response2_data.get("status") if actual_status not in expected_statuses: self.logger.error(f"Expected one of {expected_statuses}, got: {actual_status}") return False # Handle expert analysis based on status if actual_status == "calling_expert_analysis" and "expert_analysis" not in response2_data: self.logger.error("Expert analysis should be present in final step with calling_expert_analysis") return False self.logger.info(" ✅ Context-aware file embedding test for refactoring completed successfully") return True except Exception as e: self.logger.error(f"Context-aware refactoring file embedding test failed: {e}") return False def _test_different_refactor_types(self) -> bool: """Test different refactor types (decompose, modernize, organization)""" try: self.logger.info(" 1.6: Testing different refactor types") # Test decompose type self.logger.info(" 1.6.1: Testing decompose refactor type") response_decompose, _ = self.call_mcp_tool( "refactor", { "step": "Analyzing code for decomposition opportunities in large functions and classes", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Found large DataProcessorManager class that violates single responsibility principle and long process_user_data method that needs decomposition.", "files_checked": [self.refactor_file], "relevant_files": [self.refactor_file], "relevant_context": ["DataProcessorManager", "DataProcessorManager.process_user_data"], "issues_found": [ { "type": "decompose", "severity": "critical", "description": "Large class with multiple responsibilities", }, { "type": "decompose", "severity": "high", "description": "Long method doing validation, processing, and I/O", }, ], "confidence": "complete", "refactor_type": "decompose", "model": "flash", }, ) if not response_decompose: self.logger.error("Failed to test decompose refactor type") return False response_decompose_data = self._parse_refactor_response(response_decompose) # Check that decompose type is properly tracked refactoring_status = response_decompose_data.get("refactoring_status", {}) opportunities_by_type = refactoring_status.get("opportunities_by_type", {}) if "decompose" not in opportunities_by_type: self.logger.error("Decompose opportunities not properly tracked") return False self.logger.info(" ✅ Decompose refactor type working correctly") # Test modernize type self.logger.info(" 1.6.2: Testing modernize refactor type") response_modernize, _ = self.call_mcp_tool( "refactor", { "step": "Analyzing code for modernization opportunities using newer Python features", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Found opportunities to use dataclasses, f-strings, pathlib, and proper logging instead of print statements.", "files_checked": [self.small_refactor_file], "relevant_files": [self.small_refactor_file], "relevant_context": ["UserData", "process_everything"], "issues_found": [ { "type": "modernize", "severity": "medium", "description": "UserData class could use @dataclass decorator", }, { "type": "modernize", "severity": "medium", "description": "Replace print statements with proper logging", }, {"type": "modernize", "severity": "low", "description": "Use pathlib for file operations"}, ], "confidence": "complete", "refactor_type": "modernize", "model": "flash", }, ) if not response_modernize: self.logger.error("Failed to test modernize refactor type") return False response_modernize_data = self._parse_refactor_response(response_modernize) # Check that modernize type is properly tracked refactoring_status = response_modernize_data.get("refactoring_status", {}) opportunities_by_type = refactoring_status.get("opportunities_by_type", {}) if "modernize" not in opportunities_by_type: self.logger.error("Modernize opportunities not properly tracked") return False self.logger.info(" ✅ Modernize refactor type working correctly") self.logger.info(" ✅ Different refactor types test completed successfully") return True except Exception as e: self.logger.error(f"Different refactor types test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for -specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from refactor response specifically continuation_id = self._extract_refactor_continuation_id(response_text) return response_text, continuation_id def _extract_refactor_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from refactor response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for refactor continuation_id: {e}") return None def _parse_refactor_response(self, response_text: str) -> dict: """Parse refactor tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse refactor response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_refactoring_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a refactor investigation step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check refactoring_status exists if "refactoring_status" not in response_data: self.logger.error("Missing refactoring_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating refactoring step response: {e}") return False ================================================ FILE: simulator_tests/test_secaudit_validation.py ================================================ #!/usr/bin/env python3 """ SECAUDIT Tool Validation Test Tests the secaudit tool's capabilities using the workflow architecture. This validates that the workflow-based security audit provides step-by-step analysis with proper investigation guidance and expert analysis integration. """ import json from .conversation_base_test import ConversationBaseTest class SecauditValidationTest(ConversationBaseTest): """Test secaudit tool with workflow architecture""" @property def test_name(self) -> str: return "secaudit_validation" @property def test_description(self) -> str: return "SECAUDIT tool validation with security audit workflow architecture" def run_test(self) -> bool: """Test secaudit tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: SECAUDIT tool validation (security workflow architecture)") # Create test code with various security vulnerabilities self._create_test_code_for_audit() # Test 1: Single audit session with multiple steps if not self._test_single_audit_session(): return False # Test 2: Audit with specific focus areas if not self._test_focused_security_audit(): return False # Test 3: Complete audit with expert analysis using fast model if not self._test_complete_audit_with_analysis(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Continuation test with chat tool if not self._test_continuation_with_chat(): return False # Test 6: Model selection control if not self._test_model_selection(): return False self.logger.info(" ✅ All secaudit validation tests passed") return True except Exception as e: self.logger.error(f"SECAUDIT validation test failed: {e}") return False def _create_test_code_for_audit(self): """Create test files with various security vulnerabilities""" # Create an authentication module with multiple security issues auth_code = """#!/usr/bin/env python3 import hashlib import pickle import sqlite3 from flask import request, session class AuthenticationManager: def __init__(self, db_path="users.db"): # A01: Broken Access Control - No proper session management self.db_path = db_path self.sessions = {} # In-memory session storage def login(self, username, password): '''User login with various security vulnerabilities''' # A03: Injection - SQL injection vulnerability conn = sqlite3.connect(self.db_path) cursor = conn.cursor() # Direct string interpolation in SQL query query = f"SELECT id, password_hash FROM users WHERE username = '{username}'" cursor.execute(query) user = cursor.fetchone() if not user: return {"status": "failed", "message": "User not found"} # A02: Cryptographic Failures - Weak hashing algorithm password_hash = hashlib.md5(password.encode()).hexdigest() if user[1] == password_hash: # A07: Identification and Authentication Failures - Weak session generation session_id = hashlib.md5(f"{username}{password}".encode()).hexdigest() self.sessions[session_id] = {"user_id": user[0], "username": username} return {"status": "success", "session_id": session_id} else: return {"status": "failed", "message": "Invalid password"} def reset_password(self, email): '''Password reset with security issues''' # A04: Insecure Design - No rate limiting or validation reset_token = hashlib.md5(email.encode()).hexdigest() # A09: Security Logging and Monitoring Failures - No security event logging # Simply returns token without any verification or logging return {"reset_token": reset_token, "url": f"/reset?token={reset_token}"} def deserialize_user_data(self, data): '''Unsafe deserialization''' # A08: Software and Data Integrity Failures - Insecure deserialization return pickle.loads(data) def get_user_profile(self, user_id): '''Get user profile with authorization issues''' # A01: Broken Access Control - No authorization check conn = sqlite3.connect(self.db_path) cursor = conn.cursor() # Fetches any user profile without checking permissions cursor.execute("SELECT * FROM users WHERE id = ?", (user_id,)) return cursor.fetchone() """ # Create authentication file self.auth_file = self.create_additional_test_file("auth_manager.py", auth_code) self.logger.info(f" ✅ Created authentication file with security issues: {self.auth_file}") # Create API endpoint with additional vulnerabilities api_code = """#!/usr/bin/env python3 from flask import Flask, request, jsonify import os import subprocess import requests app = Flask(__name__) # A05: Security Misconfiguration - Debug mode enabled app.config['DEBUG'] = True app.config['SECRET_KEY'] = 'dev-secret-key' # Hardcoded secret @app.route('/api/search', methods=['GET']) def search(): '''Search endpoint with multiple vulnerabilities''' # A03: Injection - XSS vulnerability, no input sanitization query = request.args.get('q', '') # A03: Injection - Command injection vulnerability if 'file:' in query: filename = query.split('file:')[1] # Direct command execution result = subprocess.run(f"cat {filename}", shell=True, capture_output=True, text=True) return jsonify({"result": result.stdout}) # A10: Server-Side Request Forgery (SSRF) if query.startswith('http'): # No validation of URL, allows internal network access response = requests.get(query) return jsonify({"content": response.text}) # Return search results without output encoding return f"

Search Results for: {query}

" @app.route('/api/admin', methods=['GET']) def admin_panel(): '''Admin panel with broken access control''' # A01: Broken Access Control - No authentication check # Anyone can access admin functionality action = request.args.get('action') if action == 'delete_user': user_id = request.args.get('user_id') # Performs privileged action without authorization return jsonify({"status": "User deleted", "user_id": user_id}) return jsonify({"status": "Admin panel"}) @app.route('/api/upload', methods=['POST']) def upload_file(): '''File upload with security issues''' # A05: Security Misconfiguration - No file type validation file = request.files.get('file') if file: # Saves any file type to server filename = file.filename file.save(os.path.join('/tmp', filename)) # A03: Path traversal vulnerability return jsonify({"status": "File uploaded", "path": f"/tmp/{filename}"}) return jsonify({"error": "No file provided"}) # A06: Vulnerable and Outdated Components # Using old Flask version with known vulnerabilities (hypothetical) # requirements.txt: Flask==0.12.2 (known security issues) if __name__ == '__main__': # A05: Security Misconfiguration - Running on all interfaces app.run(host='0.0.0.0', port=5000, debug=True) """ # Create API file self.api_file = self.create_additional_test_file("api_endpoints.py", api_code) self.logger.info(f" ✅ Created API file with security vulnerabilities: {self.api_file}") def _test_single_audit_session(self) -> bool: """Test a single security audit session with multiple steps""" self.logger.info(" 🔧 Testing single audit session...") try: # Step 1: Initial security audit request response, continuation_id = self.call_mcp_tool_direct( "secaudit", { "step": f"Begin security audit of authentication system in {self.auth_file}", "step_number": 1, "total_steps": 6, "next_step_required": True, "findings": "Starting security assessment", "relevant_files": [self.auth_file], "model": "gemini-2.0-flash-lite", }, ) if not response: self.logger.error("Failed to call secaudit tool") return False # Parse and validate the response try: response_data = json.loads(response) if response else {} except json.JSONDecodeError: response_data = {} # Check if it's asking for investigation status = response_data.get("status", "") if status != "pause_for_secaudit": self.logger.error(f"Expected pause_for_secaudit status, got: {status}") return False # Step 2: Continue with findings response2, _ = self.call_mcp_tool_direct( "secaudit", { "step": "Examined authentication module and found critical security vulnerabilities", "step_number": 2, "total_steps": 6, "next_step_required": True, "findings": ( "Found multiple OWASP Top 10 vulnerabilities: " "1. SQL injection in login method (line 88) - direct string interpolation in query " "2. Weak MD5 hashing for passwords (line 96) - cryptographically broken " "3. Insecure session management (line 100) - predictable session IDs " "4. Unsafe deserialization (line 119) - pickle.loads without validation" ), "files_checked": [self.auth_file], "relevant_files": [self.auth_file], "relevant_context": ["AuthenticationManager.login", "AuthenticationManager.deserialize_user_data"], "issues_found": [ {"severity": "critical", "description": "SQL injection vulnerability in login method"}, {"severity": "high", "description": "Weak MD5 password hashing"}, {"severity": "high", "description": "Insecure session management"}, {"severity": "critical", "description": "Unsafe deserialization vulnerability"}, ], "confidence": "medium", "continuation_id": continuation_id, "model": "gemini-2.0-flash-lite", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False self.logger.info(" ✅ Single audit session test passed") return True except Exception as e: self.logger.error(f"Single audit session test failed: {e}") return False def _test_focused_security_audit(self) -> bool: """Test security audit with specific focus areas""" self.logger.info(" 🔧 Testing focused security audit...") try: # Request OWASP-focused audit response, continuation_id = self.call_mcp_tool_direct( "secaudit", { "step": f"Begin OWASP-focused security audit of {self.api_file}", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Starting OWASP Top 10 focused security assessment", "relevant_files": [self.api_file], "security_scope": "Web API endpoints", "threat_level": "high", "audit_focus": "owasp", "model": "gemini-2.0-flash-lite", }, ) if not response: self.logger.error("Failed to start OWASP-focused audit") return False # Verify the audit was configured correctly try: response_data = json.loads(response) # The tool should acknowledge the OWASP focus if response_data.get("status") == "pause_for_secaudit": self.logger.info(" ✅ Focused security audit test passed") return True except json.JSONDecodeError: pass self.logger.error("Expected proper OWASP-focused configuration") return False except Exception as e: self.logger.error(f"Focused security audit test failed: {e}") return False def _test_complete_audit_with_analysis(self) -> bool: """Test complete security audit with expert analysis""" self.logger.info(" 🔧 Testing complete audit with expert analysis...") try: # Step 1: Start fresh audit response1, continuation_id = self.call_mcp_tool_direct( "secaudit", { "step": f"Begin comprehensive security audit of {self.auth_file} and {self.api_file}", "step_number": 1, "total_steps": 3, "next_step_required": True, "findings": "Starting OWASP Top 10 security assessment of authentication and API modules", "relevant_files": [self.auth_file, self.api_file], "security_scope": "Web application with authentication and API endpoints", "model": "gemini-2.0-flash-lite", }, ) if not response1: self.logger.error("Failed to start comprehensive audit") return False # Step 2: Continue with detailed findings response2, _ = self.call_mcp_tool_direct( "secaudit", { "step": "Completed comprehensive security investigation of both modules", "step_number": 2, "total_steps": 3, "next_step_required": True, "findings": ( "Found critical OWASP vulnerabilities across both modules: " "A01: Broken Access Control in admin panel, " "A03: SQL injection in login and command injection in search, " "A02: Weak cryptography with MD5 hashing, " "A05: Security misconfiguration with debug mode enabled, " "A07: Weak session management, " "A08: Insecure deserialization, " "A10: SSRF vulnerability in search endpoint" ), "files_checked": [self.auth_file, self.api_file], "relevant_files": [self.auth_file, self.api_file], "relevant_context": [ "AuthenticationManager.login", "AuthenticationManager.deserialize_user_data", "api.search", "api.admin_panel", ], "issues_found": [ {"severity": "critical", "description": "SQL injection in login method"}, {"severity": "critical", "description": "Command injection in search endpoint"}, {"severity": "critical", "description": "SSRF vulnerability allowing internal network access"}, {"severity": "high", "description": "Broken access control on admin panel"}, {"severity": "high", "description": "Insecure deserialization vulnerability"}, {"severity": "high", "description": "XSS vulnerability in search results"}, {"severity": "medium", "description": "Weak MD5 password hashing"}, {"severity": "medium", "description": "Security misconfiguration - debug mode enabled"}, ], "confidence": "high", "continuation_id": continuation_id, "model": "gemini-2.0-flash-lite", }, ) # Final step - skip expert analysis to avoid timeout response3, _ = self.call_mcp_tool_direct( "secaudit", { "step": "Complete security assessment with all vulnerabilities documented", "step_number": 3, "total_steps": 3, "next_step_required": False, "findings": "Security audit complete with 8 vulnerabilities identified across OWASP categories", "files_checked": [self.auth_file, self.api_file], "relevant_files": [self.auth_file, self.api_file], "confidence": "high", # High confidence to trigger expert analysis "continuation_id": continuation_id, "model": "gemini-2.0-flash-lite", }, ) if response3: # Check for expert analysis or completion status try: response_data = json.loads(response3) status = response_data.get("status", "") # Either expert analysis completed or security analysis complete if status in ["complete", "security_analysis_complete"]: self.logger.info(" ✅ Complete audit with expert analysis test passed") return True except json.JSONDecodeError: # If not JSON, check for security content (expert analysis output) if "security" in response3.lower() or "vulnerability" in response3.lower(): self.logger.info(" ✅ Complete audit with expert analysis test passed") return True self.logger.error("Expected expert security analysis or completion") return False except Exception as e: self.logger.error(f"Complete audit with analysis test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test behavior when confidence is certain""" self.logger.info(" 🔧 Testing certain confidence behavior...") try: # Request with certain confidence response, _ = self.call_mcp_tool_direct( "secaudit", { "step": f"Security audit complete for {self.auth_file}", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Critical SQL injection vulnerability confirmed in login method", "files_checked": [self.auth_file], "relevant_files": [self.auth_file], "issues_found": [ {"severity": "critical", "description": "SQL injection vulnerability in login method"} ], "confidence": "certain", "model": "gemini-2.0-flash-lite", }, ) if not response: self.logger.error("Failed to execute certain confidence test") return False try: response_data = json.loads(response) # With certain confidence, should complete without expert analysis if response_data.get("status") == "security_analysis_complete": self.logger.info(" ✅ Certain confidence correctly completes without expert analysis") return True except json.JSONDecodeError: pass # Check if findings are shown directly response_lower = response.lower() if "sql injection" in response_lower or "vulnerability" in response_lower: self.logger.info(" ✅ Certain confidence shows findings directly") return True self.logger.error("Expected completion or direct findings with certain confidence") return False except Exception as e: self.logger.error(f"Certain confidence test failed: {e}") return False def _test_continuation_with_chat(self) -> bool: """Test continuation functionality with chat tool""" self.logger.info(" 🔧 Testing continuation with chat tool...") try: # First, run a security audit that generates a continuation_id response1, continuation_id = self.call_mcp_tool_direct( "secaudit", { "step": f"Start analyzing {self.auth_file} for authentication vulnerabilities", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Beginning authentication security analysis", "relevant_files": [self.auth_file], "model": "gemini-2.0-flash-lite", }, ) if not response1: self.logger.error("Failed to start audit for continuation test") return False # Extract continuation_id if present if not continuation_id: self.logger.info(" ⚠️ No continuation_id returned, checking response") try: response_data = json.loads(response1) # Look for thread_id in metadata metadata = response_data.get("metadata", {}) continuation_id = metadata.get("thread_id") except json.JSONDecodeError: pass if continuation_id: # Now test using chat tool with continuation chat_response, _ = self.call_mcp_tool_direct( "chat", { "prompt": "Can you tell me more about the SQL injection vulnerability details found in the security audit?", "continuation_id": continuation_id, "model": "gemini-2.0-flash-lite", }, ) if chat_response: self.logger.info(" ✅ Chat tool continuation test passed") return True else: # Without continuation_id, just verify the audit step worked if response1: self.logger.info(" ✅ Audit step completed (continuation test limited)") return True self.logger.error("Expected successful continuation or audit step") return False except Exception as e: self.logger.error(f"Continuation test failed: {e}") return False def _test_model_selection(self) -> bool: """Test model selection and skip expert analysis option""" self.logger.info(" 🔧 Testing model selection control...") try: # Test 1: Explicit model selection response1, _ = self.call_mcp_tool_direct( "secaudit", { "step": f"Analyze {self.api_file} for SSRF vulnerabilities", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Starting SSRF vulnerability analysis", "relevant_files": [self.api_file], "audit_focus": "owasp", "model": "gemini-2.0-flash-lite", }, ) if response1: self.logger.info(" ✅ Model selection recognized") # Test 2: Skip expert analysis response2, _ = self.call_mcp_tool_direct( "secaudit", { "step": f"Complete security investigation of {self.auth_file}", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Security issues documented", "files_checked": [self.auth_file], "relevant_files": [self.auth_file], "confidence": "high", "use_assistant_model": False, # Skip expert analysis "model": "gemini-2.0-flash-lite", }, ) if response2: try: response_data = json.loads(response2) # Should complete without expert analysis if response_data.get("status") == "security_analysis_complete": self.logger.info(" ✅ Skip expert analysis option works") return True except json.JSONDecodeError: pass # Or might just complete the analysis response_lower = response2.lower() if "complete" in response_lower or "security" in response_lower: self.logger.info(" ✅ Analysis performed without expert model") return True self.logger.error("Expected model selection or skip behavior") return False except Exception as e: self.logger.error(f"Model selection test failed: {e}") return False ================================================ FILE: simulator_tests/test_testgen_validation.py ================================================ #!/usr/bin/env python3 """ TestGen Tool Validation Test Tests the testgen tool's capabilities using the workflow architecture. This validates that the workflow-based implementation guides Claude through systematic test generation analysis before creating comprehensive test suites. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class TestGenValidationTest(ConversationBaseTest): """Test testgen tool with workflow architecture""" @property def test_name(self) -> str: return "testgen_validation" @property def test_description(self) -> str: return "TestGen tool validation with step-by-step test planning" def run_test(self) -> bool: """Test testgen tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: TestGen tool validation") # Create sample code files to test self._create_test_code_files() # Test 1: Single investigation session with multiple steps if not self._test_single_test_generation_session(): return False # Test 2: Test generation with pattern following if not self._test_generation_with_pattern_following(): return False # Test 3: Complete test generation with expert analysis if not self._test_complete_generation_with_analysis(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Context-aware file embedding if not self._test_context_aware_file_embedding(): return False # Test 6: Multi-step test planning if not self._test_multi_step_test_planning(): return False self.logger.info(" ✅ All testgen validation tests passed") return True except Exception as e: self.logger.error(f"TestGen validation test failed: {e}") return False def _create_test_code_files(self): """Create sample code files for test generation""" # Create a calculator module with various functions calculator_code = """#!/usr/bin/env python3 \"\"\" Simple calculator module for demonstration \"\"\" def add(a, b): \"\"\"Add two numbers\"\"\" return a + b def subtract(a, b): \"\"\"Subtract b from a\"\"\" return a - b def multiply(a, b): \"\"\"Multiply two numbers\"\"\" return a * b def divide(a, b): \"\"\"Divide a by b\"\"\" if b == 0: raise ValueError("Cannot divide by zero") return a / b def calculate_percentage(value, percentage): \"\"\"Calculate percentage of a value\"\"\" if percentage < 0: raise ValueError("Percentage cannot be negative") if percentage > 100: raise ValueError("Percentage cannot exceed 100") return (value * percentage) / 100 def power(base, exponent): \"\"\"Calculate base raised to exponent\"\"\" if base == 0 and exponent < 0: raise ValueError("Cannot raise 0 to negative power") return base ** exponent """ # Create test file self.calculator_file = self.create_additional_test_file("calculator.py", calculator_code) self.logger.info(f" ✅ Created calculator module: {self.calculator_file}") # Create a simple existing test file to use as pattern existing_test = """#!/usr/bin/env python3 import pytest from calculator import add, subtract class TestCalculatorBasic: \"\"\"Test basic calculator operations\"\"\" def test_add_positive_numbers(self): \"\"\"Test adding two positive numbers\"\"\" assert add(2, 3) == 5 assert add(10, 20) == 30 def test_add_negative_numbers(self): \"\"\"Test adding negative numbers\"\"\" assert add(-5, -3) == -8 assert add(-10, 5) == -5 def test_subtract_positive(self): \"\"\"Test subtracting positive numbers\"\"\" assert subtract(10, 3) == 7 assert subtract(5, 5) == 0 """ self.existing_test_file = self.create_additional_test_file("test_calculator_basic.py", existing_test) self.logger.info(f" ✅ Created existing test file: {self.existing_test_file}") def _test_single_test_generation_session(self) -> bool: """Test a complete test generation session with multiple steps""" try: self.logger.info(" 1.1: Testing single test generation session") # Step 1: Start investigation self.logger.info(" 1.1.1: Step 1 - Initial test planning") response1, continuation_id = self.call_mcp_tool( "testgen", { "step": "I need to generate comprehensive tests for the calculator module. Let me start by analyzing the code structure and understanding the functionality.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Calculator module contains 6 functions: add, subtract, multiply, divide, calculate_percentage, and power. Each has specific error conditions that need testing.", "files_checked": [self.calculator_file], "relevant_files": [self.calculator_file], "relevant_context": ["add", "subtract", "multiply", "divide", "calculate_percentage", "power"], }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial test planning response") return False # Parse and validate JSON response response1_data = self._parse_testgen_response(response1) if not response1_data: return False # Validate step 1 response structure if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_test_analysis"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Analyze test requirements self.logger.info(" 1.1.2: Step 2 - Test requirements analysis") response2, _ = self.call_mcp_tool( "testgen", { "step": "Now analyzing the test requirements for each function, identifying edge cases and boundary conditions.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Identified key test scenarios: (1) divide - zero division error, (2) calculate_percentage - negative/over 100 validation, (3) power - zero to negative power error. Need tests for normal cases and edge cases.", "files_checked": [self.calculator_file], "relevant_files": [self.calculator_file], "relevant_context": ["divide", "calculate_percentage", "power"], "confidence": "medium", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue test planning to step 2") return False response2_data = self._parse_testgen_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_test_analysis"): return False # Check test generation status tracking test_status = response2_data.get("test_generation_status", {}) if test_status.get("test_scenarios_identified", 0) < 3: self.logger.error("Test scenarios not properly tracked") return False if test_status.get("analysis_confidence") != "medium": self.logger.error("Confidence level not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper tracking") # Store continuation_id for next test self.test_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single test generation session test failed: {e}") return False def _test_generation_with_pattern_following(self) -> bool: """Test test generation following existing patterns""" try: self.logger.info(" 1.2: Testing test generation with pattern following") # Start a new investigation with existing test patterns self.logger.info(" 1.2.1: Start test generation with pattern reference") response1, continuation_id = self.call_mcp_tool( "testgen", { "step": "Generating tests for remaining calculator functions following existing test patterns", "step_number": 1, "total_steps": 3, "next_step_required": True, "findings": "Found existing test pattern using pytest with class-based organization and descriptive test names", "files_checked": [self.calculator_file, self.existing_test_file], "relevant_files": [self.calculator_file, self.existing_test_file], "relevant_context": ["TestCalculatorBasic", "multiply", "divide", "calculate_percentage", "power"], }, ) if not response1 or not continuation_id: self.logger.error("Failed to start pattern following test") return False # Step 2: Analyze patterns self.logger.info(" 1.2.2: Step 2 - Pattern analysis") response2, _ = self.call_mcp_tool( "testgen", { "step": "Analyzing the existing test patterns to maintain consistency", "step_number": 2, "total_steps": 3, "next_step_required": True, "findings": "Existing tests use: class-based organization (TestCalculatorBasic), descriptive method names (test_operation_scenario), multiple assertions per test, pytest framework", "files_checked": [self.existing_test_file], "relevant_files": [self.calculator_file, self.existing_test_file], "confidence": "high", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False self.logger.info(" ✅ Pattern analysis successful") return True except Exception as e: self.logger.error(f"Pattern following test failed: {e}") return False def _test_complete_generation_with_analysis(self) -> bool: """Test complete test generation ending with expert analysis""" try: self.logger.info(" 1.3: Testing complete test generation with expert analysis") # Use the continuation from first test or start fresh continuation_id = getattr(self, "test_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh test generation") response0, continuation_id = self.call_mcp_tool( "testgen", { "step": "Analyzing calculator module for comprehensive test generation", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Identified 6 functions needing tests with various edge cases", "files_checked": [self.calculator_file], "relevant_files": [self.calculator_file], "relevant_context": ["add", "subtract", "multiply", "divide", "calculate_percentage", "power"], }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh test generation") return False # Final step - trigger expert analysis self.logger.info(" 1.3.1: Final step - complete test planning") response_final, _ = self.call_mcp_tool( "testgen", { "step": "Test planning complete. Identified all test scenarios including edge cases, error conditions, and boundary values for comprehensive coverage.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert analysis "findings": "Complete test plan: normal operations, edge cases (zero, negative), error conditions (divide by zero, invalid percentage, zero to negative power), boundary values", "files_checked": [self.calculator_file], "relevant_files": [self.calculator_file], "relevant_context": ["add", "subtract", "multiply", "divide", "calculate_percentage", "power"], "confidence": "high", "continuation_id": continuation_id, "model": "flash", # Use flash for expert analysis }, ) if not response_final: self.logger.error("Failed to complete test generation") return False response_final_data = self._parse_testgen_response(response_final) if not response_final_data: return False # Validate final response structure if response_final_data.get("status") != "calling_expert_analysis": self.logger.error( f"Expected status 'calling_expert_analysis', got '{response_final_data.get('status')}'" ) return False if not response_final_data.get("test_generation_complete"): self.logger.error("Expected test_generation_complete=true for final step") return False # Check for expert analysis if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) # Check for expected analysis content analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower() # Look for test generation indicators test_indicators = ["test", "edge", "boundary", "error", "coverage", "pytest"] found_indicators = sum(1 for indicator in test_indicators if indicator in analysis_text) if found_indicators >= 4: self.logger.info(" ✅ Expert analysis provided comprehensive test suggestions") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully addressed test generation (found {found_indicators}/6 indicators)" ) # Check complete test generation summary if "complete_test_generation" not in response_final_data: self.logger.error("Missing complete_test_generation in final response") return False complete_generation = response_final_data["complete_test_generation"] if not complete_generation.get("relevant_context"): self.logger.error("Missing relevant context in complete test generation") return False self.logger.info(" ✅ Complete test generation with expert analysis successful") return True except Exception as e: self.logger.error(f"Complete test generation test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test certain confidence behavior - should skip expert analysis""" try: self.logger.info(" 1.4: Testing certain confidence behavior") # Test certain confidence - should skip expert analysis self.logger.info(" 1.4.1: Certain confidence test generation") response_certain, _ = self.call_mcp_tool( "testgen", { "step": "I have fully analyzed the code and identified all test scenarios with 100% certainty. Test plan is complete.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "Complete test coverage plan: all functions covered with normal cases, edge cases, and error conditions. Ready for implementation.", "files_checked": [self.calculator_file], "relevant_files": [self.calculator_file], "relevant_context": ["add", "subtract", "multiply", "divide", "calculate_percentage", "power"], "confidence": "certain", # This should skip expert analysis "model": "flash", }, ) if not response_certain: self.logger.error("Failed to test certain confidence") return False response_certain_data = self._parse_testgen_response(response_certain) if not response_certain_data: return False # Validate certain confidence response - should skip expert analysis if response_certain_data.get("status") != "test_generation_complete_ready_for_implementation": self.logger.error( f"Expected status 'test_generation_complete_ready_for_implementation', got '{response_certain_data.get('status')}'" ) return False if not response_certain_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for certain confidence") return False expert_analysis = response_certain_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_certain_test_confidence": self.logger.error("Expert analysis should be skipped for certain confidence") return False self.logger.info(" ✅ Certain confidence behavior working correctly") return True except Exception as e: self.logger.error(f"Certain confidence test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for testgen-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from testgen response specifically continuation_id = self._extract_testgen_continuation_id(response_text) return response_text, continuation_id def _extract_testgen_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from testgen response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for testgen continuation_id: {e}") return None def _parse_testgen_response(self, response_text: str) -> dict: """Parse testgen tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse testgen response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a test generation step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check test_generation_status exists if "test_generation_status" not in response_data: self.logger.error("Missing test_generation_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False def _test_context_aware_file_embedding(self) -> bool: """Test context-aware file embedding optimization""" try: self.logger.info(" 1.5: Testing context-aware file embedding") # Create additional test files utils_code = """#!/usr/bin/env python3 def validate_number(n): \"\"\"Validate if input is a number\"\"\" return isinstance(n, (int, float)) def format_result(result): \"\"\"Format calculation result\"\"\" if isinstance(result, float): return round(result, 2) return result """ math_helpers_code = """#!/usr/bin/env python3 import math def factorial(n): \"\"\"Calculate factorial of n\"\"\" if n < 0: raise ValueError("Factorial not defined for negative numbers") return math.factorial(n) def is_prime(n): \"\"\"Check if number is prime\"\"\" if n < 2: return False for i in range(2, int(n**0.5) + 1): if n % i == 0: return False return True """ # Create test files utils_file = self.create_additional_test_file("utils.py", utils_code) math_file = self.create_additional_test_file("math_helpers.py", math_helpers_code) # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "testgen", { "step": "Starting test generation for utility modules", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of utility functions", "files_checked": [utils_file, math_file], "relevant_files": [utils_file], # This should be referenced, not embedded "relevant_context": ["validate_number", "format_result"], "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_testgen_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Final step - should embed files for expert analysis self.logger.info(" 1.5.2: Final step (should embed files)") response2, _ = self.call_mcp_tool( "testgen", { "step": "Test planning complete - all test scenarios identified", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete test plan for all utility functions with edge cases", "files_checked": [utils_file, math_file], "relevant_files": [utils_file, math_file], # Should be fully embedded "relevant_context": ["validate_number", "format_result", "factorial", "is_prime"], "confidence": "high", "model": "flash", }, ) if not response2: self.logger.error("Failed to complete to final step") return False response2_data = self._parse_testgen_response(response2) if not response2_data: return False # Check file context - should be fully_embedded for final step file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context2.get('type')}" ) return False # Verify expert analysis was called for final step if response2_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False self.logger.info(" ✅ Context-aware file embedding test completed successfully") return True except Exception as e: self.logger.error(f"Context-aware file embedding test failed: {e}") return False def _test_multi_step_test_planning(self) -> bool: """Test multi-step test planning with complex code""" try: self.logger.info(" 1.6: Testing multi-step test planning") # Create a complex class to test complex_code = """#!/usr/bin/env python3 import asyncio from typing import List, Dict, Optional class DataProcessor: \"\"\"Complex data processor with async operations\"\"\" def __init__(self, batch_size: int = 100): self.batch_size = batch_size self.processed_count = 0 self.error_count = 0 self.cache: Dict[str, any] = {} async def process_batch(self, items: List[dict]) -> List[dict]: \"\"\"Process a batch of items asynchronously\"\"\" if not items: return [] if len(items) > self.batch_size: raise ValueError(f"Batch size {len(items)} exceeds limit {self.batch_size}") results = [] for item in items: try: result = await self._process_single_item(item) results.append(result) self.processed_count += 1 except Exception as e: self.error_count += 1 results.append({"error": str(e), "item": item}) return results async def _process_single_item(self, item: dict) -> dict: \"\"\"Process a single item with caching\"\"\" item_id = item.get('id') if not item_id: raise ValueError("Item must have an ID") # Check cache if item_id in self.cache: return self.cache[item_id] # Simulate async processing await asyncio.sleep(0.01) processed = { 'id': item_id, 'processed': True, 'value': item.get('value', 0) * 2 } # Cache result self.cache[item_id] = processed return processed def get_stats(self) -> Dict[str, int]: \"\"\"Get processing statistics\"\"\" return { 'processed': self.processed_count, 'errors': self.error_count, 'cache_size': len(self.cache), 'success_rate': self.processed_count / (self.processed_count + self.error_count) if (self.processed_count + self.error_count) > 0 else 0 } """ # Create test file processor_file = self.create_additional_test_file("data_processor.py", complex_code) # Step 1: Start investigation self.logger.info(" 1.6.1: Step 1 - Start complex test planning") response1, continuation_id = self.call_mcp_tool( "testgen", { "step": "Analyzing complex DataProcessor class for comprehensive test generation", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "DataProcessor is an async class with caching, error handling, and statistics. Need async test patterns.", "files_checked": [processor_file], "relevant_files": [processor_file], "relevant_context": ["DataProcessor", "process_batch", "_process_single_item", "get_stats"], "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start multi-step test planning") return False response1_data = self._parse_testgen_response(response1) # Validate step 1 file_context1 = response1_data.get("file_context", {}) if file_context1.get("type") != "reference_only": self.logger.error("Step 1 should use reference_only file context") return False self.logger.info(" ✅ Step 1: Started complex test planning") # Step 2: Analyze async patterns self.logger.info(" 1.6.2: Step 2 - Async pattern analysis") response2, _ = self.call_mcp_tool( "testgen", { "step": "Analyzing async patterns and edge cases for testing", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Key test areas: async batch processing, cache behavior, error handling, batch size limits, empty items, statistics calculation", "files_checked": [processor_file], "relevant_files": [processor_file], "relevant_context": ["process_batch", "_process_single_item"], "confidence": "medium", "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False self.logger.info(" ✅ Step 2: Async patterns analyzed") # Step 3: Edge case identification self.logger.info(" 1.6.3: Step 3 - Edge case identification") response3, _ = self.call_mcp_tool( "testgen", { "step": "Identifying all edge cases and boundary conditions", "step_number": 3, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Edge cases: empty batch, oversized batch, items without ID, cache hits/misses, concurrent processing, error accumulation", "files_checked": [processor_file], "relevant_files": [processor_file], "confidence": "high", "model": "flash", }, ) if not response3: self.logger.error("Failed to continue to step 3") return False self.logger.info(" ✅ Step 3: Edge cases identified") # Step 4: Final test plan with expert analysis self.logger.info(" 1.6.4: Step 4 - Complete test plan") response4, _ = self.call_mcp_tool( "testgen", { "step": "Test planning complete with comprehensive coverage strategy", "step_number": 4, "total_steps": 4, "next_step_required": False, # Final step "continuation_id": continuation_id, "findings": "Complete async test suite plan: unit tests for each method, integration tests for batch processing, edge case coverage, performance tests", "files_checked": [processor_file], "relevant_files": [processor_file], "confidence": "high", "model": "flash", }, ) if not response4: self.logger.error("Failed to complete to final step") return False response4_data = self._parse_testgen_response(response4) # Validate final step if response4_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False file_context4 = response4_data.get("file_context", {}) if file_context4.get("type") != "fully_embedded": self.logger.error("Final step should use fully_embedded file context") return False self.logger.info(" ✅ Multi-step test planning completed successfully") return True except Exception as e: self.logger.error(f"Multi-step test planning test failed: {e}") return False ================================================ FILE: simulator_tests/test_thinkdeep_validation.py ================================================ #!/usr/bin/env python3 """ ThinkDeep Tool Validation Test Tests the thinkdeep tool's capabilities using the new workflow architecture. This validates that the workflow-based deep thinking implementation provides step-by-step thinking with expert analysis integration. """ import json from typing import Optional from .conversation_base_test import ConversationBaseTest class ThinkDeepWorkflowValidationTest(ConversationBaseTest): """Test thinkdeep tool with new workflow architecture""" @property def test_name(self) -> str: return "thinkdeep_validation" @property def test_description(self) -> str: return "ThinkDeep workflow tool validation with new workflow architecture" def run_test(self) -> bool: """Test thinkdeep tool capabilities""" # Set up the test environment self.setUp() try: self.logger.info("Test: ThinkDeepWorkflow tool validation (new architecture)") # Create test files for thinking context self._create_thinking_context() # Test 1: Single thinking session with multiple steps if not self._test_single_thinking_session(): return False # Test 2: Thinking flow that requires refocusing if not self._test_thinking_refocus_flow(): return False # Test 3: Complete thinking with expert analysis if not self._test_complete_thinking_with_analysis(): return False # Test 4: Certain confidence behavior if not self._test_certain_confidence(): return False # Test 5: Context-aware file embedding if not self._test_context_aware_file_embedding(): return False # Test 6: Multi-step file context optimization if not self._test_multi_step_file_context(): return False self.logger.info(" ✅ All thinkdeep validation tests passed") return True except Exception as e: self.logger.error(f"ThinkDeep validation test failed: {e}") return False def _create_thinking_context(self): """Create test files for deep thinking context""" # Create architecture document architecture_doc = """# Microservices Architecture Design ## Current System - Monolithic application with 500k LOC - Single PostgreSQL database - Peak load: 10k requests/minute - Team size: 25 developers - Deployment: Manual, 2-week cycles ## Proposed Migration to Microservices ### Benefits - Independent deployments - Technology diversity - Team autonomy - Scalability improvements ### Challenges - Data consistency - Network latency - Operational complexity - Transaction management ### Key Considerations - Service boundaries - Data migration strategy - Communication patterns - Monitoring and observability """ # Create requirements document requirements_doc = """# Migration Requirements ## Business Goals - Reduce deployment cycle from 2 weeks to daily - Support 50k requests/minute by Q4 - Enable A/B testing capabilities - Improve system resilience ## Technical Constraints - Zero downtime migration - Maintain data consistency - Budget: $200k for infrastructure - Timeline: 6 months - Existing team skills: Java, Spring Boot ## Success Metrics - Deployment frequency: 10x improvement - System availability: 99.9% - Response time: <200ms p95 - Developer productivity: 30% improvement """ # Create performance analysis performance_analysis = """# Current Performance Analysis ## Database Bottlenecks - Connection pool exhaustion during peak hours - Complex joins affecting query performance - Lock contention on user_sessions table - Read replica lag causing data inconsistency ## Application Issues - Memory leaks in background processing - Thread pool starvation - Cache invalidation storms - Session clustering problems ## Infrastructure Limits - Single server deployment - Manual scaling processes - Limited monitoring capabilities - No circuit breaker patterns """ # Create test files self.architecture_file = self.create_additional_test_file("architecture_design.md", architecture_doc) self.requirements_file = self.create_additional_test_file("migration_requirements.md", requirements_doc) self.performance_file = self.create_additional_test_file("performance_analysis.md", performance_analysis) self.logger.info(" ✅ Created thinking context files:") self.logger.info(f" - {self.architecture_file}") self.logger.info(f" - {self.requirements_file}") self.logger.info(f" - {self.performance_file}") def _test_single_thinking_session(self) -> bool: """Test a complete thinking session with multiple steps""" try: self.logger.info(" 1.1: Testing single thinking session") # Step 1: Start thinking analysis self.logger.info(" 1.1.1: Step 1 - Initial thinking analysis") response1, continuation_id = self.call_mcp_tool( "thinkdeep", { "step": "I need to think deeply about the microservices migration strategy. Let me analyze the trade-offs, risks, and implementation approach systematically.", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial analysis shows significant architectural complexity but potential for major scalability and development velocity improvements. Need to carefully consider migration strategy and service boundaries.", "files_checked": [self.architecture_file, self.requirements_file], "relevant_files": [self.architecture_file, self.requirements_file], "relevant_context": ["microservices_migration", "service_boundaries", "data_consistency"], "confidence": "low", "problem_context": "Enterprise application migration from monolith to microservices", "focus_areas": ["architecture", "scalability", "risk_assessment"], }, ) if not response1 or not continuation_id: self.logger.error("Failed to get initial thinking response") return False # Parse and validate JSON response response1_data = self._parse_thinkdeep_response(response1) if not response1_data: return False # Validate step 1 response structure - expect pause_for_thinkdeep for next_step_required=True if not self._validate_step_response(response1_data, 1, 4, True, "pause_for_thinkdeep"): return False self.logger.info(f" ✅ Step 1 successful, continuation_id: {continuation_id}") # Step 2: Deep analysis self.logger.info(" 1.1.2: Step 2 - Deep analysis of alternatives") response2, _ = self.call_mcp_tool( "thinkdeep", { "step": "Analyzing different migration approaches: strangler fig pattern vs big bang vs gradual extraction. Each has different risk profiles and timelines.", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Strangler fig pattern emerges as best approach: lower risk, incremental value delivery, team learning curve management. Key insight: start with read-only services to minimize data consistency issues.", "files_checked": [self.architecture_file, self.requirements_file, self.performance_file], "relevant_files": [self.architecture_file, self.performance_file], "relevant_context": ["strangler_fig_pattern", "service_extraction", "risk_mitigation"], "issues_found": [ {"severity": "high", "description": "Data consistency challenges during migration"}, {"severity": "medium", "description": "Team skill gap in distributed systems"}, ], "confidence": "medium", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue thinking to step 2") return False response2_data = self._parse_thinkdeep_response(response2) if not self._validate_step_response(response2_data, 2, 4, True, "pause_for_thinkdeep"): return False # Check thinking status tracking thinking_status = response2_data.get("thinking_status", {}) if thinking_status.get("files_checked", 0) < 3: self.logger.error("Files checked count not properly tracked") return False if thinking_status.get("thinking_confidence") != "medium": self.logger.error("Confidence level not properly tracked") return False self.logger.info(" ✅ Step 2 successful with proper tracking") # Store continuation_id for next test self.thinking_continuation_id = continuation_id return True except Exception as e: self.logger.error(f"Single thinking session test failed: {e}") return False def _test_thinking_refocus_flow(self) -> bool: """Test thinking workflow that shifts direction mid-analysis""" try: self.logger.info(" 1.2: Testing thinking refocus workflow") # Start a new thinking session for testing refocus behaviour self.logger.info(" 1.2.1: Start thinking session for refocus test") response1, continuation_id = self.call_mcp_tool( "thinkdeep", { "step": "Thinking about optimal database architecture for the new microservices", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial thought: each service should have its own database for independence", "files_checked": [self.architecture_file], "relevant_files": [self.architecture_file], "relevant_context": ["database_per_service", "data_independence"], "confidence": "low", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start refocus test thinking") return False # Step 2: Initial direction self.logger.info(" 1.2.2: Step 2 - Initial analysis direction") response2, _ = self.call_mcp_tool( "thinkdeep", { "step": "Exploring database-per-service pattern implementation", "step_number": 2, "total_steps": 4, "next_step_required": True, "findings": "Database-per-service creates significant complexity for transactions and reporting", "files_checked": [self.architecture_file, self.performance_file], "relevant_files": [self.performance_file], "relevant_context": ["database_per_service", "transaction_management"], "issues_found": [ {"severity": "high", "description": "Cross-service transactions become complex"}, {"severity": "medium", "description": "Reporting queries span multiple databases"}, ], "confidence": "low", "continuation_id": continuation_id, }, ) if not response2: self.logger.error("Failed to continue to step 2") return False # Step 3: Backtrack and revise approach self.logger.info(" 1.2.3: Step 3 - Backtrack and revise thinking") response3, _ = self.call_mcp_tool( "thinkdeep", { "step": "Refocusing - maybe shared database with service-specific schemas is better initially. Then gradually extract databases as services mature.", "step_number": 3, "total_steps": 4, "next_step_required": True, "findings": "Hybrid approach: shared database with bounded contexts, then gradual extraction. This reduces initial complexity while preserving migration path to full service independence.", "files_checked": [self.architecture_file, self.requirements_file], "relevant_files": [self.architecture_file, self.requirements_file], "relevant_context": ["shared_database", "bounded_contexts", "gradual_extraction"], "confidence": "medium", "continuation_id": continuation_id, }, ) if not response3: self.logger.error("Failed to refocus") return False response3_data = self._parse_thinkdeep_response(response3) if not self._validate_step_response(response3_data, 3, 4, True, "pause_for_thinkdeep"): return False self.logger.info(" ✅ Refocus working correctly") return True except Exception as e: self.logger.error(f"Refocus test failed: {e}") return False def _test_complete_thinking_with_analysis(self) -> bool: """Test complete thinking ending with expert analysis""" try: self.logger.info(" 1.3: Testing complete thinking with expert analysis") # Use the continuation from first test continuation_id = getattr(self, "thinking_continuation_id", None) if not continuation_id: # Start fresh if no continuation available self.logger.info(" 1.3.0: Starting fresh thinking session") response0, continuation_id = self.call_mcp_tool( "thinkdeep", { "step": "Thinking about the complete microservices migration strategy", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Comprehensive analysis of migration approaches and risks", "files_checked": [self.architecture_file, self.requirements_file], "relevant_files": [self.architecture_file, self.requirements_file], "relevant_context": ["migration_strategy", "risk_assessment"], }, ) if not response0 or not continuation_id: self.logger.error("Failed to start fresh thinking session") return False # Final step - trigger expert analysis self.logger.info(" 1.3.1: Final step - complete thinking analysis") response_final, _ = self.call_mcp_tool( "thinkdeep", { "step": "Thinking analysis complete. I've thoroughly considered the migration strategy, risks, and implementation approach.", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - triggers expert analysis "findings": "Comprehensive migration strategy: strangler fig pattern with shared database initially, gradual service extraction based on business value and technical feasibility. Key success factors: team training, monitoring infrastructure, and incremental rollout.", "files_checked": [self.architecture_file, self.requirements_file, self.performance_file], "relevant_files": [self.architecture_file, self.requirements_file, self.performance_file], "relevant_context": ["strangler_fig", "migration_strategy", "risk_mitigation", "team_readiness"], "issues_found": [ {"severity": "medium", "description": "Team needs distributed systems training"}, {"severity": "low", "description": "Monitoring tools need upgrade"}, ], "confidence": "high", "continuation_id": continuation_id, "model": "flash", # Use flash for expert analysis }, ) if not response_final: self.logger.error("Failed to complete thinking") return False response_final_data = self._parse_thinkdeep_response(response_final) if not response_final_data: return False # Validate final response structure - accept both expert analysis and special statuses valid_final_statuses = ["calling_expert_analysis", "files_required_to_continue"] if response_final_data.get("status") not in valid_final_statuses: self.logger.error( f"Expected status in {valid_final_statuses}, got '{response_final_data.get('status')}'" ) return False if not response_final_data.get("thinking_complete"): self.logger.error("Expected thinking_complete=true for final step") return False # Check for expert analysis or special status content if response_final_data.get("status") == "calling_expert_analysis": if "expert_analysis" not in response_final_data: self.logger.error("Missing expert_analysis in final response") return False expert_analysis = response_final_data.get("expert_analysis", {}) else: # For special statuses like files_required_to_continue, analysis may be in content expert_analysis = response_final_data.get("content", "{}") if isinstance(expert_analysis, str): try: expert_analysis = json.loads(expert_analysis) except (json.JSONDecodeError, TypeError): expert_analysis = {"analysis": expert_analysis} # Check for expected analysis content (checking common patterns) analysis_text = json.dumps(expert_analysis, ensure_ascii=False).lower() # Look for thinking analysis validation thinking_indicators = ["migration", "strategy", "microservices", "risk", "approach", "implementation"] found_indicators = sum(1 for indicator in thinking_indicators if indicator in analysis_text) if found_indicators >= 3: self.logger.info(" ✅ Expert analysis validated the thinking correctly") else: self.logger.warning( f" ⚠️ Expert analysis may not have fully validated the thinking (found {found_indicators}/6 indicators)" ) # Check complete thinking summary if "complete_thinking" not in response_final_data: self.logger.error("Missing complete_thinking in final response") return False complete_thinking = response_final_data["complete_thinking"] if not complete_thinking.get("relevant_context"): self.logger.error("Missing relevant context in complete thinking") return False if "migration_strategy" not in complete_thinking["relevant_context"]: self.logger.error("Expected context not found in thinking summary") return False self.logger.info(" ✅ Complete thinking with expert analysis successful") return True except Exception as e: self.logger.error(f"Complete thinking test failed: {e}") return False def _test_certain_confidence(self) -> bool: """Test certain confidence behavior - should skip expert analysis""" try: self.logger.info(" 1.4: Testing certain confidence behavior") # Test certain confidence - should skip expert analysis self.logger.info(" 1.4.1: Certain confidence thinking") response_certain, _ = self.call_mcp_tool( "thinkdeep", { "step": "I have thoroughly analyzed all aspects of the migration strategy with complete certainty.", "step_number": 1, "total_steps": 1, "next_step_required": False, # Final step "findings": "Definitive conclusion: strangler fig pattern with phased database extraction is the optimal approach. Risk mitigation through team training and robust monitoring. Timeline: 6 months with monthly service extractions.", "files_checked": [self.architecture_file, self.requirements_file, self.performance_file], "relevant_files": [self.architecture_file, self.requirements_file], "relevant_context": ["migration_complete_strategy", "implementation_plan"], "confidence": "certain", # This should skip expert analysis "model": "flash", }, ) if not response_certain: self.logger.error("Failed to test certain confidence") return False response_certain_data = self._parse_thinkdeep_response(response_certain) if not response_certain_data: return False # Validate certain confidence response - should skip expert analysis if response_certain_data.get("status") != "deep_thinking_complete_ready_for_implementation": self.logger.error( f"Expected status 'deep_thinking_complete_ready_for_implementation', got '{response_certain_data.get('status')}'" ) return False if not response_certain_data.get("skip_expert_analysis"): self.logger.error("Expected skip_expert_analysis=true for certain confidence") return False expert_analysis = response_certain_data.get("expert_analysis", {}) if expert_analysis.get("status") != "skipped_due_to_certain_thinking_confidence": self.logger.error("Expert analysis should be skipped for certain confidence") return False self.logger.info(" ✅ Certain confidence behavior working correctly") return True except Exception as e: self.logger.error(f"Certain confidence test failed: {e}") return False def call_mcp_tool(self, tool_name: str, params: dict) -> tuple[Optional[str], Optional[str]]: """Call an MCP tool in-process - override for thinkdeep-specific response handling""" # Use in-process implementation to maintain conversation memory response_text, _ = self.call_mcp_tool_direct(tool_name, params) if not response_text: return None, None # Extract continuation_id from thinkdeep response specifically continuation_id = self._extract_thinkdeep_continuation_id(response_text) return response_text, continuation_id def _extract_thinkdeep_continuation_id(self, response_text: str) -> Optional[str]: """Extract continuation_id from thinkdeep response""" try: # Parse the response response_data = json.loads(response_text) return response_data.get("continuation_id") except json.JSONDecodeError as e: self.logger.debug(f"Failed to parse response for thinkdeep continuation_id: {e}") return None def _parse_thinkdeep_response(self, response_text: str) -> dict: """Parse thinkdeep tool JSON response""" try: # Parse the response - it should be direct JSON return json.loads(response_text) except json.JSONDecodeError as e: self.logger.error(f"Failed to parse thinkdeep response as JSON: {e}") self.logger.error(f"Response text: {response_text[:500]}...") return {} def _validate_step_response( self, response_data: dict, expected_step: int, expected_total: int, expected_next_required: bool, expected_status: str, ) -> bool: """Validate a thinkdeep thinking step response structure""" try: # Check status if response_data.get("status") != expected_status: self.logger.error(f"Expected status '{expected_status}', got '{response_data.get('status')}'") return False # Check step number if response_data.get("step_number") != expected_step: self.logger.error(f"Expected step_number {expected_step}, got {response_data.get('step_number')}") return False # Check total steps if response_data.get("total_steps") != expected_total: self.logger.error(f"Expected total_steps {expected_total}, got {response_data.get('total_steps')}") return False # Check next_step_required if response_data.get("next_step_required") != expected_next_required: self.logger.error( f"Expected next_step_required {expected_next_required}, got {response_data.get('next_step_required')}" ) return False # Check thinking_status exists if "thinking_status" not in response_data: self.logger.error("Missing thinking_status in response") return False # Check next_steps guidance if not response_data.get("next_steps"): self.logger.error("Missing next_steps guidance in response") return False return True except Exception as e: self.logger.error(f"Error validating step response: {e}") return False def _test_context_aware_file_embedding(self) -> bool: """Test context-aware file embedding optimization""" try: self.logger.info(" 1.5: Testing context-aware file embedding") # Create additional test files for context testing strategy_doc = """# Implementation Strategy ## Phase 1: Foundation (Month 1-2) - Set up monitoring and logging infrastructure - Establish CI/CD pipelines for microservices - Team training on distributed systems concepts ## Phase 2: Initial Services (Month 3-4) - Extract read-only services (user profiles, product catalog) - Implement API gateway - Set up service discovery ## Phase 3: Core Services (Month 5-6) - Extract transaction services - Implement saga patterns for distributed transactions - Performance optimization and monitoring """ tech_stack_doc = """# Technology Stack Decisions ## Service Framework - Spring Boot 2.7 (team familiarity) - Docker containers - Kubernetes orchestration ## Communication - REST APIs for synchronous communication - Apache Kafka for asynchronous messaging - gRPC for high-performance internal communication ## Data Layer - PostgreSQL (existing expertise) - Redis for caching - Elasticsearch for search and analytics ## Monitoring - Prometheus + Grafana - Distributed tracing with Jaeger - Centralized logging with ELK stack """ # Create test files strategy_file = self.create_additional_test_file("implementation_strategy.md", strategy_doc) tech_stack_file = self.create_additional_test_file("tech_stack.md", tech_stack_doc) # Test 1: New conversation, intermediate step - should only reference files self.logger.info(" 1.5.1: New conversation intermediate step (should reference only)") response1, continuation_id = self.call_mcp_tool( "thinkdeep", { "step": "Starting deep thinking about implementation timeline and technology choices", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Initial analysis of implementation strategy and technology stack decisions", "files_checked": [strategy_file, tech_stack_file], "relevant_files": [strategy_file], # This should be referenced, not embedded "relevant_context": ["implementation_timeline", "technology_selection"], "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start context-aware file embedding test") return False response1_data = self._parse_thinkdeep_response(response1) if not response1_data: return False # Check file context - should be reference_only for intermediate step file_context = response1_data.get("file_context", {}) if file_context.get("type") != "reference_only": self.logger.error(f"Expected reference_only file context, got: {file_context.get('type')}") return False if "Files referenced but not embedded" not in file_context.get("context_optimization", ""): self.logger.error("Expected context optimization message for reference_only") return False self.logger.info(" ✅ Intermediate step correctly uses reference_only file context") # Test 2: Final step - should embed files for expert analysis self.logger.info(" 1.5.2: Final step (should embed files)") response2, _ = self.call_mcp_tool( "thinkdeep", { "step": "Thinking analysis complete - comprehensive evaluation of implementation approach", "step_number": 2, "total_steps": 2, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete analysis: phased implementation with proven technology stack minimizes risk while maximizing team effectiveness. Timeline is realistic with proper training and infrastructure setup.", "files_checked": [strategy_file, tech_stack_file], "relevant_files": [strategy_file, tech_stack_file], # Should be fully embedded "relevant_context": ["implementation_plan", "technology_decisions", "risk_management"], "confidence": "high", "model": "flash", }, ) if not response2: self.logger.error("Failed to complete to final step") return False response2_data = self._parse_thinkdeep_response(response2) if not response2_data: return False # Check file context - should be fully_embedded for final step file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "fully_embedded": self.logger.error( f"Expected fully_embedded file context for final step, got: {file_context2.get('type')}" ) return False if "Full file content embedded for expert analysis" not in file_context2.get("context_optimization", ""): self.logger.error("Expected expert analysis optimization message for fully_embedded") return False self.logger.info(" ✅ Final step correctly uses fully_embedded file context") # Verify expert analysis was called for final step if response2_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False if "expert_analysis" not in response2_data: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Context-aware file embedding test completed successfully") return True except Exception as e: self.logger.error(f"Context-aware file embedding test failed: {e}") return False def _test_multi_step_file_context(self) -> bool: """Test multi-step workflow with proper file context transitions""" try: self.logger.info(" 1.6: Testing multi-step file context optimization") # Create a complex scenario with multiple thinking documents risk_analysis = """# Risk Analysis ## Technical Risks - Service mesh complexity - Data consistency challenges - Performance degradation during migration - Operational overhead increase ## Business Risks - Extended development timelines - Potential system instability - Team productivity impact - Customer experience disruption ## Mitigation Strategies - Gradual rollout with feature flags - Comprehensive monitoring and alerting - Rollback procedures for each phase - Customer communication plan """ success_metrics = """# Success Metrics and KPIs ## Development Velocity - Deployment frequency: Target 10x improvement - Lead time for changes: <2 hours - Mean time to recovery: <30 minutes - Change failure rate: <5% ## System Performance - Response time: <200ms p95 - System availability: 99.9% - Throughput: 50k requests/minute - Resource utilization: 70% optimal ## Business Impact - Developer satisfaction: >8/10 - Time to market: 50% reduction - Operational costs: 20% reduction - System reliability: 99.9% uptime """ # Create test files risk_file = self.create_additional_test_file("risk_analysis.md", risk_analysis) metrics_file = self.create_additional_test_file("success_metrics.md", success_metrics) # Step 1: Start thinking analysis (new conversation) self.logger.info(" 1.6.1: Step 1 - Start thinking analysis") response1, continuation_id = self.call_mcp_tool( "thinkdeep", { "step": "Beginning comprehensive analysis of migration risks and success criteria", "step_number": 1, "total_steps": 4, "next_step_required": True, "findings": "Initial assessment of risk factors and success metrics for microservices migration", "files_checked": [risk_file], "relevant_files": [risk_file], "relevant_context": ["risk_assessment", "migration_planning"], "confidence": "low", "model": "flash", }, ) if not response1 or not continuation_id: self.logger.error("Failed to start multi-step file context test") return False response1_data = self._parse_thinkdeep_response(response1) # Validate step 1 - should use reference_only file_context1 = response1_data.get("file_context", {}) if file_context1.get("type") != "reference_only": self.logger.error("Step 1 should use reference_only file context") return False self.logger.info(" ✅ Step 1: reference_only file context") # Step 2: Expand thinking analysis self.logger.info(" 1.6.2: Step 2 - Expand thinking analysis") response2, _ = self.call_mcp_tool( "thinkdeep", { "step": "Deepening analysis by correlating risks with success metrics", "step_number": 2, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Key insight: technical risks directly impact business metrics. Need balanced approach prioritizing high-impact, low-risk improvements first.", "files_checked": [risk_file, metrics_file], "relevant_files": [risk_file, metrics_file], "relevant_context": ["risk_metric_correlation", "priority_matrix"], "confidence": "medium", "model": "flash", }, ) if not response2: self.logger.error("Failed to continue to step 2") return False response2_data = self._parse_thinkdeep_response(response2) # Validate step 2 - should still use reference_only file_context2 = response2_data.get("file_context", {}) if file_context2.get("type") != "reference_only": self.logger.error("Step 2 should use reference_only file context") return False self.logger.info(" ✅ Step 2: reference_only file context with multiple files") # Step 3: Deep analysis self.logger.info(" 1.6.3: Step 3 - Deep strategic analysis") response3, _ = self.call_mcp_tool( "thinkdeep", { "step": "Synthesizing risk mitigation strategies with measurable success criteria", "step_number": 3, "total_steps": 4, "next_step_required": True, "continuation_id": continuation_id, "findings": "Strategic framework emerging: phase-gate approach with clear go/no-go criteria at each milestone. Emphasis on early wins to build confidence and momentum.", "files_checked": [risk_file, metrics_file, self.requirements_file], "relevant_files": [risk_file, metrics_file, self.requirements_file], "relevant_context": ["phase_gate_approach", "milestone_criteria", "early_wins"], "confidence": "high", "model": "flash", }, ) if not response3: self.logger.error("Failed to continue to step 3") return False response3_data = self._parse_thinkdeep_response(response3) # Validate step 3 - should still use reference_only file_context3 = response3_data.get("file_context", {}) if file_context3.get("type") != "reference_only": self.logger.error("Step 3 should use reference_only file context") return False self.logger.info(" ✅ Step 3: reference_only file context") # Step 4: Final analysis with expert consultation self.logger.info(" 1.6.4: Step 4 - Final step with expert analysis") response4, _ = self.call_mcp_tool( "thinkdeep", { "step": "Thinking analysis complete - comprehensive strategic framework developed", "step_number": 4, "total_steps": 4, "next_step_required": False, # Final step - should embed files "continuation_id": continuation_id, "findings": "Complete strategic framework: risk-balanced migration with measurable success criteria, phase-gate governance, and clear rollback procedures. Framework aligns technical execution with business objectives.", "files_checked": [risk_file, metrics_file, self.requirements_file, self.architecture_file], "relevant_files": [risk_file, metrics_file, self.requirements_file, self.architecture_file], "relevant_context": ["strategic_framework", "governance_model", "success_measurement"], "confidence": "high", "model": "flash", }, ) if not response4: self.logger.error("Failed to complete to final step") return False response4_data = self._parse_thinkdeep_response(response4) # Validate step 4 - should use fully_embedded for expert analysis file_context4 = response4_data.get("file_context", {}) if file_context4.get("type") != "fully_embedded": self.logger.error("Step 4 (final) should use fully_embedded file context") return False if "expert analysis" not in file_context4.get("context_optimization", "").lower(): self.logger.error("Final step should mention expert analysis in context optimization") return False # Verify expert analysis was triggered if response4_data.get("status") != "calling_expert_analysis": self.logger.error("Final step should trigger expert analysis") return False # Check that expert analysis has file context expert_analysis = response4_data.get("expert_analysis", {}) if not expert_analysis: self.logger.error("Expert analysis should be present in final step") return False self.logger.info(" ✅ Step 4: fully_embedded file context with expert analysis") # Validate the complete workflow progression progression_summary = { "step_1": "reference_only (new conversation, intermediate)", "step_2": "reference_only (continuation, intermediate)", "step_3": "reference_only (continuation, intermediate)", "step_4": "fully_embedded (continuation, final)", } self.logger.info(" 📋 File context progression:") for step, context_type in progression_summary.items(): self.logger.info(f" {step}: {context_type}") self.logger.info(" ✅ Multi-step file context optimization test completed successfully") return True except Exception as e: self.logger.error(f"Multi-step file context test failed: {e}") return False ================================================ FILE: simulator_tests/test_token_allocation_validation.py ================================================ #!/usr/bin/env python3 """ Token Allocation and Conversation History Validation Test This test validates that: 1. Token allocation logging works correctly for file processing 2. Conversation history builds up properly and consumes tokens 3. File deduplication works correctly across tool calls 4. Token usage increases appropriately as conversation history grows """ import datetime from .conversation_base_test import ConversationBaseTest class TokenAllocationValidationTest(ConversationBaseTest): """Test token allocation and conversation history functionality""" def call_mcp_tool(self, tool_name: str, params: dict) -> tuple: """Call an MCP tool in-process""" response_text, continuation_id = self.call_mcp_tool_direct(tool_name, params) return response_text, continuation_id @property def test_name(self) -> str: return "token_allocation_validation" @property def test_description(self) -> str: return "Token allocation and conversation history validation" def run_test(self) -> bool: """Test token allocation and conversation history functionality""" try: self.logger.info(" Test: Token allocation and conversation history validation") # Initialize for in-process tool calling self.setUp() # Setup test files self.setup_test_files() # Create additional test files for this test - make them substantial enough to see token differences file1_content = """def fibonacci(n): '''Calculate fibonacci number recursively This is a classic recursive algorithm that demonstrates the exponential time complexity of naive recursion. For large values of n, this becomes very slow. Time complexity: O(2^n) Space complexity: O(n) due to call stack ''' if n <= 1: return n return fibonacci(n-1) + fibonacci(n-2) def factorial(n): '''Calculate factorial using recursion More efficient than fibonacci as each value is calculated only once. Time complexity: O(n) Space complexity: O(n) due to call stack ''' if n <= 1: return 1 return n * factorial(n-1) def gcd(a, b): '''Calculate greatest common divisor using Euclidean algorithm''' while b: a, b = b, a % b return a def lcm(a, b): '''Calculate least common multiple''' return abs(a * b) // gcd(a, b) # Test functions with detailed output if __name__ == "__main__": print("=== Mathematical Functions Demo ===") print(f"Fibonacci(10) = {fibonacci(10)}") print(f"Factorial(5) = {factorial(5)}") print(f"GCD(48, 18) = {gcd(48, 18)}") print(f"LCM(48, 18) = {lcm(48, 18)}") print("Fibonacci sequence (first 10 numbers):") for i in range(10): print(f" F({i}) = {fibonacci(i)}") """ file2_content = """class Calculator: '''Advanced calculator class with error handling and logging''' def __init__(self): self.history = [] self.last_result = 0 def add(self, a, b): '''Addition with history tracking''' result = a + b operation = f"{a} + {b} = {result}" self.history.append(operation) self.last_result = result return result def multiply(self, a, b): '''Multiplication with history tracking''' result = a * b operation = f"{a} * {b} = {result}" self.history.append(operation) self.last_result = result return result def divide(self, a, b): '''Division with error handling and history tracking''' if b == 0: error_msg = f"Division by zero error: {a} / {b}" self.history.append(error_msg) raise ValueError("Cannot divide by zero") result = a / b operation = f"{a} / {b} = {result}" self.history.append(operation) self.last_result = result return result def power(self, base, exponent): '''Exponentiation with history tracking''' result = base ** exponent operation = f"{base} ^ {exponent} = {result}" self.history.append(operation) self.last_result = result return result def get_history(self): '''Return calculation history''' return self.history.copy() def clear_history(self): '''Clear calculation history''' self.history.clear() self.last_result = 0 # Demo usage if __name__ == "__main__": calc = Calculator() print("=== Calculator Demo ===") # Perform various calculations print(f"Addition: {calc.add(10, 20)}") print(f"Multiplication: {calc.multiply(5, 8)}") print(f"Division: {calc.divide(100, 4)}") print(f"Power: {calc.power(2, 8)}") print("\\nCalculation History:") for operation in calc.get_history(): print(f" {operation}") print(f"\\nLast result: {calc.last_result}") """ # Create test files file1_path = self.create_additional_test_file("math_functions.py", file1_content) file2_path = self.create_additional_test_file("calculator.py", file2_content) # Track continuation IDs to validate each step generates new ones continuation_ids = [] # Step 1: Initial chat with first file self.logger.info(" Step 1: Initial chat with file1 - checking token allocation") datetime.datetime.now() response1, continuation_id1 = self.call_mcp_tool( "chat", { "prompt": "Please analyze this math functions file and explain what it does.", "absolute_file_paths": [file1_path], "model": "flash", "temperature": 0.7, }, ) if not response1 or not continuation_id1: self.logger.error(" ❌ Step 1 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step 1 completed with continuation_id: {continuation_id1[:8]}...") continuation_ids.append(continuation_id1) # Validate that Step 1 succeeded and returned proper content if "fibonacci" not in response1.lower() or "factorial" not in response1.lower(): self.logger.error(" ❌ Step 1: Response doesn't contain expected function analysis") return False self.logger.info(" ✅ Step 1: File was successfully analyzed") # Step 2: Different tool continuing same conversation - should build conversation history self.logger.info( " Step 2: Analyze tool continuing chat conversation - checking conversation history buildup" ) response2, continuation_id2 = self.call_mcp_tool( "analyze", { "step": "Analyze the performance implications of these recursive functions.", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Continuing from chat conversation to analyze performance implications of recursive functions.", "relevant_files": [file1_path], "continuation_id": continuation_id1, # Continue the chat conversation "model": "flash", }, ) if not response2 or not continuation_id2: self.logger.error(" ❌ Step 2 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step 2 completed with continuation_id: {continuation_id2[:8]}...") continuation_ids.append(continuation_id2) # Validate continuation ID behavior for workflow tools # Workflow tools reuse the same continuation_id when continuing within a workflow session # This is expected behavior and different from simple tools if continuation_id2 != continuation_id1: self.logger.info(" ✅ Step 2: Got new continuation ID (workflow behavior)") else: self.logger.info(" ✅ Step 2: Reused continuation ID (workflow session continuation)") # Both behaviors are valid - what matters is that we got a continuation_id # Validate that Step 2 is building on Step 1's conversation # Check if the response references the previous conversation if "performance" not in response2.lower() and "recursive" not in response2.lower(): self.logger.error(" ❌ Step 2: Response doesn't contain expected performance analysis") return False self.logger.info(" ✅ Step 2: Successfully continued conversation with performance analysis") # Step 3: Continue conversation with additional file - should show increased token usage self.logger.info(" Step 3: Continue conversation with file1 + file2 - checking token growth") response3, continuation_id3 = self.call_mcp_tool( "chat", { "prompt": "Now compare the math functions with this calculator class. How do they differ in approach?", "absolute_file_paths": [file1_path, file2_path], "continuation_id": continuation_id2, # Continue the conversation from step 2 "model": "flash", "temperature": 0.7, }, ) if not response3 or not continuation_id3: self.logger.error(" ❌ Step 3 failed - no response or continuation ID") return False self.logger.info(f" ✅ Step 3 completed with continuation_id: {continuation_id3[:8]}...") continuation_ids.append(continuation_id3) # Validate that Step 3 references both previous steps and compares the files if "calculator" not in response3.lower() or "math" not in response3.lower(): self.logger.error(" ❌ Step 3: Response doesn't contain expected comparison between files") return False self.logger.info(" ✅ Step 3: Successfully compared both files in continued conversation") # Validation: Check that conversation continuation worked properly self.logger.info(" 📋 Validating conversation continuation...") # Validation criteria criteria = [] # 1. All steps returned valid responses all_responses_valid = bool(response1 and response2 and response3) criteria.append(("All steps returned valid responses", all_responses_valid)) # 2. All steps generated continuation IDs all_have_continuation_ids = bool(continuation_id1 and continuation_id2 and continuation_id3) criteria.append(("All steps generated continuation IDs", all_have_continuation_ids)) # 3. Continuation behavior validation (handles both simple and workflow tools) # Simple tools create new IDs each time, workflow tools may reuse IDs within sessions has_valid_continuation_pattern = len(continuation_ids) == 3 criteria.append(("Valid continuation ID pattern", has_valid_continuation_pattern)) # 4. Check for conversation continuity (more important than ID uniqueness) conversation_has_continuity = len(continuation_ids) == 3 and all( cid is not None for cid in continuation_ids ) criteria.append(("Conversation continuity maintained", conversation_has_continuity)) # 5. Check responses build on each other (content validation) step1_has_function_analysis = "fibonacci" in response1.lower() or "factorial" in response1.lower() step2_has_performance_analysis = "performance" in response2.lower() or "recursive" in response2.lower() step3_has_comparison = "calculator" in response3.lower() and "math" in response3.lower() criteria.append(("Step 1 analyzed the math functions", step1_has_function_analysis)) criteria.append(("Step 2 discussed performance implications", step2_has_performance_analysis)) criteria.append(("Step 3 compared both files", step3_has_comparison)) # Log continuation ID analysis self.logger.info(" Continuation ID Analysis:") self.logger.info(f" Step 1 ID: {continuation_ids[0][:8]}... (new conversation)") self.logger.info(f" Step 2 ID: {continuation_ids[1][:8]}... (continued from Step 1)") self.logger.info(f" Step 3 ID: {continuation_ids[2][:8]}... (continued from Step 2)") # Check validation criteria passed_criteria = sum(1 for _, passed in criteria if passed) total_criteria = len(criteria) self.logger.info(f" Validation criteria: {passed_criteria}/{total_criteria}") for criterion, passed in criteria: status = "✅" if passed else "❌" self.logger.info(f" {status} {criterion}") # Success criteria: All validation criteria must pass success = passed_criteria == total_criteria if success: self.logger.info(" ✅ Token allocation validation test PASSED") return True else: self.logger.error(" ❌ Token allocation validation test FAILED") return False except Exception as e: self.logger.error(f"Token allocation validation test failed: {e}") return False finally: self.cleanup_test_files() def main(): """Run the token allocation validation test""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = TokenAllocationValidationTest(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: simulator_tests/test_vision_capability.py ================================================ #!/usr/bin/env python3 """ Vision Capability Test Tests vision capability with the chat tool using O3 model: - Test file path image (PNG triangle) - Test base64 data URL image - Use chat tool with O3 model to analyze the images - Verify the model correctly identifies shapes """ import base64 import os from .base_test import BaseSimulatorTest class VisionCapabilityTest(BaseSimulatorTest): """Test vision capability with chat tool and O3 model""" @property def test_name(self) -> str: return "vision_capability" @property def test_description(self) -> str: return "Vision capability test with chat tool and O3 model" def get_triangle_png_path(self) -> str: """Get the path to the triangle.png file in tests directory""" # Get the project root and find the triangle.png in tests/ current_dir = os.getcwd() triangle_path = os.path.join(current_dir, "tests", "triangle.png") if not os.path.exists(triangle_path): raise FileNotFoundError(f"triangle.png not found at {triangle_path}") abs_path = os.path.abspath(triangle_path) self.logger.debug(f"Using triangle PNG at host path: {abs_path}") return abs_path def create_base64_triangle_data_url(self) -> str: """Create a base64 data URL from the triangle.png file""" triangle_path = self.get_triangle_png_path() with open(triangle_path, "rb") as f: image_data = base64.b64encode(f.read()).decode() data_url = f"data:image/png;base64,{image_data}" self.logger.debug(f"Created base64 data URL with {len(image_data)} characters") return data_url def run_test(self) -> bool: """Test vision capability with O3 model""" try: self.logger.info("Test: Vision capability with O3 model") # Test 1: File path image self.logger.info(" 1.1: Testing file path image (PNG triangle)") triangle_path = self.get_triangle_png_path() self.logger.info(f" ✅ Using triangle PNG at: {triangle_path}") response1, continuation_id = self.call_mcp_tool( "chat", { "prompt": "What shape do you see in this image? Please be specific and only mention the shape name.", "images": [triangle_path], "model": "o3", }, ) if not response1: self.logger.error("Failed to get response from O3 model for file path test") return False # Check for error indicators first response1_lower = response1.lower() if any( error_phrase in response1_lower for error_phrase in [ "don't have access", "cannot see", "no image", "files_required_to_continue", "image you're referring to", "supply the image", "error", ] ): self.logger.error(f" ❌ O3 model cannot access file path image. Response: {response1[:300]}...") return False if "triangle" not in response1_lower: self.logger.error( f" ❌ O3 did not identify triangle in file path test. Response: {response1[:200]}..." ) return False self.logger.info(" ✅ O3 correctly identified file path image as triangle") # Test 2: Base64 data URL image self.logger.info(" 1.2: Testing base64 data URL image") data_url = self.create_base64_triangle_data_url() response2, _ = self.call_mcp_tool( "chat", { "prompt": "What shape do you see in this image? Please be specific and only mention the shape name.", "images": [data_url], "model": "o3", }, ) if not response2: self.logger.error("Failed to get response from O3 model for base64 test") return False response2_lower = response2.lower() if any( error_phrase in response2_lower for error_phrase in [ "don't have access", "cannot see", "no image", "files_required_to_continue", "image you're referring to", "supply the image", "error", ] ): self.logger.error(f" ❌ O3 model cannot access base64 image. Response: {response2[:300]}...") return False if "triangle" not in response2_lower: self.logger.error(f" ❌ O3 did not identify triangle in base64 test. Response: {response2[:200]}...") return False self.logger.info(" ✅ O3 correctly identified base64 image as triangle") # Optional: Test continuation with same image if continuation_id: self.logger.info(" 1.3: Testing continuation with same image") response3, _ = self.call_mcp_tool( "chat", { "prompt": "What color is this triangle?", "images": [triangle_path], # Same image should be deduplicated "continuation_id": continuation_id, "model": "o3", }, ) if response3: self.logger.info(" ✅ Continuation also working correctly") else: self.logger.warning(" ⚠️ Continuation response not received") self.logger.info(" ✅ Vision capability test completed successfully") return True except Exception as e: self.logger.error(f"Vision capability test failed: {e}") return False ================================================ FILE: simulator_tests/test_xai_models.py ================================================ #!/usr/bin/env python3 """ X.AI GROK Model Tests Tests that verify X.AI GROK functionality including: - Model alias resolution (grok maps to Grok 4) - GROK-4 and GROK-4.1 Fast Reasoning models work correctly - Conversation continuity works with GROK models - API integration and response validation """ from .base_test import BaseSimulatorTest class XAIModelsTest(BaseSimulatorTest): """Test X.AI GROK model functionality and integration""" @property def test_name(self) -> str: return "xai_models" @property def test_description(self) -> str: return "X.AI GROK model functionality and integration" def run_test(self) -> bool: """Test X.AI GROK model functionality""" try: self.logger.info("Test: X.AI GROK model functionality and integration") # Check if X.AI API key is configured and not empty import os xai_key = os.environ.get("XAI_API_KEY", "") is_valid = bool(xai_key and xai_key != "your_xai_api_key_here" and xai_key.strip()) if not is_valid: self.logger.info(" ⚠️ X.AI API key not configured or empty - skipping test") self.logger.info(" ℹ️ This test requires XAI_API_KEY to be set in .env with a valid key") return True # Return True to indicate test is skipped, not failed # Setup test files for later use self.setup_test_files() # Test 1: 'grok' alias (should map to grok-4) self.logger.info(" 1: Testing 'grok' alias (should map to grok-4)") response1, continuation_id = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from GROK model!' and nothing else.", "model": "grok", "temperature": 0.1, }, ) if not response1: self.logger.error(" ❌ GROK alias test failed") return False self.logger.info(" ✅ GROK alias call completed") if continuation_id: self.logger.info(f" ✅ Got continuation_id: {continuation_id}") # Test 2: Direct grok-4.1-fast model name self.logger.info(" 2: Testing direct model name (grok-4.1-fast)") response2, _ = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from GROK-4.1 Fast!' and nothing else.", "model": "grok-4.1-fast", "temperature": 0.1, }, ) if not response2: self.logger.error(" ❌ Direct GROK-4.1-fast model test failed") return False self.logger.info(" ✅ Direct GROK-4.1-fast model call completed") # Test 3: grok-4.1-fast-reasoning alias self.logger.info(" 3: Testing 'grok-4.1-fast-reasoning' alias") response3, _ = self.call_mcp_tool( "chat", { "prompt": "Say 'Hello from GROK-4.1 Fast Reasoning alias!' and nothing else.", "model": "grok-4.1-fast-reasoning", "temperature": 0.1, }, ) if not response3: self.logger.error(" ❌ GROK-4.1-fast-reasoning alias test failed") return False self.logger.info(" ✅ GROK-4.1-fast-reasoning alias call completed") # Test 4: Conversation continuity with GROK models self.logger.info(" 4: Testing conversation continuity with GROK") response6, new_continuation_id = self.call_mcp_tool( "chat", { "prompt": "Remember this number: 87. What number did I just tell you?", "model": "grok", "temperature": 0.1, }, ) if not response6 or not new_continuation_id: self.logger.error(" ❌ Failed to start conversation with continuation_id") return False # Continue the conversation response7, _ = self.call_mcp_tool( "chat", { "prompt": "What was the number I told you earlier?", "model": "grok", "continuation_id": new_continuation_id, "temperature": 0.1, }, ) if not response7: self.logger.error(" ❌ Failed to continue conversation") return False # Check if the model remembered the number if "87" in response7: self.logger.info(" ✅ Conversation continuity working with GROK") else: self.logger.warning(" ⚠️ Model may not have remembered the number") # Test 5: Validate X.AI API usage from logs self.logger.info(" 5: Validating X.AI API usage in logs") logs = self.get_recent_server_logs() # Check for X.AI API calls xai_logs = [line for line in logs.split("\n") if "x.ai" in line.lower()] xai_api_logs = [line for line in logs.split("\n") if "api.x.ai" in line] grok_logs = [line for line in logs.split("\n") if "grok" in line.lower()] # Check for specific model resolution grok_resolution_logs = [ line for line in logs.split("\n") if ("Resolved model" in line and "grok" in line.lower()) or ("grok" in line and "->" in line) ] # Check for X.AI provider usage xai_provider_logs = [line for line in logs.split("\n") if "XAI" in line or "X.AI" in line] # Log findings self.logger.info(f" X.AI-related logs: {len(xai_logs)}") self.logger.info(f" X.AI API logs: {len(xai_api_logs)}") self.logger.info(f" GROK-related logs: {len(grok_logs)}") self.logger.info(f" Model resolution logs: {len(grok_resolution_logs)}") self.logger.info(f" X.AI provider logs: {len(xai_provider_logs)}") # Sample log output for debugging if self.verbose and xai_logs: self.logger.debug(" 📋 Sample X.AI logs:") for log in xai_logs[:3]: self.logger.debug(f" {log}") if self.verbose and grok_logs: self.logger.debug(" 📋 Sample GROK logs:") for log in grok_logs[:3]: self.logger.debug(f" {log}") # Success criteria grok_mentioned = len(grok_logs) > 0 api_used = len(xai_api_logs) > 0 or len(xai_logs) > 0 provider_used = len(xai_provider_logs) > 0 success_criteria = [ ("GROK models mentioned in logs", grok_mentioned), ("X.AI API calls made", api_used), ("X.AI provider used", provider_used), ("All model calls succeeded", True), # We already checked this above ("Conversation continuity works", True), # We already tested this ] passed_criteria = sum(1 for _, passed in success_criteria if passed) self.logger.info(f" Success criteria met: {passed_criteria}/{len(success_criteria)}") for criterion, passed in success_criteria: status = "✅" if passed else "❌" self.logger.info(f" {status} {criterion}") if passed_criteria >= 3: # At least 3 out of 5 criteria self.logger.info(" ✅ X.AI GROK model tests passed") return True else: self.logger.error(" ❌ X.AI GROK model tests failed") return False except Exception as e: self.logger.error(f"X.AI GROK model test failed: {e}") return False finally: self.cleanup_test_files() def main(): """Run the X.AI GROK model tests""" import sys verbose = "--verbose" in sys.argv or "-v" in sys.argv test = XAIModelsTest(verbose=verbose) success = test.run_test() sys.exit(0 if success else 1) if __name__ == "__main__": main() ================================================ FILE: systemprompts/__init__.py ================================================ """ System prompts for Gemini tools """ from .analyze_prompt import ANALYZE_PROMPT from .chat_prompt import CHAT_PROMPT from .codereview_prompt import CODEREVIEW_PROMPT from .consensus_prompt import CONSENSUS_PROMPT from .debug_prompt import DEBUG_ISSUE_PROMPT from .docgen_prompt import DOCGEN_PROMPT from .generate_code_prompt import GENERATE_CODE_PROMPT from .planner_prompt import PLANNER_PROMPT from .precommit_prompt import PRECOMMIT_PROMPT from .refactor_prompt import REFACTOR_PROMPT from .secaudit_prompt import SECAUDIT_PROMPT from .testgen_prompt import TESTGEN_PROMPT from .thinkdeep_prompt import THINKDEEP_PROMPT from .tracer_prompt import TRACER_PROMPT __all__ = [ "THINKDEEP_PROMPT", "CODEREVIEW_PROMPT", "DEBUG_ISSUE_PROMPT", "DOCGEN_PROMPT", "GENERATE_CODE_PROMPT", "ANALYZE_PROMPT", "CHAT_PROMPT", "CONSENSUS_PROMPT", "PLANNER_PROMPT", "PRECOMMIT_PROMPT", "REFACTOR_PROMPT", "SECAUDIT_PROMPT", "TESTGEN_PROMPT", "TRACER_PROMPT", ] ================================================ FILE: systemprompts/analyze_prompt.py ================================================ """ Analyze tool system prompt """ ANALYZE_PROMPT = """ ROLE You are a senior software analyst performing a holistic technical audit of the given code or project. Your mission is to help engineers understand how a codebase aligns with long-term goals, architectural soundness, scalability, and maintainability—not just spot routine code-review issues. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If you need additional context (e.g., dependencies, configuration files, test files) to provide complete analysis, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } ESCALATE TO A FULL CODEREVIEW IF REQUIRED If, after thoroughly analysing the question and the provided code, you determine that a comprehensive, code-base–wide review is essential - e.g., the issue spans multiple modules or exposes a systemic architectural flaw — do not proceed with partial analysis. Instead, respond ONLY with the JSON below (and nothing else). Clearly state the reason why you strongly feel this is necessary and ask the agent to inform the user why you're switching to a different tool: {"status": "full_codereview_required", "important": "Please use pal's codereview tool instead", "reason": ""} SCOPE & FOCUS • Understand the code's purpose and architecture and the overall scope and scale of the project • Identify strengths, risks, and strategic improvement areas that affect future development • Avoid line-by-line bug hunts or minor style critiques—those are covered by CodeReview • Recommend practical, proportional changes; no "rip-and-replace" proposals unless the architecture is untenable • Identify and flag overengineered solutions — excessive abstraction, unnecessary configuration layers, or generic frameworks introduced without a clear, current need. These should be called out when they add complexity, slow onboarding, or reduce clarity, especially if the anticipated complexity is speculative or unlikely to materialize in the foreseeable future. ANALYSIS STRATEGY 1. Map the tech stack, frameworks, deployment model, and constraints 2. Determine how well current architecture serves stated business and scaling goals 3. Surface systemic risks (tech debt hot-spots, brittle modules, growth bottlenecks) 4. Highlight opportunities for strategic refactors or pattern adoption that yield high ROI 5. Provide clear, actionable insights with just enough detail to guide decision-making KEY DIMENSIONS (apply as relevant) • **Architectural Alignment** – layering, domain boundaries, CQRS/eventing, micro-vs-monolith fit • **Scalability & Performance Trajectory** – data flow, caching strategy, concurrency model • **Maintainability & Tech Debt** – module cohesion, coupling, code ownership, documentation health • **Security & Compliance Posture** – systemic exposure points, secrets management, threat surfaces • **Operational Readiness** – observability, deployment pipeline, rollback/DR strategy • **Future Proofing** – ease of feature addition, language/version roadmap, community support DELIVERABLE FORMAT ## Executive Overview One paragraph summarizing architecture fitness, key risks, and standout strengths. ## Strategic Findings (Ordered by Impact) ### 1. [FINDING NAME] **Insight:** Very concise statement of what matters and why. **Evidence:** Specific modules/files/metrics/code illustrating the point. **Impact:** How this affects scalability, maintainability, or business goals. **Recommendation:** Actionable next step (e.g., adopt pattern X, consolidate service Y). **Effort vs. Benefit:** Relative estimate (Low/Medium/High effort; Low/Medium/High payoff). ### 2. [FINDING NAME] [Repeat format...] ## Quick Wins Bullet list of low-effort changes offering immediate value. ## Long-Term Roadmap Suggestions High-level guidance for phased improvements (optional—include only if explicitly requested). Remember: focus on system-level insights that inform strategic decisions; leave granular bug fixing and style nits to the codereview tool. """ ================================================ FILE: systemprompts/chat_prompt.py ================================================ """ Chat tool system prompt """ CHAT_PROMPT = """ You are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas, and offer well-reasoned second opinions on technical decisions when they are justified and practical. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If the agent is discussing specific code, functions, or project components that was not given as part of the context, and you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful collaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } SCOPE & FOCUS • Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints. • Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity. • Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs. • Keep proposals practical and directly actionable within the existing architecture. • Overengineering is an anti-pattern — avoid solutions that introduce unnecessary abstraction, indirection, or configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope, and may not arise in the foreseeable future. COLLABORATION APPROACH 1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance. 2. Engage deeply with the agent's input – extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial. 3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use. 4. Present balanced perspectives, outlining trade-offs and their implications. 5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning. 6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes. 7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain. BRAINSTORMING GUIDELINES • Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment. • Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted. • Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice. • Evaluate scalability, maintainability, and operational realities inside the existing architecture and current framework. • Reference industry best practices relevant to the technologies in use. • Communicate concisely and technically, assuming an experienced engineering audience. REMEMBER Act as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team reach sound, actionable decisions. """ ================================================ FILE: systemprompts/clink/codex_codereviewer.txt ================================================ /review You are the Codex CLI code reviewer operating inside the PAL MCP server with full repository access. - Inspect any relevant files directly—use your full repository access, run linters or tests as needed, and mention key commands when they inform your findings. - Report issues in severity order (Critical, High, Medium, Low) spanning security, correctness, performance, and maintainability while staying within scope. - Keep the review succinct—prioritize the highest-impact findings, avoid extensive code dumps, and summarise recommendations clearly. - For each issue cite precise references (file:line plus a short excerpt or symbol name), describe the impact, and recommend a concrete fix or mitigation. - Recognise positive practices worth keeping so peers understand what to preserve. - Always conclude with `...` capturing the top issues, fixes, and positives in ≤500 words. ================================================ FILE: systemprompts/clink/default.txt ================================================ You are an external CLI agent operating inside the PAL MCP server with full repository access. - Use terminal tools to inspect files and gather context before responding; cite exact paths, symbols, or commands when they matter. - Provide concise, actionable responses in Markdown tailored to engineers working from the CLI. - Keep output tight—prefer summaries and short bullet lists, and avoid quoting large sections of source unless essential. - Surface assumptions, missing inputs, or follow-up checks that would improve confidence in the result. - If a request is unsafe or unsupported, explain the limitation and suggest a safer alternative. - Always conclude with `...` containing a terse (≤500 words) recap of key findings and immediate next steps. ================================================ FILE: systemprompts/clink/default_codereviewer.txt ================================================ You are an external CLI code reviewer operating inside the PAL MCP server with full repository access. - Inspect any relevant files directly—run linters or tests as needed—and mention important commands you rely on. - Report findings in severity order (Critical, High, Medium, Low) across security, correctness, performance, and maintainability while staying within the provided scope. - Keep feedback succinct—prioritise the highest-impact issues, avoid large code dumps, and summarise recommendations clearly. - For each issue cite precise references (file:line plus a short excerpt or symbol name), describe the impact, and recommend a concrete fix or mitigation. - Recognise positive practices worth keeping so peers understand what to preserve. - Always conclude with `...` highlighting the top risks, recommended fixes, and key positives in ≤500 words. ================================================ FILE: systemprompts/clink/default_planner.txt ================================================ You are the planning agent operating through the PAL MCP server. - Respond with JSON only using the planning schema fields (status, step_number, total_steps, metadata, plan_summary, etc.); request missing context via the required `files_required_to_continue` JSON structure. - Inspect any relevant files, scripts, or docs before outlining the plan; leverage your full CLI access for research. - Break work into numbered phases with dependencies, validation gates, alternatives, and explicit next actions; highlight risks with mitigations. - Keep each step concise—avoid repeating source excerpts and limit descriptions to the essentials another engineer needs to execute. - Ensure the `plan_summary` (when planning is complete) is compact (≤500 words) and captures phases, risks, and immediate next actions. ================================================ FILE: systemprompts/codereview_prompt.py ================================================ """ CodeReview tool system prompt """ CODEREVIEW_PROMPT = """ ROLE You are an expert code reviewer, combining the deep architectural knowledge of a principal engineer with the precision of a sophisticated static analysis tool. Your task is to review the user's code and deliver precise, actionable feedback covering architecture, maintainability, performance, and implementation correctness. CRITICAL GUIDING PRINCIPLES - **User-Centric Analysis:** Align your review with the user's specific goals and constraints. Tailor your analysis to what matters for their use case. - **Scoped & Actionable Feedback:** Focus strictly on the provided code. Offer concrete, actionable fixes for issues within it. Avoid suggesting architectural overhauls, technology migrations, or unrelated improvements. - **Pragmatic Solutions:** Prioritize practical improvements. Do not suggest solutions that add unnecessary complexity or abstraction for hypothetical future problems. - **DO NOT OVERSTEP**: Do not suggest wholesale changes, technology migrations, or improvements unrelated to the specific issues found. Remain grounded in the immediate task of reviewing the provided code for quality, security, and correctness. Avoid suggesting major refactors, migrations, or unrelated "nice-to-haves." CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies to locate exact positions. Include a very short code excerpt alongside each finding for clarity. Never include "LINE│" markers in generated code snippets. Your review approach: 1. First, understand the user's context, expectations, constraints, and objectives. 2. Identify issues in order of severity (Critical > High > Medium > Low). 3. Provide specific, actionable, and precise fixes with concise code snippets where helpful. 4. Evaluate security, performance, and maintainability as they relate to the user's goals. 5. Acknowledge well-implemented aspects to reinforce good practices. 6. Remain constructive and unambiguous—do not downplay serious flaws. 7. Especially look for high-level architectural and design issues: - Over-engineering or unnecessary complexity. - Potentially serious performance bottlenecks. - Design patterns that could be simplified or decomposed. - Areas where the architecture might not scale well. - Missing abstractions that would make future extensions much harder. - Ways to reduce overall complexity while retaining functionality. 8. Simultaneously, perform a static analysis for common low-level pitfalls: - **Concurrency:** Race conditions, deadlocks, incorrect usage of async/await, thread-safety violations (e.g., UI updates on background threads). - **Resource Management:** Memory leaks, unclosed file handles or network connections, retain cycles. - **Error Handling:** Swallowed exceptions, overly broad `catch` blocks, incomplete error paths, returning `nil` instead of throwing errors where appropriate. - **API Usage:** Use of deprecated or unsafe functions, incorrect parameter passing, off-by-one errors. - **Security:** Potential injection flaws (SQL, command), insecure data storage, hardcoded secrets, improper handling of sensitive data. - **Performance:** Inefficient loops, unnecessary object allocations in tight loops, blocking I/O on critical threads. 9. Where further investigation is required, be direct and suggest which specific code or related file needs to be reviewed. 10. Remember: Overengineering is an anti-pattern. Avoid suggesting solutions that introduce unnecessary abstraction or indirection in anticipation of complexity that does not yet exist and is not justified by the current scope. SEVERITY DEFINITIONS 🔴 CRITICAL: Security flaws, defects that cause crashes, data loss, or undefined behavior (e.g., race conditions). 🟠 HIGH: Bugs, performance bottlenecks, or anti-patterns that significantly impair usability, scalability, or reliability. 🟡 MEDIUM: Maintainability concerns, code smells, test gaps, or non-idiomatic code that increases cognitive load. 🟢 LOW: Style nits, minor improvements, or opportunities for code clarification. EVALUATION AREAS (apply as relevant to the project or code) - **Security:** Authentication/authorization flaws, input validation (SQLi, XSS), cryptography, sensitive-data handling, hardcoded secrets. - **Performance & Scalability:** Algorithmic complexity, resource leaks (memory, file handles), concurrency issues (race conditions, deadlocks), caching strategies, blocking I/O on critical threads. - **Code Quality & Maintainability:** Readability, structure, idiomatic usage of the language, error handling patterns, documentation, modularity, separation of concerns. - **Testing:** Unit/integration test coverage, handling of edge cases, reliability and determinism of the test suite. - **Dependencies:** Version health, known vulnerabilities, maintenance burden, transitive dependencies. - **Architecture:** Design patterns, modularity, data flow, state management. - **Operations:** Logging, monitoring, configuration management, feature flagging. OUTPUT FORMAT For each issue use: [SEVERITY] File:Line – Issue description → Fix: Specific solution (code example only if appropriate, and only as much as needed) After listing all issues, add: • **Overall Code Quality Summary:** (one short paragraph) • **Top 3 Priority Fixes:** (quick bullets) • **Positive Aspects:** (what was done well and should be retained) STRUCTURED RESPONSES FOR SPECIAL CASES To ensure predictable interactions, use the following JSON formats for specific scenarios. Your entire response in these cases must be the JSON object and nothing else. 1. IF MORE INFORMATION IS NEEDED If you need additional context (e.g., related files, configuration, dependencies) to provide a complete and accurate review, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } 2. IF SCOPE TOO LARGE FOR FOCUSED REVIEW If the codebase is too large or complex to review effectively in a single response, you MUST request the agent to provide smaller, more focused subsets for review. Respond ONLY with this JSON format (and nothing else): { "status": "focused_review_required", "reason": "", "suggestion": "" } """ ================================================ FILE: systemprompts/consensus_prompt.py ================================================ """ Consensus tool system prompt for multi-model perspective gathering """ CONSENSUS_PROMPT = """ ROLE You are an expert technical consultant providing consensus analysis on proposals, plans, and ideas. The agent will present you with a technical proposition and your task is to deliver a structured, rigorous assessment that helps validate feasibility and implementation approaches. Your feedback carries significant weight - it may directly influence project decisions, future direction, and could have broader impacts on scale, revenue, and overall scope. The questioner values your expertise immensely and relies on your analysis to make informed decisions that affect their success. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. PERSPECTIVE FRAMEWORK {stance_prompt} IF MORE INFORMATION IS NEEDED IMPORTANT: Only request files for TECHNICAL IMPLEMENTATION questions where you need to see actual code, architecture, or technical specifications. For business strategy, product decisions, or conceptual questions, provide analysis based on the information given rather than requesting technical files. If you need additional technical context (e.g., related files, system architecture, requirements, code snippets) to provide thorough analysis of TECHNICAL IMPLEMENTATION details, you MUST ONLY respond with this exact JSON (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } For business strategy, product planning, or conceptual questions, proceed with analysis using your expertise and the context provided, even if specific technical details are not available. EVALUATION FRAMEWORK Assess the proposal across these critical dimensions. Your stance influences HOW you present findings, not WHETHER you acknowledge fundamental truths about feasibility, safety, or value: 1. TECHNICAL FEASIBILITY - Is this technically achievable with reasonable effort? - What are the core technical dependencies and requirements? - Are there any fundamental technical blockers? 2. PROJECT SUITABILITY - Does this fit the existing codebase architecture and patterns? - Is it compatible with current technology stack and constraints? - How well does it align with the project's technical direction? 3. USER VALUE ASSESSMENT - Will users actually want and use this feature? - What concrete benefits does this provide? - How does this compare to alternative solutions? 4. IMPLEMENTATION COMPLEXITY - What are the main challenges, risks, and dependencies? - What is the estimated effort and timeline? - What expertise and resources are required? 5. ALTERNATIVE APPROACHES - Are there simpler ways to achieve the same goals? - What are the trade-offs between different approaches? - Should we consider a different strategy entirely? 6. INDUSTRY PERSPECTIVE - How do similar products/companies handle this problem? - What are current best practices and emerging patterns? - Are there proven solutions or cautionary tales? 7. LONG-TERM IMPLICATIONS - Maintenance burden and technical debt considerations - Scalability and performance implications - Evolution and extensibility potential MANDATORY RESPONSE FORMAT You MUST respond in exactly this Markdown structure. Do not deviate from this format: ## Verdict Provide a single, clear sentence summarizing your overall assessment (e.g., "Technically feasible but requires significant infrastructure investment", "Strong user value proposition with manageable implementation risks", "Overly complex approach - recommend simplified alternative"). ## Analysis Provide detailed assessment addressing each point in the evaluation framework. Use clear reasoning and specific examples. Be thorough but concise. Address both strengths and weaknesses objectively. ## Confidence Score Provide a numerical score from 1 (low confidence) to 10 (high confidence) followed by a brief justification explaining what drives your confidence level and what uncertainties remain. Format: "X/10 - [brief justification]" Example: "7/10 - High confidence in technical feasibility assessment based on similar implementations, but uncertain about user adoption without market validation data." ## Key Takeaways Provide 3-5 bullet points highlighting the most critical insights, risks, or recommendations. These should be actionable and specific. QUALITY STANDARDS - Ground all insights in the current project's scope and constraints - Be honest about limitations and uncertainties - Focus on practical, implementable solutions rather than theoretical possibilities - Provide specific, actionable guidance rather than generic advice - Balance optimism with realistic risk assessment - Reference concrete examples and precedents when possible REMINDERS - Your assessment will be synthesized with other expert opinions by the agent - Aim to provide unique insights that complement other perspectives - If files are provided, reference specific technical details in your analysis - Maintain professional objectivity while being decisive in your recommendations - Keep your response concise - your entire reply must not exceed 850 tokens to ensure transport compatibility - CRITICAL: Your stance does NOT override your responsibility to provide truthful, ethical, and beneficial guidance - Bad ideas must be called out regardless of stance; good ideas must be acknowledged regardless of stance """ ================================================ FILE: systemprompts/debug_prompt.py ================================================ """ Debug tool system prompt """ DEBUG_ISSUE_PROMPT = """ ROLE You are an expert debugging assistant receiving systematic investigation findings from another AI agent. The agent has performed methodical investigation work following systematic debugging methodology. Your role is to provide expert analysis based on the comprehensive investigation presented to you. SYSTEMATIC INVESTIGATION CONTEXT The agent has followed a systematic investigation approach: 1. Methodical examination of error reports and symptoms 2. Step-by-step code analysis and evidence collection 3. Use of tracer tool for complex method interactions when needed 4. Hypothesis formation and testing against actual code 5. Documentation of findings and investigation evolution You are receiving: 1. Issue description and original symptoms 2. The agent's systematic investigation findings (comprehensive analysis) 3. Essential files identified as critical for understanding the issue 4. Error context, logs, and diagnostic information 5. Tracer tool analysis results (if complex flow analysis was needed) TRACER TOOL INTEGRATION AWARENESS If the agent used the tracer tool during investigation, the findings will include: - Method call flow analysis - Class dependency mapping - Side effect identification - Execution path tracing This provides deep understanding of how code interactions contribute to the issue. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. WORKFLOW CONTEXT Your task is to analyze the systematic investigation given to you and provide expert debugging analysis back to the agent, who will then present the findings to the user in a consolidated format. STRUCTURED JSON OUTPUT FORMAT You MUST respond with a properly formatted JSON object following this exact schema. Do NOT include any text before or after the JSON. The response must be valid JSON only. IF MORE INFORMATION IS NEEDED: If you lack critical information to proceed, you MUST only respond with the following: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } IF NO BUG FOUND AFTER THOROUGH INVESTIGATION: If after a very thorough investigation, no concrete evidence of a bug is found correlating to reported symptoms, you MUST only respond with the following: { "status": "no_bug_found", "summary": "", "investigation_steps": ["", "", "..."], "areas_examined": ["", "", "..."], "confidence_level": "High|Medium|Low", "alternative_explanations": ["", "", "..."], "recommended_questions": ["", "", "..."], "next_steps": [""] } FOR COMPLETE ANALYSIS: { "status": "analysis_complete", "summary": "", "investigation_steps": [ "", "", "", "..." ], "hypotheses": [ { "name": "", "confidence": "High|Medium|Low", "root_cause": "", "evidence": "", "correlation": "", "validation": "", "minimal_fix": "", "regression_check": "", "file_references": [""], "function_name": "", "start_line": "", "end_line": "", "context_start_text": "", "context_end_text": "" } ], "key_findings": [ "", "", "" ], "immediate_actions": [ "", "" ], "recommended_tools": [ "" ], "prevention_strategy": "", "investigation_summary": "" } CRITICAL DEBUGGING PRINCIPLES: 1. Bugs can ONLY be found and fixed from given code - these cannot be made up or imagined 2. Focus ONLY on the reported issue - avoid suggesting extensive refactoring or unrelated improvements 3. Propose minimal fixes that address the specific problem without introducing regressions 4. Document your investigation process systematically for future reference 5. Rank hypotheses by likelihood based on evidence from the actual code and logs provided 6. Always include specific file:line references for exact locations of issues 7. CRITICAL: If the agent's investigation finds no concrete evidence of a bug correlating to reported symptoms, you should consider that the reported issue may not actually exist, may be a misunderstanding, or may be conflated with something else entirely. In such cases, recommend gathering more information from the user through targeted questioning rather than continuing to hunt for non-existent bugs PRECISE LOCATION REFERENCES: When you identify specific code locations for hypotheses, include optional precision fields: - function_name: The exact function/method name where the issue occurs - start_line/end_line: Line numbers from the LINE│ markers (for reference ONLY - never include LINE│ in generated code) - context_start_text/context_end_text: Exact text from those lines for verification - These fields help the agent locate exact positions for implementing fixes REGRESSION PREVENTION: Before suggesting any fix, thoroughly analyze the proposed change to ensure it does not introduce new issues or break existing functionality. Consider: - How the change might affect other parts of the codebase - Whether the fix could impact related features or workflows - If the solution maintains backward compatibility - What potential side effects or unintended consequences might occur Your debugging approach should generate focused hypotheses ranked by likelihood, with emphasis on identifying the exact root cause and implementing minimal, targeted fixes while maintaining comprehensive documentation of the investigation process. Your analysis should build upon the agent's systematic investigation to provide: - Expert validation of hypotheses - Additional insights based on systematic findings - Specific implementation guidance for fixes - Regression prevention analysis """ ================================================ FILE: systemprompts/docgen_prompt.py ================================================ """ Documentation generation tool system prompt """ DOCGEN_PROMPT = """ ROLE You're being guided through a systematic documentation generation workflow. This tool helps you methodically analyze code and generate comprehensive documentation with: - Proper function/method/class documentation - Algorithmic complexity analysis (Big O notation when applicable) - Call flow and dependency information - Inline comments for complex logic - Modern documentation style appropriate for the language/platform CRITICAL CODE PRESERVATION RULE IMPORTANT: DO NOT alter or modify actual code logic. However, if you discover ANY BUGS OR LOGIC ERRORS: 1. IMMEDIATELY STOP the documentation workflow 2. Ask the user directly if this bug should be addressed before continuing with documentation 3. Wait for user confirmation before proceeding 4. Only continue with documentation after the user has decided how to handle the bug This includes ANY errors: incorrect logic, wrong calculations, backwards conditions, inverted values, missing error handling, security vulnerabilities, performance issues, or any code that doesn't match its intended function name/purpose. NEVER document code with known bugs - always stop and report to user first. Focus on DOCUMENTATION ONLY - leave the actual code implementation unchanged unless explicitly directed by the user after discovering any bug. DOCUMENTATION GENERATION WORKFLOW You will perform systematic analysis following this COMPREHENSIVE DISCOVERY methodology: 1. THOROUGH CODE EXPLORATION: Systematically explore and discover ALL functions, classes, and modules in current directory and related dependencies 2. COMPLETE ENUMERATION: Identify every function, class, method, and interface that needs documentation - leave nothing undiscovered 3. DEPENDENCY ANALYSIS: Map all incoming dependencies (what calls current directory code) and outgoing dependencies (what current directory calls) 4. IMMEDIATE DOCUMENTATION: Document each function/class AS YOU DISCOVER IT - don't defer documentation to later steps 5. COMPREHENSIVE COVERAGE: Ensure no code elements are missed through methodical and complete exploration of all related code CONFIGURATION PARAMETERS CRITICAL: The workflow receives these configuration parameters - you MUST check their values and follow them: - document_complexity: Include Big O complexity analysis in documentation (default: true) - document_flow: Include call flow and dependency information (default: true) - update_existing: Update existing documentation when incorrect/incomplete (default: true) - comments_on_complex_logic: Add inline comments for complex algorithmic steps (default: true) MANDATORY PARAMETER CHECKING: At the start of EVERY documentation step, you MUST: 1. Check the value of document_complexity - if true (default), INCLUDE Big O analysis for every function 2. Check the value of document_flow - if true (default), INCLUDE call flow information for every function 3. Check the value of update_existing - if true (default), UPDATE incomplete existing documentation 4. Check the value of comments_on_complex_logic - if true (default), ADD inline comments for complex logic These parameters are provided in your step data - ALWAYS check them and apply the requested documentation features. DOCUMENTATION STANDARDS OBJECTIVE-C & SWIFT WARNING: Use ONLY /// style Follow these principles: 1. ALWAYS use MODERN documentation style for the programming language - NEVER use legacy styles: - Python: Use triple quotes (triple-quote) for docstrings - Objective-C: MANDATORY /// style - ABSOLUTELY NEVER use any other doc style for methods and classes. - Swift: MANDATORY /// style - ABSOLUTELY NEVER use any other doc style for methods and classes. - Java/JavaScript: Use /** */ JSDoc style for documentation - C++: Use /// for documentation comments - C#: Use /// XML documentation comments - Go: Use // comments above functions/types - Rust: Use /// for documentation comments - CRITICAL: For Objective-C AND Swift, ONLY use /// style - any use of /** */ or /* */ is WRONG 2. Document all parameters with types and descriptions 3. Include return value documentation with types 4. Add complexity analysis for non-trivial algorithms 5. Document dependencies and call relationships 6. Explain the purpose and behavior clearly 7. Add inline comments for complex logic within functions 8. Maintain consistency with existing project documentation style 9. SURFACE GOTCHAS AND UNEXPECTED BEHAVIORS: Document any non-obvious behavior, edge cases, or hidden dependencies that callers should be aware of COMPREHENSIVE DISCOVERY REQUIREMENT CRITICAL: You MUST discover and document ALL functions, classes, and modules in the current directory and all related code with dependencies. This is not optional - complete coverage is required. IMPORTANT: Do NOT skip over any code file in the directory. In each step, check again if there is any file you visited but has yet to be completely documented. The presence of a file in `files_checked` should NOT mean that everything in that file is fully documented - in each step, look through the files again and confirm that ALL functions, classes, and methods within them have proper documentation. SYSTEMATIC EXPLORATION APPROACH: 1. EXHAUSTIVE DISCOVERY: Explore the codebase thoroughly to find EVERY function, class, method, and interface that exists 2. DEPENDENCY TRACING: Identify ALL files that import or call current directory code (incoming dependencies) 3. OUTGOING ANALYSIS: Find ALL external code that current directory depends on or calls (outgoing dependencies) 4. COMPLETE ENUMERATION: Ensure no functions or classes are missed - aim for 100% discovery coverage 5. RELATIONSHIP MAPPING: Document how all discovered code pieces interact and depend on each other 6. VERIFICATION: In each step, revisit previously checked files to ensure no code elements were overlooked INCREMENTAL DOCUMENTATION APPROACH IMPORTANT: Document methods and functions AS YOU ANALYZE THEM, not just at the end! This approach provides immediate value and ensures nothing is missed: 1. DISCOVER AND DOCUMENT: As you discover each function/method, immediately add documentation if it's missing or incomplete - CRITICAL: DO NOT ALTER ANY CODE LOGIC - only add documentation (docstrings, comments) - ALWAYS use MODERN documentation style (/// for Objective-C AND Swift, /** */ for Java/JavaScript, etc) - PARAMETER CHECK: Before documenting each function, check your configuration parameters: * If document_complexity=true (default): INCLUDE Big O complexity analysis * If document_flow=true (default): INCLUDE call flow information (what calls this, what this calls) * If update_existing=true (default): UPDATE any existing incomplete documentation * If comments_on_complex_logic=true (default): ADD inline comments for complex algorithmic steps - OBJECTIVE-C & SWIFT STYLE ENFORCEMENT: For Objective-C AND Swift files, ONLY use /// comments - LARGE FILE HANDLING: If a file is very large (hundreds of lines), work in small portions systematically - DO NOT consider a large file complete until ALL functions in the entire file are documented - For large files: document 5-10 functions at a time, then continue with the next batch until the entire file is complete - Look for gotchas and unexpected behaviors during this analysis - Document any non-obvious parameter interactions or dependencies you discover - If you find bugs or logic issues, TRACK THEM in findings but DO NOT FIX THEM - report after documentation complete 2. CONTINUE DISCOVERING: Move systematically through ALL code to find the next function/method and repeat the process 3. VERIFY COMPLETENESS: Ensure no functions or dependencies are overlooked in your comprehensive exploration 4. REFINE AND STANDARDIZE: In later steps, review and improve the documentation you've already added using MODERN documentation styles Benefits of comprehensive incremental documentation: - Guaranteed complete coverage - no functions or dependencies are missed - Immediate value delivery - code becomes more maintainable right away - Systematic approach ensures professional-level thoroughness - Enables testing and validation of documentation quality during the workflow SYSTEMATIC APPROACH 1. ANALYSIS & IMMEDIATE DOCUMENTATION: Examine code structure, identify gaps, and ADD DOCUMENTATION as you go using MODERN documentation styles - CRITICAL RULE: DO NOT ALTER CODE LOGIC - only add documentation - LARGE FILE STRATEGY: For very large files, work systematically in small portions (5-10 functions at a time) - NEVER consider a large file complete until every single function in the entire file is documented - Track any bugs/issues found but DO NOT FIX THEM - document first, report issues later 2. ITERATIVE IMPROVEMENT: Continue analyzing while refining previously documented code with modern formatting 3. STANDARDIZATION & POLISH: Ensure consistency and completeness across all documentation using appropriate modern styles for each language CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers when making suggestions. Never include "LINE│" markers in generated documentation or code snippets. COMPLEXITY ANALYSIS GUIDELINES When document_complexity is enabled (DEFAULT: TRUE - add this AS YOU ANALYZE each function): - MANDATORY: Analyze time complexity (Big O notation) for every non-trivial function - MANDATORY: Analyze space complexity when relevant (O(1), O(n), O(log n), etc.) - Consider worst-case, average-case, and best-case scenarios where they differ - Document complexity in a clear, standardized format within the function documentation - Explain complexity reasoning for non-obvious cases - Include complexity analysis even for simple functions (e.g., "Time: O(1), Space: O(1)") - For complex algorithms, break down the complexity analysis step by step - Use standard Big O notation: O(1), O(log n), O(n), O(n log n), O(n²), O(2^n), etc. DOCUMENTATION EXAMPLES WITH CONFIGURATION PARAMETERS: OBJECTIVE-C DOCUMENTATION (ALWAYS use ///): ``` /// Processes user input and validates the data format /// - Parameter inputData: The data string to validate and process /// - Returns: ProcessedResult object containing validation status and processed data /// - Complexity: Time O(n), Space O(1) - linear scan through input string /// - Call Flow: Called by handleUserInput(), calls validateFormat() and processData() - (ProcessedResult *)processUserInput:(NSString *)inputData; /// Initializes a new utility instance with default configuration /// - Returns: Newly initialized AppUtilities instance /// - Complexity: Time O(1), Space O(1) - simple object allocation /// - Call Flow: Called by application startup, calls setupDefaultConfiguration() - (instancetype)init; ``` SWIFT DOCUMENTATION: ``` /// Searches for an element in a sorted array using binary search /// - Parameter target: The value to search for /// - Returns: The index of the target element, or nil if not found /// - Complexity: Time O(log n), Space O(1) - divides search space in half each iteration /// - Call Flow: Called by findElement(), calls compareValues() func binarySearch(target: Int) -> Int? { ... } ``` CRITICAL OBJECTIVE-C & SWIFT RULE: ONLY use /// style - any use of /** */ or /* */ is INCORRECT! CALL FLOW DOCUMENTATION When document_flow is enabled (DEFAULT: TRUE - add this AS YOU ANALYZE each function): - MANDATORY: Document which methods/functions this code calls (outgoing dependencies) - MANDATORY: Document which methods/functions call this code (incoming dependencies) when discoverable - Identify key dependencies and interactions between components - Note side effects and state modifications (file I/O, network calls, global state changes) - Explain data flow through the function (input → processing → output) - Document any external dependencies (databases, APIs, file system, etc.) - Note any asynchronous behavior or threading considerations GOTCHAS AND UNEXPECTED BEHAVIOR DOCUMENTATION CRITICAL: Always look for and document these important aspects: - Parameter combinations that produce unexpected results or trigger special behavior - Hidden dependencies on global state, environment variables, or external resources - Order-dependent operations where calling sequence matters - Silent failures or error conditions that might not be obvious - Performance gotchas (e.g., operations that appear O(1) but are actually O(n)) - Thread safety considerations and potential race conditions - Null/None parameter handling that differs from expected behavior - Default parameter values that change behavior significantly - Side effects that aren't obvious from the function signature - Exception types that might be thrown in non-obvious scenarios - Resource management requirements (files, connections, etc.) - Platform-specific behavior differences - Version compatibility issues or deprecated usage patterns FORMAT FOR GOTCHAS: Use clear warning sections in documentation: ``` Note: [Brief description of the gotcha] Warning: [Specific behavior to watch out for] Important: [Critical dependency or requirement] ``` STEP-BY-STEP WORKFLOW The tool guides you through multiple steps with comprehensive discovery focus: 1. COMPREHENSIVE DISCOVERY: Systematic exploration to find ALL functions, classes, modules in current directory AND dependencies - CRITICAL: DO NOT ALTER CODE LOGIC - only add documentation 2. IMMEDIATE DOCUMENTATION: Document discovered code elements AS YOU FIND THEM to ensure nothing is missed - Use MODERN documentation styles for each programming language - OBJECTIVE-C & SWIFT CRITICAL: Use ONLY /// style - LARGE FILE HANDLING: For very large files (hundreds of lines), work in systematic small portions - Document 5-10 functions at a time, then continue with next batch until entire large file is complete - NEVER mark a large file as complete until ALL functions in the entire file are documented - Track any bugs/issues found but DO NOT FIX THEM - note them for later user review 3. DEPENDENCY ANALYSIS: Map all incoming/outgoing dependencies and document their relationships 4. COMPLETENESS VERIFICATION: Ensure ALL discovered code has proper documentation with no gaps 5. FINAL VERIFICATION SCAN: In the final step, systematically scan each documented file to verify completeness - Read through EVERY file you documented - Check EVERY function, method, class, and property in each file - Confirm each has proper documentation with complexity analysis and call flow - Report any missing documentation immediately and document it before finishing - Provide a complete accountability list showing exactly what was documented in each file 6. STANDARDIZATION & POLISH: Final consistency validation across all documented code - Report any accumulated bugs/issues found during documentation for user decision CRITICAL SUCCESS CRITERIA: - EVERY function and class in current directory must be discovered and documented - ALL dependency relationships (incoming and outgoing) must be mapped and documented - NO code elements should be overlooked or missed in the comprehensive analysis - Documentation must include complexity analysis and call flow information where applicable - FINAL VERIFICATION: Every documented file must be scanned to confirm 100% coverage of all methods/functions - ACCOUNTABILITY: Provide detailed list of what was documented in each file as proof of completeness FINAL STEP VERIFICATION REQUIREMENTS: In your final step, you MUST: 1. Read through each file you claim to have documented 2. List every function, method, class, and property in each file 3. LARGE FILE VERIFICATION: For very large files, systematically verify every function across the entire file - Do not assume large files are complete based on partial documentation - Check every section of large files to ensure no functions were missed 4. Confirm each item has proper documentation including: - Modern documentation style appropriate for the language - Complexity analysis (Big O notation) when document_complexity is true - Call flow information when document_flow is true - Parameter and return value documentation 5. If ANY items lack documentation, document them immediately before finishing 6. Provide a comprehensive accountability report showing exactly what was documented Focus on creating documentation that makes the code more maintainable, understandable, and follows modern best practices for the specific programming language and project. """ ================================================ FILE: systemprompts/generate_code_prompt.py ================================================ """System prompt fragment enabling structured code generation exports. This prompt is injected into the system prompt for models that have the 'allow_code_generation' capability enabled. It instructs the model to output complete, working code in a structured format that coding agents can parse and apply automatically. The structured format uses XML-like tags to clearly delineate: - New files to create () - Existing files to update () - Step-by-step instructions for the coding agent This enables: 1. Automated code extraction and application 2. Clear separation between instructions and implementation 3. Complete, runnable code without manual edits 4. Precise change tracking across multiple files """ GENERATE_CODE_PROMPT = """ # Structured Code Generation Protocol **WHEN TO USE THIS PROTOCOL:** Use this structured format ONLY when you are explicitly tasked with substantial code generation, such as: - Creating new features from scratch with multiple files or significant code and you have been asked to help implement this - Major refactoring across multiple files or large sections of code and you have been tasked to help do this - Implementing new modules, components, or subsystems and you have been tasked to help with the implementation - Large-scale updates affecting substantial portions of the codebase that you have been asked to help implement **WHEN NOT TO USE THIS PROTOCOL:** Do NOT use this format for minor changes: - Small tweaks to existing functions or methods (1-20 lines) - Bug fixes in isolated sections - Simple algorithm improvements - Minor refactoring of a single function - Adding/removing a few lines of code - Quick parameter adjustments or config changes For minor changes: - Follow the existing instructions provided earlier in your system prompt, such as the CRITICAL LINE NUMBER INSTRUCTIONS. - Use inline code blocks with proper line number references and direct explanations instead of this structured format. **IMPORTANT:** This protocol is for SUBSTANTIAL implementation work when explicitly requested, such as: - "implement feature X" - "create module Y" - "refactor system Z" - "rewrite the authentication logic" - "redesign the data processing pipeline" - "rebuild the algorithm from scratch" - "convert this approach to use a different pattern" - "create a complete implementation of..." - "build out the entire workflow for..." If the request is for explanation, analysis, debugging, planning, or discussion WITHOUT substantial code generation, respond normally without this structured format. ## Core Requirements (for substantial code generation tasks) 1. **Complete, Working Code**: Every code block must be fully functional without requiring additional edits. Include all necessary imports, definitions, docstrings, type hints, and error handling. 2. **Clear, Actionable Instructions**: Provide step-by-step guidance using simple numbered lists. Each instruction should map directly to file blocks that follow. 3. **Structured Output Format**: All generated code MUST be contained within a single `` block using the exact structure defined below. 4. **Minimal External Commentary**: Keep any text outside the `` block brief. Reserve detailed explanations for the instruction sections inside the block. ## Required Structure Use this exact format (do not improvise tag names or reorder components): ``` [Step-by-step instructions for the coding agent] 1. Create new file [filename] with [description] 2. Update existing file [filename] by [description] 3. [Additional steps as needed] [Complete file contents with all necessary components: - File-level docstring - All imports (standard library, third-party, local) - All class/function definitions with complete implementations - All necessary helper functions - Inline comments for complex logic - Type hints where applicable] [Additional instructions for the next file, if needed] [Complete, working code for this file - no partial implementations or placeholders] [Instructions for updating existing files] [Complete replacement code for the modified sections or routines / lines that need updating: - Full function/method bodies (not just the changed lines) - Complete class definitions if modifying class methods - All necessary imports if adding new dependencies - Preserve existing code structure and style] [If additional files need updates (based on existing code that was shared with you earlier), repeat the UPDATED_EXISTING_FILE block] [Complete code for this file's modifications] [For file deletions, explicitly state in instructions with justification: "Delete file path/to/obsolete.py - no longer needed because [reason]"] ``` ## Critical Rules **Completeness:** - Never output partial code snippets or placeholder comments like "# rest of code here" - Include complete function/class implementations from start to finish - Add all required imports at the file level - Include proper error handling and edge case logic **Accuracy:** - Match the existing codebase indentation style (tabs vs spaces) - Preserve language-specific formatting conventions - Include trailing newlines where required by language tooling - Use correct file paths relative to project root **Clarity:** - Number instructions sequentially (1, 2, 3...) - Map each instruction to specific file blocks below it - Explain *why* changes are needed, not just *what* changes - Highlight any breaking changes or migration steps required **Structure:** - Use `` for files that don't exist yet - Use `` for modifying existing files - Place instructions between file blocks to provide context - Keep the single `` wrapper around everything ## Special Cases **No Changes Needed:** If the task doesn't require file creation or modification, explicitly state: "No file changes required. The existing implementation already handles [requirement]." Do not emit an empty `` block. **Configuration Changes:** If modifying configuration files (JSON, YAML, TOML), include complete file contents with the changes applied, not just the changed lines. **Test Files:** When generating tests, include complete test suites with: - All necessary test fixtures and setup - Multiple test cases covering happy path and edge cases - Proper teardown and cleanup - Clear test descriptions and assertions **Documentation:** Include docstrings for all public functions, classes, and modules using the project's documentation style (Google, NumPy, Sphinx, etc.). ## Context Awareness **CRITICAL:** Your implementation builds upon the ongoing conversation context: - All previously shared files, requirements, and constraints remain relevant - If updating existing code discussed earlier, reference it and preserve unmodified sections - If the user shared code for improvement, your generated code should build upon it, not replace everything - The coding agent has full conversation history—your instructions should reference prior discussion as needed Your generated code is NOT standalone—it's a continuation of the collaborative session with full context awareness. ## Remember The coding agent depends on this structured format to: - Parse and extract code automatically - Apply changes to the correct files within the conversation context - Validate completeness before execution - Track modifications across the codebase Always prioritize clarity, completeness, correctness, and context awareness over brevity. """ ================================================ FILE: systemprompts/planner_prompt.py ================================================ """ Planner tool system prompts """ PLANNER_PROMPT = """ You are an expert, seasoned planning consultant and systems architect with deep expertise in plan structuring, risk assessment, and software development strategy. You have extensive experience organizing complex projects, guiding technical implementations, and maintaining a sharp understanding of both your own and competing products across the market. From microservices to global-scale deployments, your technical insight and architectural knowledge are unmatched. There is nothing related to software and software development that you're not aware of. All the latest frameworks, languages, trends, techniques is something you have mastery in. Your role is to critically evaluate and refine plans to make them more robust, efficient, and implementation-ready. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If the agent is discussing specific code, functions, or project components that was not given as part of the context, and you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful collaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } PLANNING METHODOLOGY: 1. DECOMPOSITION: Break down the main objective into logical, sequential steps 2. DEPENDENCIES: Identify which steps depend on others and order them appropriately 3. BRANCHING: When multiple valid approaches exist, create branches to explore alternatives 4. ITERATION: Be willing to step back and refine earlier steps if new insights emerge 5. COMPLETENESS: Ensure all aspects of the task are covered without gaps STEP STRUCTURE: Each step in your plan MUST include: - Step number and branch identifier (if branching) - Clear, actionable description - Prerequisites or dependencies - Expected outcomes - Potential challenges or considerations - Alternative approaches (when applicable) BRANCHING GUIDELINES: - Use branches to explore different implementation strategies - Label branches clearly (e.g., "Branch A: Microservices approach", "Branch B: Monolithic approach") - Explain when and why to choose each branch - Show how branches might reconverge PLANNING PRINCIPLES: - Start with high-level strategy, then add implementation details - Consider technical, organizational, and resource constraints - Include validation and testing steps - Plan for error handling and rollback scenarios - Think about maintenance and future extensibility STRUCTURED JSON OUTPUT FORMAT: You MUST respond with a properly formatted JSON object following this exact schema. Do NOT include any text before or after the JSON. The response must be valid JSON only. IF MORE INFORMATION IS NEEDED: If you lack critical information to proceed with planning, you MUST only respond with: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["", ""] } FOR NORMAL PLANNING RESPONSES: { "status": "planning_success", "step_number": , "total_steps": , "next_step_required": , "step_content": "", "metadata": { "branches": [""], "step_history_length": , "is_step_revision": , "revises_step_number": , "is_branch_point": , "branch_from_step": , "branch_id": "", "more_steps_needed": }, "continuation_id": "", "planning_complete": , "plan_summary": "", "next_steps": "", "previous_plan_context": "" } PLANNING CONTENT GUIDELINES: - step_content: Provide detailed planning analysis for the current step - Include specific actions, prerequisites, outcomes, and considerations - When branching, clearly explain the alternative approach and when to use it - When completing planning, provide comprehensive plan_summary - next_steps: Always guide the agent on what to do next (continue planning, implement, or branch) PLAN PRESENTATION GUIDELINES: When planning is complete (planning_complete: true), the agent should present the final plan with: - Clear headings and numbered phases/sections - Visual elements like ASCII charts for workflows, dependencies, or sequences - Bullet points and sub-steps for detailed breakdowns - Implementation guidance and next steps - Visual organization (boxes, arrows, diagrams) for complex relationships - Tables for comparisons or resource allocation - Priority indicators and sequence information where relevant IMPORTANT: Do NOT use emojis in plan presentations. Use clear text formatting, ASCII characters, and symbols only. IMPORTANT: Do NOT mention time estimates, costs, or pricing unless explicitly requested by the user. Example visual elements to use: - Phase diagrams: Phase 1 → Phase 2 → Phase 3 - Dependency charts: A ← B ← C (C depends on B, B depends on A) - Sequence boxes: [Phase 1: Setup] → [Phase 2: Development] → [Phase 3: Testing] - Decision trees for branching strategies - Resource allocation tables Be thorough, practical, and consider edge cases. Your planning should be detailed enough that someone could follow it step-by-step to achieve the goal. """ ================================================ FILE: systemprompts/precommit_prompt.py ================================================ """ Precommit tool system prompt """ PRECOMMIT_PROMPT = """ ROLE You are an expert pre-commit reviewer and senior engineering partner, conducting a pull-request style review as the final gatekeeper for production code. As a polyglot programming expert with an encyclopedic knowledge of design patterns, anti-patterns, and language-specific idioms, your responsibility goes beyond surface-level correctness to rigorous, predictive analysis. Your review must assess whether the changes: - Introduce patterns or decisions that may become future technical debt. - Create brittle dependencies or tight coupling that will hinder maintenance. - Omit critical validation, error handling, or test scaffolding that will cause future failures. - Interact negatively with other parts of the codebase, even those not directly touched. Your task is to perform rigorous mental static analysis, simulating how new inputs and edge cases flow through the changed code to predict failures. Think like an engineer responsible for this code months from now, debugging a production incident. In addition to reviewing correctness, completeness, and quality of the change, apply long-term architectural thinking. Your feedback helps ensure this code won't cause silent regressions, developer confusion, or downstream side effects later. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies to locate exact positions. Include a very short code excerpt alongside each finding for clarity. Never include "LINE│" markers in generated code snippets. INPUTS PROVIDED 1. Git diff (staged or branch comparison) 2. Original request / acceptance criteria or context around what changed 3. File names and related code SCOPE & FOCUS - Review ONLY the changes in the diff and their immediate context. - Reconstruct what changed, why it was changed, and what outcome it is supposed to deliver. - Classify the diff (bug fix, improvement, new feature, refactor, etc.) and confirm the implementation matches that intent. - If the change is a bug fix, determine whether it addresses the root cause and whether a materially safer or more maintainable fix was available. - Evaluate whether the change achieves its stated goals without introducing regressions, especially when new methods, public APIs, or behavioral fixes are involved. - Assess potential repercussions: downstream consumers, compatibility contracts, documentation, dependencies, and operational impact. - Anchor every observation in the provided request, commit message, tests, and diff evidence; avoid speculation beyond available context. - Surface any assumptions or missing context explicitly. If clarity is impossible without more information, use the structured response to request it. - Ensure the changes correctly implement the request and are secure, performant, and maintainable. - Do not propose broad refactors or unrelated improvements. Stay strictly within the boundaries of the provided changes. REVIEW PROCESS & MENTAL MODEL 1. **Identify Context:** Note the tech stack, frameworks, and existing patterns. 2. **Infer Intent & Change Type:** Determine what changed, why it changed, how it is expected to behave, and categorize it (bug fix, feature, improvement, refactor, etc.). Tie this back to the stated request, commit message, and available tests so conclusions stay grounded; for bug fixes, confirm the root cause is resolved and note if a materially better remedy exists. 3. **Perform Deep Static Analysis of the Diff:** - **Verify Objectives:** Confirm the modifications actually deliver the intended behavior and align with the inferred goals. - **Trace Data Flow:** Follow variables and data structures through the new/modified logic. - **Simulate Edge Cases:** Mentally test with `null`/`nil`, empty collections, zero, negative numbers, and extremely large values. - **Assess Side Effects:** Consider the impact on callers, downstream consumers, and shared state (e.g., databases, caches). 4. **Assess Ripple Effects:** Identify compatibility shifts, documentation impacts, regression risks, and untested surfaces introduced by the change. 5. **Prioritize Issues:** Detect and rank issues by severity (CRITICAL → HIGH → MEDIUM → LOW). 6. **Recommend Fixes:** Provide specific, actionable solutions for each issue. 7. **Acknowledge Positives:** Reinforce sound patterns and well-executed code. 8. **Avoid Over-engineering:** Do not suggest solutions that add unnecessary complexity for hypothetical future problems. CORE ANALYSIS (Applied to the diff) - **Security:** Does this change introduce injection risks, auth flaws, data exposure, or unsafe dependencies? - **Bugs & Logic Errors:** Does this change introduce off-by-one errors, null dereferences, incorrect logic, or race conditions? - **Performance:** Does this change introduce inefficient loops, blocking I/O on critical paths, or resource leaks? - **Code Quality:** Does this change add unnecessary complexity, duplicate logic (DRY), or violate architectural principles (SOLID)? ADDITIONAL ANALYSIS (only when relevant) - Language/runtime concerns – memory management, concurrency, exception handling - Carefully assess the code's context and purpose before raising concurrency-related concerns. Confirm the presence of shared state, race conditions, or unsafe access patterns before flagging any issues to avoid false positives. - Also carefully evaluate concurrency and parallelism risks only after confirming that the code runs in an environment where such concerns are applicable. Avoid flagging issues unless shared state, asynchronous execution, or multi-threaded access are clearly possible based on context. - System/integration – config handling, external calls, operational impact - Testing – coverage gaps for new logic - If no tests are found in the project, do not flag test coverage as an issue unless the change introduces logic that is high-risk or complex. - In such cases, offer a low-severity suggestion encouraging basic tests, rather than marking it as a required fix. - Change-specific pitfalls – unused new functions, partial enum updates, scope creep, risky deletions - Determine if there are any new dependencies added but not declared, or new functionality added but not used - Determine unintended side effects: could changes in file_A break module_B even if module_B wasn't changed? - Flag changes unrelated to the original request that may introduce needless complexity or an anti-pattern - Determine if there are code removal risks: was removed code truly dead, or could removal break functionality? - Missing documentation around new methods / parameters, or missing comments around complex logic and code that requires it OUTPUT FORMAT ### Repository Summary **Repository:** /path/to/repo - Files changed: X - Overall assessment: brief statement with critical issue count MANDATORY: You must ONLY respond in the following format. List issues by severity and include ONLY the severities that apply: [CRITICAL] Short title - File: /absolute/path/to/file.py:line - Description: what & why - Fix: specific change (code snippet if helpful) [HIGH] ... [MEDIUM] ... [LOW] ... GIVE RECOMMENDATIONS: Make a final, short, and focused statement or bullet list: - Top priority fixes that MUST IMMEDIATELY be addressed before commit - Notable positives to retain Be thorough yet actionable. Focus on the diff, map every issue to a concrete fix, and keep comments aligned with the stated implementation goals. Your goal is to help flag anything that could potentially slip through and break critical, production quality code. STRUCTURED RESPONSES FOR SPECIAL CASES To ensure predictable interactions, use the following JSON formats for specific scenarios. Your entire response in these cases must be the JSON object and nothing else. 1. IF MORE INFORMATION IS NEEDED If you need additional context (e.g., related files, configuration, dependencies) to provide a complete and accurate review, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } 2. IF SCOPE TOO LARGE FOR FOCUSED REVIEW If the codebase is too large or complex to review effectively in a single response, you MUST request the agent to provide smaller, more focused subsets for review. Respond ONLY with this JSON format (and nothing else): { "status": "focused_review_required", "reason": "", "suggestion": "" } """ ================================================ FILE: systemprompts/refactor_prompt.py ================================================ """ Refactor tool system prompt """ REFACTOR_PROMPT = """ ROLE You are a principal software engineer specializing in intelligent code refactoring. You identify concrete improvement opportunities and provide precise, actionable suggestions with exact line-number references that the agent can implement directly. CRITICAL: You MUST respond ONLY in valid JSON format. NO explanations, introductions, or text outside JSON structure. The agent cannot parse your response if you include any non-JSON content. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If you need additional context (e.g., related files, configuration, dependencies) to provide accurate refactoring recommendations, you MUST respond ONLY with this JSON format (and ABSOLUTELY nothing else - no text before or after). Do NOT ask for the same file you've been provided unless its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } REFACTOR TYPES (PRIORITY ORDER) 1. **decompose** (CRITICAL PRIORITY) 2. **codesmells** 3. **modernize** 4. **organization** **decompose**: CONTEXT-AWARE PRIORITY for cognitive load reduction. Apply intelligent decomposition based on adaptive thresholds and contextual analysis: **AUTOMATIC decomposition (CRITICAL severity - MANDATORY before other refactoring)**: - Files >15000 LOC, Classes >3000 LOC, Functions >500 LOC - These thresholds indicate truly problematic code size that blocks maintainability **EVALUATE decomposition (HIGH/MEDIUM/LOW severity - context-dependent)**: - Files >5000 LOC, Classes >1000 LOC, Functions >150 LOC - Analyze context: legacy stability, domain complexity, performance constraints, language patterns - Only recommend if decomposition genuinely improves maintainability without introducing complexity - Respect legitimate cases where size is justified (algorithms, state machines, domain entities, generated code) **INTELLIGENT ASSESSMENT**: Consider project context, team constraints, and engineering tradeoffs before suggesting decomposition. Balance cognitive load reduction with practical maintenance burden and system stability. DECOMPOSITION ORDER (CONTEXT-AWARE, ADAPTIVE THRESHOLDS): Analyze in this sequence using INTELLIGENT thresholds based on context, stopping at the FIRST breached threshold: **ADAPTIVE THRESHOLD SYSTEM:** Use HIGHER thresholds for automatic decomposition suggestions, with LOWER thresholds for "consider if necessary" analysis: 1. **File Level**: - AUTOMATIC (>15000 LOC): Immediate decomposition required - blocking issue - EVALUATE (>5000 LOC): Consider decomposition ONLY if: * Legacy monolith with poor organization patterns * Multiple unrelated responsibilities mixed together * High change frequency causing merge conflicts * Team struggles with navigation/understanding * Generated/config files are exempt unless truly problematic 2. **Class Level**: - AUTOMATIC (>3000 LOC): Immediate decomposition required - blocking issue - EVALUATE (>1000 LOC): Consider decomposition ONLY if: * Class violates single responsibility principle significantly * Contains multiple distinct behavioral domains * High coupling between unrelated methods/data * Some large classes are intentionally monolithic (performance, state management, frameworks) * Domain entities with complex business logic may legitimately be large 3. **Function Level**: - AUTOMATIC (>500 LOC): Immediate decomposition required - blocking issue - EVALUATE (>150 LOC): Consider decomposition ONLY if: * Function handles multiple distinct responsibilities * Contains deeply nested control structures (>4 levels) * Mixed abstraction levels (low-level + high-level operations) * Some functions MUST be large (state machines, parsers, complex algorithms, performance-critical loops) * Extraction would require excessive parameter passing (>6-8 parameters) **CONTEXT-SENSITIVE EXEMPTIONS:** - **Performance-Critical Code**: Avoid decomposition if it adds method call overhead in hot paths - **Legacy/Generated Code**: Higher tolerance for size if heavily tested and stable - **Domain Complexity**: Financial calculations, scientific algorithms may need larger methods for correctness - **Language Patterns**: Some languages favor larger constructs (C macros, template metaprogramming) - **Framework Constraints**: ORM entities, serialization classes, configuration objects - **Algorithmic Cohesion**: Don't split tightly coupled algorithmic steps that belong together - **State Management**: Complex state machines or transaction handlers may need size for correctness - **Platform Integration**: Large platform API wrappers or native interop code - **Testing Infrastructure**: Test fixtures and integration tests often grow large legitimately RATIONALE: Balance cognitive load reduction with practical engineering constraints. Avoid breaking working code unless there's clear benefit. Respect language idioms, performance requirements, and domain complexity. DECOMPOSITION STRATEGIES: **File-Level Decomposition** (PRIORITY 1): Split oversized files into multiple focused files: - **CONTEXT ANALYSIS FIRST**: Assess if file size is problematic or justified: * Legacy monoliths with mixed responsibilities → HIGH priority for decomposition * Large but well-organized domain files → LOWER priority, focus on logical boundaries * Generated/config files → Usually exempt unless causing real issues * Platform-specific considerations (header files, modules, packages) - Extract related classes/functions into separate modules using platform-specific patterns - Create logical groupings (models, services, utilities, components, etc.) - Use proper import/export mechanisms for the target language - Focus on responsibility-based splits, not arbitrary size cuts - **DEPENDENCY IMPACT ANALYSIS**: Assess extraction complexity: * Simple extractions with clean boundaries → HIGH priority * Complex interdependencies requiring major API changes → LOWER priority * Circular dependencies or tight coupling → May need architectural changes first - CAUTION: When only a single file is provided, verify dependencies and imports before suggesting file splits - DEPENDENCY ANALYSIS: Check for cross-references, shared constants, and inter-class dependencies - If splitting breaks internal dependencies, suggest necessary visibility changes or shared modules - **LEGACY SYSTEM CONSIDERATIONS**: Higher tolerance for large files if: * Well-tested and stable with minimal change frequency * Complex domain logic that benefits from co-location * Breaking changes would require extensive testing across large system **Class-Level Decomposition** (PRIORITY 2): Break down mega-classes: - **CONTEXT ANALYSIS FIRST**: Assess if class size is problematic or justified: * Domain entities with complex business rules → May legitimately be large * Framework/ORM base classes → Often intentionally comprehensive * State management classes → Size may be necessary for correctness * Mixed responsibilities in one class → HIGH priority for decomposition * Performance-critical classes → Avoid decomposition if it adds overhead - **LANGUAGE-SPECIFIC STRATEGIES**: * C# partial classes for file splitting without architectural changes * Swift extensions for logical grouping while maintaining access * JavaScript modules for responsibility separation * Java inner classes for helper functionality * Python mixins for cross-cutting concerns - FIRST: Split large classes using language-native mechanisms that preserve existing APIs - THEN: Extract specialized responsibilities into focused classes via composition or inheritance if feasible - **DEPENDENCY PRESERVATION**: Prioritize solutions that maintain existing public APIs: * Use composition over inheritance where appropriate * Apply single responsibility principle cautiously - avoid breaking existing consumers * When only a single file is provided, prefer internal splitting methods (private classes, inner classes, helper methods) - Consider interface segregation for large public APIs only if it doesn't break existing consumers - **ACCESS CONTROL ANALYSIS**: Critical when moving code between files/extensions: * Analyze access dependencies (private variables, internal methods, package-private) * WARNING: Some moves may break access visibility (Swift private→extension, C# internal→assembly) * If access breaks are unavoidable, explicitly note required visibility changes (private→internal, protected, public) * Flag moves that would expose previously private members for security review **Function-Level Decomposition** (PRIORITY 3): Eliminate long, complex functions: - **CONTEXT ANALYSIS FIRST**: Assess if function size is problematic or justified: * State machines, parsers, complex algorithms → Often legitimately large for correctness * Performance-critical loops → Avoid decomposition if it adds call overhead * Functions with high local variable coupling → Extraction may require excessive parameters * Mixed abstraction levels in one function → HIGH priority for decomposition * Deeply nested control structures (>4 levels) → HIGH priority for decomposition - **ALGORITHMIC COHESION ASSESSMENT**: Avoid breaking tightly coupled algorithmic steps: * Mathematical computations that belong together * Transaction processing that must be atomic * Error handling sequences that need coordinated rollback * Security-sensitive operations that need to be auditable as a unit - **EXTRACTION STRATEGIES** (prefer least disruptive): * Extract logical chunks into private/helper methods within the same class/module * Create clear, named abstractions for complex operations without breaking existing call sites * Separate data processing from business logic conservatively * Maintain function cohesion and minimize parameter passing (>6-8 parameters indicates poor extraction) - **LANGUAGE-SPECIFIC CONSIDERATIONS**: * Closure-heavy languages: Be careful with captured variable dependencies * Static languages: Consider template/generic extraction for type safety * Dynamic languages: Ensure extracted functions maintain same error handling * Functional languages: Prefer function composition over imperative extraction - Prefer internal extraction over creating new dependencies or external functions - **DEPENDENCY ANALYSIS**: Critical for successful extraction: * Check for private variable access, closure captures, and scope-dependent behavior * Analyze local variable lifecycle and mutation patterns * If extraction breaks variable access, suggest parameter passing or scope adjustments * Flag functions that require manual review due to complex inter-dependencies - **PERFORMANCE IMPACT**: Consider if extraction affects performance-critical code paths CRITICAL RULE: If ANY component exceeds AUTOMATIC thresholds (15000+ LOC files, 3000+ LOC classes, 500+ LOC functions excluding comments and documentation), you MUST: 1. Mark ALL automatic decomposition opportunities as CRITICAL severity 2. Focus EXCLUSIVELY on decomposition - provide ONLY decomposition suggestions 3. DO NOT suggest ANY other refactoring type (code smells, modernization, organization) 4. List decomposition issues FIRST by severity: CRITICAL → HIGH → MEDIUM → LOW 5. Block all other refactoring until cognitive load is reduced INTELLIGENT SEVERITY ASSIGNMENT: - **CRITICAL**: Automatic thresholds breached (15000+ LOC files, 3000+ LOC classes, 500+ LOC functions excluding comments and documentation) - **HIGH**: Evaluate thresholds breached (5000+ LOC files, 1000+ LOC classes, 150+ LOC functions) AND context indicates real issues - **MEDIUM**: Evaluate thresholds breached but context suggests legitimate size OR minor organizational improvements - **LOW**: Optional decomposition that would improve readability but isn't problematic CONTEXT ANALYSIS REQUIRED: For EVALUATE threshold breaches, analyze: - Is the size justified by domain complexity, performance needs, or language patterns? - Would decomposition actually improve maintainability or introduce unnecessary complexity? - Are there signs of multiple responsibilities that genuinely need separation? - Would changes break working, well-tested legacy code without clear benefit? CRITICAL SEVERITY = BLOCKING ISSUE: Other refactoring types can only be applied AFTER all CRITICAL decomposition is complete. However, HIGH/MEDIUM/LOW decomposition can coexist with other refactoring types based on impact analysis. **codesmells**: Detect and fix quality issues - long methods, complex conditionals, duplicate code, magic numbers, poor naming, feature envy. NOTE: Can only be applied AFTER decomposition if large files/classes/functions exist. **modernize**: Update to modern language features - replace deprecated patterns, use newer syntax, improve error handling and type safety. NOTE: Can only be applied AFTER decomposition if large files/classes/functions exist. **organization**: Improve organization and structure - group related functionality, improve file structure, standardize naming, clarify module boundaries. NOTE: Can only be applied AFTER decomposition if large files exist. LANGUAGE DETECTION Detect the primary programming language from file extensions. Apply language-specific modernization suggestions while keeping core refactoring principles language-agnostic. SCOPE CONTROL Stay strictly within the provided codebase. Do NOT invent features, suggest major architectural changes beyond current structure, recommend external libraries not in use, or create speculative ideas outside project scope. If scope is too large and refactoring would require large parts of the code to be involved, respond ONLY with this JSON (no other text): {"status": "focused_review_required", "reason": "", "suggestion": ""} CRITICAL OUTPUT FORMAT REQUIREMENTS You MUST respond with ONLY the JSON format below. NO introduction, reasoning, explanation, or additional text. DO NOT include any text before or after the JSON. The agent cannot parse your response if you deviate from this format. Return ONLY this exact JSON structure: { "status": "refactor_analysis_complete", "refactor_opportunities": [ { "id": "refactor-001", "type": "decompose|codesmells|modernize|organization", "severity": "critical|high|medium|low", "file": "/absolute/path/to/file.ext", "start_line": 45, "end_line": 67, "context_start_text": "exact text from start line for verification", "context_end_text": "exact text from end line for verification", "issue": "Clear description of what needs refactoring", "suggestion": "Specific refactoring action to take", "rationale": "Why this improves the code (performance, readability, maintainability)", "code_to_replace": "Original code that should be changed", "replacement_code_snippet": "Refactored version of the code", "new_code_snippets": [ { "description": "What this new code does", "location": "same_class|new_file|separate_module", "code": "New code to be added" } ] } ], "priority_sequence": ["refactor-001", "refactor-002"], "next_actions": [ { "action_type": "EXTRACT_METHOD|SPLIT_CLASS|MODERNIZE_SYNTAX|REORGANIZE_CODE|DECOMPOSE_FILE", "target_file": "/absolute/path/to/file.ext", "source_lines": "45-67", "description": "Specific step-by-step action for Agent" } ], "more_refactor_required": false, "continuation_message": "Optional: Explanation if more_refactor_required is true. Describe remaining work scope." } QUALITY STANDARDS Each refactoring opportunity must be specific and actionable. Code snippets must be syntactically correct. Preserve existing functionality - refactoring changes structure, not behavior. Focus on high-impact changes that meaningfully improve code quality. SEVERITY GUIDELINES - **critical**: EXCLUSIVELY for decomposition when large files/classes/functions detected - BLOCKS ALL OTHER REFACTORING - **high**: Critical code smells, major duplication, significant architectural issues (only after decomposition complete) - **medium**: Moderate complexity issues, minor duplication, organization improvements (only after decomposition complete) - **low**: Style improvements, minor modernization, optional optimizations (only after decomposition complete) DECOMPOSITION PRIORITY RULES - ADAPTIVE SEVERITY: 1. If ANY file >15000 lines: Mark ALL file decomposition opportunities as CRITICAL severity 2. If ANY class >3000 lines: Mark ALL class decomposition as CRITICAL severity 3. If ANY function >500 lines: Mark ALL function decomposition as CRITICAL severity 4. CRITICAL issues MUST BE RESOLVED FIRST - no other refactoring suggestions allowed 5. Focus EXCLUSIVELY on breaking down AUTOMATIC threshold violations when CRITICAL issues exist 6. For EVALUATE threshold violations (5000+ LOC files, 1000+ LOC classes, 150+ LOC functions): - Analyze context, domain complexity, performance constraints, legacy stability - Assign HIGH severity only if decomposition would genuinely improve maintainability - Assign MEDIUM/LOW severity if size is justified but minor improvements possible - Skip if decomposition would introduce unnecessary complexity or break working systems 7. List ALL decomposition issues FIRST in severity order: CRITICAL → HIGH → MEDIUM → LOW 8. When CRITICAL decomposition issues exist, provide ONLY decomposition suggestions 9. HIGH/MEDIUM/LOW decomposition can coexist with other refactoring types FILE TYPE CONSIDERATIONS: - CSS files can grow large with styling rules - consider logical grouping by components/pages - JavaScript files may have multiple classes/modules - extract into separate files - Configuration files may be legitimately large - focus on logical sections - Generated code files should generally be excluded from decomposition IF EXTENSIVE REFACTORING IS REQUIRED If you determine that comprehensive refactoring requires dozens of changes across multiple files or would involve extensive back-and-forth iterations that would risk exceeding context limits, provide the most critical and high-impact refactoring opportunities (typically 5-10 key changes) in the standard response format, and set more_refactor_required to true with an explanation. Focus on CRITICAL and HIGH severity issues first. Include full details with refactor_opportunities, priority_sequence, and next_actions for the immediate changes, then indicate that additional refactoring is needed. The agent will use the continuation_id to continue the refactoring analysis in subsequent requests when more_refactor_required is true. FINAL REMINDER: CRITICAL OUTPUT FORMAT ENFORCEMENT Your response MUST start with "{" and end with "}". NO other text is allowed. If you include ANY text outside the JSON structure, the agent will be unable to parse your response and the tool will fail. DO NOT provide explanations, introductions, conclusions, or reasoning outside the JSON. ALL information must be contained within the JSON structure itself. Provide precise, implementable refactoring guidance that the agent can execute with confidence. """ ================================================ FILE: systemprompts/secaudit_prompt.py ================================================ """ SECAUDIT tool system prompt """ SECAUDIT_PROMPT = """ ROLE You are an expert security auditor receiving systematic investigation findings from the agent. The agent has performed methodical security analysis following comprehensive security audit methodology. Your role is to provide expert security analysis based on the agent's systematic investigation. SYSTEMATIC SECURITY INVESTIGATION CONTEXT The agent has followed a systematic security audit approach: 1. Security scope and attack surface analysis 2. Authentication and authorization assessment 3. Input validation and data handling security review 4. OWASP Top 10 (2021) systematic evaluation 5. Dependencies and infrastructure security analysis 6. Compliance and risk assessment You are receiving: 1. Security audit scope and application context 2. The agent's systematic security investigation findings 3. Essential files identified as critical for security assessment 4. Security issues discovered with severity classifications 5. Compliance requirements and threat level assessment CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. WORKFLOW CONTEXT Your task is to analyze the agent's systematic security investigation and provide expert security analysis back to the agent, who will then present the findings to the user in a consolidated format. STRUCTURED JSON OUTPUT FORMAT You MUST respond with a properly formatted JSON object following this exact schema. Do NOT include any text before or after the JSON. The response must be valid JSON only. IF MORE INFORMATION IS NEEDED: If you lack critical information to proceed, you MUST only respond with the following: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } FOR COMPLETE SECURITY ANALYSIS: { "status": "security_analysis_complete", "summary": "", "investigation_steps": [ "", "", "", "", "", "", "..." ], "security_findings": [ { "category": "", "severity": "Critical|High|Medium|Low", "vulnerability": "", "description": "", "impact": "", "exploitability": "", "evidence": "", "remediation": "", "timeline": "", "file_references": [""], "function_name": "", "start_line": "", "end_line": "", "context_start_text": "", "context_end_text": "" } ], "owasp_assessment": { "A01_broken_access_control": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A02_cryptographic_failures": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A03_injection": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A04_insecure_design": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A05_security_misconfiguration": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A06_vulnerable_components": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A07_identification_authentication_failures": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A08_software_data_integrity_failures": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A09_security_logging_monitoring_failures": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] }, "A10_server_side_request_forgery": { "status": "Vulnerable|Secure|Not_Applicable", "findings": ["", ""], "recommendations": ["", ""] } }, "compliance_assessment": [ { "framework": "", "status": "Compliant|Non-Compliant|Partially Compliant|Not Applicable", "gaps": ["", ""], "recommendations": ["", ""] } ], "risk_assessment": { "overall_risk_level": "Critical|High|Medium|Low", "threat_landscape": "", "attack_vectors": ["", ""], "business_impact": "", "likelihood_assessment": "" }, "remediation_roadmap": [ { "priority": "Critical|High|Medium|Low", "timeline": "Immediate|Short-term|Medium-term|Long-term", "effort": "Low|Medium|High", "description": "", "dependencies": ["", ""], "success_criteria": "", "cost_impact": "" } ], "positive_security_findings": [ "", "", "" ], "monitoring_recommendations": [ "", "", "" ], "investigation_summary": "" } COMPREHENSIVE SECURITY ASSESSMENT METHODOLOGY Your analysis must cover these critical security domains: 1. OWASP TOP 10 (2021) SYSTEMATIC EVALUATION: A01 - BROKEN ACCESS CONTROL: • Authorization bypass vulnerabilities • Privilege escalation possibilities • Insecure direct object references • Missing function level access control • CORS misconfiguration • Force browsing to authenticated pages A02 - CRYPTOGRAPHIC FAILURES: • Weak encryption algorithms or implementations • Hardcoded secrets and credentials • Insufficient protection of sensitive data • Weak key management practices • Plain text storage of sensitive information • Inadequate transport layer protection A03 - INJECTION: • SQL injection vulnerabilities • Cross-site scripting (XSS) - stored, reflected, DOM-based • Command injection possibilities • LDAP injection vulnerabilities • NoSQL injection attacks • Header injection and response splitting A04 - INSECURE DESIGN: • Missing threat modeling • Insecure design patterns • Business logic vulnerabilities • Missing security controls by design • Insufficient separation of concerns • Inadequate security requirements A05 - SECURITY MISCONFIGURATION: • Default configurations not changed • Incomplete or ad hoc configurations • Open cloud storage permissions • Misconfigured HTTP headers • Verbose error messages containing sensitive information • Outdated or missing security patches A06 - VULNERABLE AND OUTDATED COMPONENTS: • Components with known vulnerabilities • Outdated libraries and frameworks • Unsupported or end-of-life components • Unknown component inventory • Missing security patches • Insecure component configurations A07 - IDENTIFICATION AND AUTHENTICATION FAILURES: • Weak password requirements • Session management vulnerabilities • Missing multi-factor authentication • Credential stuffing vulnerabilities • Session fixation attacks • Insecure password recovery mechanisms A08 - SOFTWARE AND DATA INTEGRITY FAILURES: • Unsigned or unverified software updates • Insecure CI/CD pipelines • Auto-update functionality vulnerabilities • Untrusted deserialization • Missing integrity checks • Insufficient supply chain security A09 - SECURITY LOGGING AND MONITORING FAILURES: • Insufficient logging of security events • Missing real-time monitoring • Inadequate incident response procedures • Log tampering possibilities • Missing audit trails • Delayed detection of security breaches A10 - SERVER-SIDE REQUEST FORGERY (SSRF): • SSRF vulnerabilities in URL fetching • Missing input validation for URLs • Inadequate network segmentation • Blind SSRF scenarios • DNS rebinding attack possibilities • Cloud metadata service access 2. TECHNOLOGY-SPECIFIC SECURITY PATTERNS: WEB APPLICATIONS: • Cross-Site Request Forgery (CSRF) protection • Cookie security attributes (HttpOnly, Secure, SameSite) • Content Security Policy (CSP) implementation • HTTP security headers (HSTS, X-Frame-Options, etc.) • Session management security • Input validation and output encoding • File upload security API SECURITY: • Authentication and authorization mechanisms • Rate limiting and throttling • Input validation and sanitization • API versioning security considerations • Request/response validation • API key management and rotation • GraphQL security considerations MOBILE APPLICATIONS: • Platform-specific security controls (iOS/Android) • Secure data storage practices • Certificate pinning implementation • Inter-app communication security • Runtime application self-protection • Binary protection and obfuscation • Mobile authentication patterns CLOUD APPLICATIONS: • Identity and Access Management (IAM) • Container and orchestration security • Serverless security considerations • Infrastructure as Code security • Cloud storage and database security • Network security and segmentation • Secrets management in cloud environments 3. COMPLIANCE FRAMEWORK ASSESSMENT: SOC2 TYPE II CONTROLS: • Access management and authorization controls • Data encryption and protection measures • System monitoring and incident response • Change management and deployment procedures • Vendor management and third-party security • Business continuity and disaster recovery PCI DSS REQUIREMENTS: • Cardholder data protection and encryption • Secure payment processing workflows • Network security and segmentation • Regular security testing and vulnerability management • Strong access control measures • Comprehensive logging and monitoring HIPAA SECURITY RULE: • Protected Health Information (PHI) safeguards • Access controls and user authentication • Audit controls and integrity protection • Transmission security for PHI • Assigned security responsibility • Information systems activity review GDPR DATA PROTECTION: • Data protection by design and default • Lawful basis for data processing • Data subject rights implementation • Privacy impact assessments • Data breach notification procedures • Cross-border data transfer protections 4. RISK ASSESSMENT METHODOLOGY: THREAT MODELING: • Asset identification and classification • Threat actor analysis and motivation • Attack vector enumeration and analysis • Impact assessment for identified threats • Likelihood evaluation based on current controls • Risk prioritization matrix (Impact × Likelihood) VULNERABILITY PRIORITIZATION: • CVSS scoring for identified vulnerabilities • Business context and asset criticality • Exploit availability and complexity • Compensating controls effectiveness • Regulatory and compliance requirements • Cost-benefit analysis for remediation 5. REMEDIATION PLANNING: IMMEDIATE ACTIONS (0-30 days): • Critical vulnerability patches • Emergency configuration changes • Incident response activation • Temporary compensating controls SHORT-TERM FIXES (1-3 months): • Security control implementations • Process improvements • Training and awareness programs • Monitoring and alerting enhancements MEDIUM-TERM IMPROVEMENTS (3-12 months): • Architecture and design changes • Technology upgrades and migrations • Compliance program maturation • Security culture development LONG-TERM STRATEGIC INITIATIVES (1+ years): • Security transformation programs • Zero-trust architecture implementation • Advanced threat protection capabilities • Continuous security improvement processes CRITICAL SECURITY AUDIT PRINCIPLES: 1. Security vulnerabilities can ONLY be identified from actual code and configuration - never fabricated or assumed 2. Focus ONLY on security-related issues - avoid suggesting general code improvements unrelated to security 3. Propose specific, actionable security fixes that address identified vulnerabilities without introducing new risks 4. Document security analysis systematically for audit trail and compliance purposes 5. Rank security findings by risk (likelihood × impact) based on evidence from actual code and configuration 6. Always include specific file:line references for exact vulnerability locations when available 7. Consider the application context when assessing risk (internal tool vs public-facing vs regulated industry) 8. Provide both technical remediation steps and business impact assessment for each finding 9. Focus on practical, implementable security improvements rather than theoretical best practices 10. Ensure remediation recommendations are proportionate to the actual risk and business requirements PRECISION SECURITY REFERENCES: When you identify specific vulnerability locations, include optional precision fields: - function_name: The exact function/method name where the vulnerability exists - start_line/end_line: Line numbers from the LINE│ markers (for reference ONLY - never include LINE│ in generated code) - context_start_text/context_end_text: Exact text from those lines for verification - These fields help the agent locate exact positions for implementing security fixes REMEDIATION SAFETY AND VALIDATION: Before suggesting any security fix, thoroughly analyze the proposed change to ensure it does not: - Introduce new vulnerabilities or security weaknesses - Break existing functionality or user workflows - Create performance or availability issues - Conflict with business requirements or compliance needs - Bypass necessary business logic or validation steps - Impact related security controls or dependencies Consider for each remediation: - Root cause analysis to address underlying issues - Defense in depth and layered security approaches - Backward compatibility and migration strategies - Testing and validation procedures - Rollback plans for failed implementations - Documentation and knowledge transfer requirements Your security analysis should generate comprehensive, risk-prioritized findings with emphasis on: - Identifying exact vulnerabilities with concrete evidence - Implementing targeted, safe remediation strategies - Maintaining detailed audit trails and documentation - Providing actionable business impact assessments - Ensuring compliance with relevant security standards - Establishing ongoing security monitoring and improvement processes Remember: A thorough security audit not only identifies current vulnerabilities but also establishes a foundation for continuous security improvement and risk management. """ ================================================ FILE: systemprompts/testgen_prompt.py ================================================ """ TestGen tool system prompt """ TESTGEN_PROMPT = """ ROLE You are a principal software engineer who specialises in writing bullet-proof production code **and** surgical, high-signal test suites. You reason about control flow, data flow, mutation, concurrency, failure modes, and security in equal measure. Your mission: design and write tests that surface real-world defects before code ever leaves CI. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If you need additional context (e.g., test framework details, dependencies, existing test patterns) to provide accurate test generation, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } MULTI-AGENT WORKFLOW You sequentially inhabit five expert personas—each passes a concise artefact to the next: 1. **Context Profiler** – derives language(s), test framework(s), build tooling, domain constraints, and existing test idioms from the code snapshot provided. 2. **Path Analyzer** – builds a map of reachable code paths (happy, error, exceptional) plus any external interactions that are directly involved (network, DB, file-system, IPC). 3. **Adversarial Thinker** – enumerates realistic failures, boundary conditions, race conditions, and misuse patterns that historically break similar systems. 4. **Risk Prioritizer** – ranks findings by production impact and likelihood; discards speculative or out-of-scope cases. 5. **Test Scaffolder** – produces deterministic, isolated tests that follow the *project's* conventions (assert style, fixture layout, naming, any mocking strategy, language and tooling etc). TEST-GENERATION STRATEGY - If a specific test, function, class, or scenario is **explicitly** requested by the agent, focus ONLY on that specific request and do not generate broader test coverage unless explicitly asked to do so. - Start from public API / interface boundaries, then walk inward to critical private helpers. - Analyze function signatures, parameters, return types, and side effects - Map all code paths including happy paths and error conditions - Test behaviour, not implementation details, unless white-box inspection is required to reach untestable paths. - Include both positive and negative test cases - Prefer property-based or table-driven tests where inputs form simple algebraic domains. - Stub or fake **only** the minimal surface area needed; prefer in-memory fakes over mocks when feasible. - Flag any code that cannot be tested deterministically and suggest realistic refactors (seams, dependency injection, pure functions). - Surface concurrency hazards with stress or fuzz tests when the language/runtime supports them. - Focus on realistic failure modes that actually occur in production - Remain within scope of language, framework, project. Do not over-step. Do not add unnecessary dependencies. - No bogus, fake tests that seemingly pass for no reason at all EDGE-CASE TAXONOMY (REAL-WORLD, HIGH-VALUE) - **Data Shape Issues**: `null` / `undefined`, zero-length, surrogate-pair emojis, malformed UTF-8, mixed EOLs. - **Numeric Boundaries**: −1, 0, 1, `MAX_…`, floating-point rounding, 64-bit truncation. - **Temporal Pitfalls**: DST shifts, leap seconds, 29 Feb, Unix epoch 2038, timezone conversions. - **Collections & Iteration**: off-by-one, concurrent modification, empty vs singleton vs large (>10⁶ items). - **State & Sequence**: API calls out of order, idempotency violations, replay attacks. - **External Dependencies**: slow responses, 5xx, malformed JSON/XML, TLS errors, retry storms, cancelled promises. - **Concurrency / Async**: race conditions, deadlocks, promise rejection leaks, thread starvation. - **Resource Exhaustion**: memory spikes, file-descriptor leaks, connection-pool saturation. - **Locale & Encoding**: RTL scripts, uncommon locales, locale-specific formatting. - **Security Surfaces**: injection (SQL, shell, LDAP), path traversal, privilege escalation on shared state. TEST QUALITY PRINCIPLES - Clear Arrange-Act-Assert sections (or given/when/then per project style) but retain and apply project norms, language norms and framework norms and best practices. - One behavioural assertion per test unless grouping is conventional. - Fast: sub-100 ms/unit test; parallelisable; no remote calls. - Deterministic: seeded randomness only; fixed stable clocks when time matters. - Self-documenting: names read like specs; failures explain *why*, not just *what*. FRAMEWORK SELECTION Always autodetect from the repository. When a test framework or existing tests are not found, detect from existing code; examples: - **Swift / Objective-C** → XCTest (Xcode default) or Swift Testing (Apple provided frameworks) - **C# / .NET** → xUnit.net preferred; fall back to NUnit or MSTest if they dominate the repo. - **C / C++** → GoogleTest (gtest/gmock) or Catch2, matching existing tooling. - **JS/TS** → Jest, Vitest, Mocha, or project-specific wrapper. - **Python** → pytest, unittest. - **Java/Kotlin** → JUnit 5, TestNG. - **Go** → built-in `testing`, `testify`. - **Rust** → `#[test]`, `proptest`. - **Anything Else** → follow existing conventions; never introduce a new framework without strong justification. IF FRAMEWORK SELECTION FAILS If you are unable to confidently determine which framework to use based on the existing test samples supplied, or if additional test samples would help in making a final decision, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: {"status": "test_sample_needed", "reason": ""} SCOPE CONTROL Stay strictly within the presented codebase, tech stack, and domain. Do **not** invent features, frameworks, or speculative integrations. Do **not** write tests for functions or classes that do not exist. If a test idea falls outside project scope, discard it. If a test would be a "good to have" but seems impossible given the current structure, setup of the project, highlight it but do not approach or offer refactoring ideas. DELIVERABLE Return only the artefacts (analysis summary, coverage plan, and generated tests) that fit the detected framework and code / project layout. Group related tests but separate them into files where this is the convention and most suitable for the project at hand. Prefer adding tests to an existing test file if one was provided and grouping these tests makes sense. Must document logic, test reason/hypothesis in delivered code. MUST NOT add any additional information, introduction, or summaries around generated code. Deliver only the essentials relevant to the test. IF ADDITIONAL TEST CASES ARE REQUIRED If you determine that comprehensive test coverage requires generating multiple test files or a large number of test cases for each file that would risk exceeding context limits, you MUST follow this structured approach: 1. **Generate Essential Tests First**: Create only the most critical and high-impact tests (typically 3-5 key test cases covering the most important paths and failure modes). Clearly state the file these tests belong to, even if these should be added to an existing test file. 2. **Request Continuation**: You MUST your message with the following added in JSON format (and nothing more after this). This will list the pending tests and their respective files (even if they belong to the same or an existing test file) as this will be used for the next follow-up test generation request. {"status": "more_tests_required", "pending_tests": "test_name (file_name), another_test_name (file_name)"} This approach ensures comprehensive test coverage while maintaining quality and avoiding context overflow. Remember: your value is catching the hard bugs—not inflating coverage numbers. """ ================================================ FILE: systemprompts/thinkdeep_prompt.py ================================================ """ ThinkDeep tool system prompt """ THINKDEEP_PROMPT = """ ROLE You are a senior engineering collaborator working alongside the agent on complex software problems. The agent will send you content—analysis, prompts, questions, ideas, or theories—to deepen, validate, or extend with rigor and clarity. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If you need additional context (e.g., related files, system architecture, requirements, code snippets) to provide thorough analysis, you MUST ONLY respond with this exact JSON (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } GUIDELINES 1. Begin with context analysis: identify tech stack, languages, frameworks, and project constraints. 2. Stay on scope: avoid speculative, over-engineered, or oversized ideas; keep suggestions practical and grounded. 3. Challenge and enrich: find gaps, question assumptions, and surface hidden complexities or risks. 4. Provide actionable next steps: offer specific advice, trade-offs, and implementation strategies. 5. Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment. 6. Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted. 7. Use concise, technical language; assume an experienced engineering audience. 8. Remember: Overengineering is an anti-pattern — avoid suggesting solutions that introduce unnecessary abstraction, indirection, or configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope, and may not arise in the foreseeable future. KEY FOCUS AREAS (apply when relevant) - Architecture & Design: modularity, boundaries, abstraction layers, dependencies - Performance & Scalability: algorithmic efficiency, concurrency, caching, bottlenecks - Security & Safety: validation, authentication/authorization, error handling, vulnerabilities - Quality & Maintainability: readability, testing, monitoring, refactoring - Integration & Deployment: ONLY IF APPLICABLE TO THE QUESTION - external systems, compatibility, configuration, operational concerns EVALUATION Your response will be reviewed by the agent before any decision is made. Your goal is to practically extend the agent's thinking, surface blind spots, and refine options—not to deliver final answers in isolation. REMINDERS - Ground all insights in the current project's architecture, limitations, and goals. - If further context is needed, request it via the clarification JSON—nothing else. - Prioritize depth over breadth; propose alternatives ONLY if they clearly add value and improve the current approach. - Be the ideal development partner—rigorous, focused, and fluent in real-world software trade-offs. """ ================================================ FILE: systemprompts/tracer_prompt.py ================================================ """ Tracer tool system prompts """ TRACER_PROMPT = """ You are an expert, seasoned software architect and code analysis specialist with deep expertise in code tracing, execution flow analysis, and dependency mapping. You have extensive experience analyzing complex codebases, tracing method calls, understanding data flow, and mapping structural relationships in software systems. From microservices to monolithic applications, your ability to understand code structure, execution paths, and dependencies is unmatched. There is nothing related to software architecture, design patterns, or code analysis that you're not aware of. Your role is to systematically trace and analyze code to provide comprehensive understanding of how software components interact and execute. CRITICAL LINE NUMBER INSTRUCTIONS Code is presented with line number markers "LINE│ code". These markers are for reference ONLY and MUST NOT be included in any code you generate. Always reference specific line numbers in your replies in order to locate exact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity. Include context_start_text and context_end_text as backup references. Never include "LINE│" markers in generated code snippets. IF MORE INFORMATION IS NEEDED If the agent is discussing specific code, functions, or project components that was not given as part of the context, and you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful analysis, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["[file name here]", "[or some folder/]"] } TRACING METHODOLOGY: 1. PRECISION MODE (Execution Flow): - Trace method/function execution paths and call chains - Identify entry points and usage patterns - Map conditional branches and control flow - Document side effects and state changes - Analyze parameter flow and return values 2. DEPENDENCIES MODE (Structural Relationships): - Map incoming and outgoing dependencies - Identify type relationships (inheritance, composition, usage) - Trace bidirectional connections between components - Document interface contracts and protocols - Analyze coupling and cohesion patterns ANALYSIS STRUCTURE: Each tracing step MUST include: - Step number and current findings - Files examined and methods analyzed - Concrete evidence from code examination - Relationships discovered (calls, dependencies, usage) - Execution paths or structural patterns identified - Areas requiring deeper investigation TRACING PRINCIPLES: - Start with target identification, then explore systematically - Follow actual code paths, not assumed behavior - Document concrete evidence with file:line references - Consider edge cases, error handling, and conditional logic - Map both direct and indirect relationships - Verify assumptions with code examination STRUCTURED JSON OUTPUT FORMAT: You MUST respond with a properly formatted JSON object following this exact schema. Do NOT include any text before or after the JSON. The response must be valid JSON only. IF MORE INFORMATION IS NEEDED: If you lack critical information to proceed with tracing, you MUST only respond with: { "status": "files_required_to_continue", "mandatory_instructions": "", "files_needed": ["", ""] } FOR NORMAL TRACING RESPONSES: { "status": "tracing_in_progress", "step_number": , "total_steps": , "next_step_required": , "step_content": "", "metadata": { "trace_mode": "", "target_description": "", "step_history_length": }, "tracing_status": { "files_checked": , "relevant_files": , "relevant_context": , "issues_found": 0, "images_collected": , "current_confidence": "", "step_history_length": }, "continuation_id": "", "tracing_complete": , "trace_summary": "", "next_steps": "", "output": { "instructions": "", "format": "", "rendering_instructions": "", "presentation_guidelines": "" } } TRACING CONTENT GUIDELINES: - step_content: Provide detailed analysis of current tracing investigation - Include specific files examined, methods analyzed, and relationships discovered - Reference exact line numbers and code snippets for evidence - Document execution paths, call chains, or dependency relationships - When completing tracing, provide comprehensive trace_summary - next_steps: Always guide the agent on what to investigate next TRACE PRESENTATION GUIDELINES: When tracing is complete (tracing_complete: true), the agent should present the final trace with: FOR PRECISION MODE: - Vertical indented call flow diagrams with exact file:line references - Branching and side effect tables with specific conditions - Usage points with context descriptions - Entry points with trigger scenarios - Visual call chains using arrows and indentation FOR DEPENDENCIES MODE: - Bidirectional arrow flow diagrams showing incoming/outgoing dependencies - Type relationship mappings (inheritance, composition, usage) - Dependency tables with file:line references - Visual connection diagrams with proper arrow directions - Structural relationship analysis IMPORTANT FORMATTING RULES: - Use exact file paths and line numbers from actual codebase - Adapt method naming to match project's programming language conventions - Use proper indentation and visual alignment for call flows - Show conditional execution with explicit condition descriptions - Mark uncertain or ambiguous paths clearly - Include comprehensive side effects categorization Be systematic, thorough, and provide concrete evidence. Your tracing should be detailed enough that someone could follow the exact execution paths or understand the complete dependency structure. """ ================================================ FILE: tests/CASSETTE_MAINTENANCE.md ================================================ # HTTP Cassette Testing - Maintenance Guide ## Overview This project uses HTTP cassettes (recorded HTTP interactions) to test API integrations without making real API calls during CI. This document explains how the cassette system works and how to maintain it. ## How Cassette Matching Works ### Standard Matching (Non-o3 Models) For most models, cassettes match requests using: - HTTP method (GET, POST, etc.) - Request path (/v1/chat/completions, etc.) - **Exact hash of the request body** If ANY part of the request changes, the hash changes and the cassette won't match. ### Semantic Matching (o3 Models) **Problem**: o3 models use system prompts and conversation memory instructions that change frequently with code updates. Using exact hash matching would require re-recording cassettes after every prompt change. **Solution**: o3 models use **semantic matching** that only compares: - Model name (e.g., "o3-pro", "o3-mini") - User's actual question (extracted from request) - Core parameters (reasoning effort, temperature) **Ignored fields** (can change without breaking cassettes): - System prompts - Conversation memory instructions - Follow-up guidance text - Token limits and other metadata ### Example These two requests will match with semantic matching: ```json // Request 1 - Old system prompt { "model": "o3-pro", "reasoning": {"effort": "medium"}, "input": [{ "role": "user", "content": [{ "text": "Old system prompt v1...\n\n=== USER REQUEST ===\nWhat is 2 + 2?\n=== END REQUEST ===\n\nOld instructions..." }] }] } // Request 2 - New system prompt (DIFFERENT) { "model": "o3-pro", "reasoning": {"effort": "medium"}, "input": [{ "role": "user", "content": [{ "text": "New system prompt v2...\n\n=== USER REQUEST ===\nWhat is 2 + 2?\n=== END REQUEST ===\n\nNew instructions..." }] }] } ``` Both extract the same semantic content: ```json { "model": "o3-pro", "reasoning": {"effort": "medium"}, "user_question": "What is 2 + 2?" } ``` ## When to Re-Record Cassettes ### You MUST re-record when: 1. **The user's test question changes** - Example: Changing "What is 2 + 2?" to "What is 3 + 3?" 2. **Core parameters change** - Model name changes (o3-pro → o3-mini) - Reasoning effort changes (medium → high) - Temperature changes 3. **For non-o3 models: ANY request body change** ### You DON'T need to re-record when (o3 models only): 1. **System prompts change** - Semantic matching ignores these 2. **Conversation memory instructions change** - Follow-up guidance text changes - Token limit instructions change 3. **Response format instructions change** - As long as the user's actual question stays the same ## How to Re-Record a Cassette ### Step 1: Delete the Old Cassette ```bash rm tests/openai_cassettes/.json ``` ### Step 2: Run the Test with Real API Key ```bash # Make sure you have a valid API key in .env export OPENAI_API_KEY="your-real-key" # Run the specific test python -m pytest tests/test_o3_pro_output_text_fix.py -v ``` The test will: 1. Detect the missing cassette 2. Make a real API call 3. Record the interaction 4. Save it as a new cassette ### Step 3: Verify the Cassette Works in Replay Mode ```bash # Test with dummy key (forces replay mode) OPENAI_API_KEY="dummy-key" python -m pytest tests/test_o3_pro_output_text_fix.py -v ``` ### Step 4: Commit the New Cassette ```bash git add tests/openai_cassettes/.json git commit -m "chore: re-record cassette for " ``` ## Troubleshooting ### Error: "No matching interaction found" **Cause**: The request body has changed in a way that affects the hash. **For o3 models**: This should NOT happen due to semantic matching. If it does: 1. Check if the user question changed 2. Check if model name or reasoning effort changed 3. Verify semantic matching is working (run `test_cassette_semantic_matching.py`) **For non-o3 models**: This is expected when request changes. Re-record the cassette. **Solution**: Re-record the cassette following the steps above. ### Error: "Cassette file not found" **Cause**: Cassette hasn't been recorded yet or was deleted. **Solution**: Re-record the cassette with a real API key. ### CI Fails but Local Tests Pass **Cause**: 1. You recorded with uncommitted code changes 2. CI is running different code than your local environment **Solution**: 1. Commit all your changes first 2. Then re-record cassettes 3. Commit the cassettes ## Best Practices ### 1. Keep Test Questions Simple - Use simple, stable questions like "What is 2 + 2?" - Avoid questions that might elicit different responses over time ### 2. Document Cassette Recording Conditions - Add comments in tests explaining when recorded - Note any special setup required ### 3. Use Semantic Matching for Prompt-Heavy Tests - If your test involves lots of system prompts, use o3 models - Or extend semantic matching to other models if needed ### 4. Test Both Record and Replay Modes - Always verify cassettes work in replay mode - Ensure tests can record new cassettes when needed ### 5. Don't Commit Cassettes with Secrets - The recording system sanitizes API keys automatically - But double-check for any other sensitive data ## Implementation Details ### Semantic Matching Code The semantic matching is implemented in `tests/http_transport_recorder.py`: - `_is_o3_model_request()`: Detects o3 model requests - `_extract_semantic_fields()`: Extracts only essential fields - `_get_request_signature()`: Generates hash from semantic fields ### Adding Semantic Matching to Other Models To add semantic matching for other models: 1. Update `_is_o3_model_request()` to include your model 2. Update `_extract_semantic_fields()` if needed 3. Add tests in `test_cassette_semantic_matching.py` Example: ```python def _is_o3_model_request(self, content_dict: dict) -> bool: """Check if this is an o3 or other semantic-matching model request.""" model = content_dict.get("model", "") return model.startswith("o3") or model.startswith("gpt-5") # Add more models ``` ## Questions? If you encounter issues with cassette testing: 1. Check this guide first 2. Review existing cassette tests for examples 3. Run semantic matching tests to verify the system 4. Open an issue if you find a bug in the matching logic ## Dual-Model Cassette Coverage Some integration tests maintain cassettes for multiple model variants to ensure regression coverage across model families. For example: ### Consensus Tool Cassettes The `test_consensus_integration.py` test uses parameterized fixtures to test both `gpt-5` and `gpt-5.2` models: - `tests/openai_cassettes/consensus_step1_gpt5_for.json` - Cassette for gpt-5 model - `tests/openai_cassettes/consensus_step1_gpt52_for.json` - Cassette for gpt-5.2 model **When updating consensus cassettes:** 1. Both cassettes should be updated if the test logic changes 2. If only one model's behavior changes, update only that cassette 3. The test uses `@pytest.mark.parametrize` to run against both models 4. Each cassette path is mapped in the `CONSENSUS_CASSETTES` dictionary **To re-record a specific model's cassette:** ```bash # Delete the specific cassette rm tests/openai_cassettes/consensus_step1_gpt5_for.json # Run the test with real API key (it will record for gpt-5) OPENAI_API_KEY="your-real-key" python -m pytest tests/test_consensus_integration.py::test_consensus_multi_model_consultations[gpt-5] -v # Or for gpt-5.2 rm tests/openai_cassettes/consensus_step1_gpt52_for.json OPENAI_API_KEY="your-real-key" python -m pytest tests/test_consensus_integration.py::test_consensus_multi_model_consultations[gpt-5.2] -v ``` This dual-coverage approach ensures that both model families continue to work correctly as the codebase evolves. ## Related Files - `tests/http_transport_recorder.py` - Cassette recording/replay implementation - `tests/transport_helpers.py` - Helper functions for injecting transports - `tests/test_cassette_semantic_matching.py` - Tests for semantic matching - `tests/test_o3_pro_output_text_fix.py` - Example of cassette usage - `tests/test_consensus_integration.py` - Example of dual-model cassette coverage - `tests/openai_cassettes/` - Directory containing recorded cassettes ================================================ FILE: tests/__init__.py ================================================ # Tests for PAL MCP Server ================================================ FILE: tests/conftest.py ================================================ """ Pytest configuration for PAL MCP Server tests """ import asyncio import importlib import os import sys import tempfile from pathlib import Path import pytest # On macOS, the default pytest temp dir is typically under /var (e.g. /private/var/folders/...). # If /var is considered a dangerous system path, tests must use a safe temp root (like /tmp). if sys.platform == "darwin": os.environ["TMPDIR"] = "/tmp" # tempfile caches the temp dir after first lookup; clear it so pytest fixtures pick up TMPDIR. tempfile.tempdir = None # Ensure the parent directory is in the Python path for imports parent_dir = Path(__file__).resolve().parent.parent if str(parent_dir) not in sys.path: sys.path.insert(0, str(parent_dir)) import utils.env as env_config # noqa: E402 # Ensure tests operate with runtime environment rather than .env overrides during imports env_config.reload_env({"PAL_MCP_FORCE_ENV_OVERRIDE": "false"}) # Set default model to a specific value for tests to avoid auto mode # This prevents all tests from failing due to missing model parameter os.environ["DEFAULT_MODEL"] = "gemini-2.5-flash" # Force reload of config module to pick up the env var import config # noqa: E402 importlib.reload(config) # Note: This creates a test sandbox environment # Tests create their own temporary directories as needed # Configure asyncio for Windows compatibility if sys.platform == "win32": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy()) # Register providers for all tests from providers.gemini import GeminiModelProvider # noqa: E402 from providers.openai import OpenAIModelProvider # noqa: E402 from providers.registry import ModelProviderRegistry # noqa: E402 from providers.shared import ProviderType # noqa: E402 from providers.xai import XAIModelProvider # noqa: E402 # Register providers at test startup ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) # Register CUSTOM provider if CUSTOM_API_URL is available (for integration tests) # But only if we're actually running integration tests, not unit tests if os.getenv("CUSTOM_API_URL") and "test_prompt_regression.py" in os.getenv("PYTEST_CURRENT_TEST", ""): from providers.custom import CustomProvider # noqa: E402 def custom_provider_factory(api_key=None): """Factory function that creates CustomProvider with proper parameters.""" base_url = os.getenv("CUSTOM_API_URL", "") return CustomProvider(api_key=api_key or "", base_url=base_url) ModelProviderRegistry.register_provider(ProviderType.CUSTOM, custom_provider_factory) @pytest.fixture def project_path(tmp_path): """ Provides a temporary directory for tests. This ensures all file operations during tests are isolated. """ # Create a subdirectory for this specific test test_dir = tmp_path / "test_workspace" test_dir.mkdir(parents=True, exist_ok=True) return test_dir def _set_dummy_keys_if_missing(): """Set dummy API keys only when they are completely absent.""" for var in ("GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY"): if not os.environ.get(var): os.environ[var] = "dummy-key-for-tests" # Pytest configuration def pytest_configure(config): """Configure pytest with custom markers""" config.addinivalue_line("markers", "asyncio: mark test as async") config.addinivalue_line("markers", "no_mock_provider: disable automatic provider mocking") # Assume we need dummy keys until we learn otherwise config._needs_dummy_keys = True def pytest_collection_modifyitems(session, config, items): """Hook that runs after test collection to check for no_mock_provider markers.""" # Always set dummy keys if real keys are missing # This ensures tests work in CI even with no_mock_provider marker _set_dummy_keys_if_missing() @pytest.fixture(autouse=True) def mock_provider_availability(request, monkeypatch): """ Automatically mock provider availability for all tests to prevent effective auto mode from being triggered when DEFAULT_MODEL is unavailable. This fixture ensures that when tests run with dummy API keys, the tools don't require model selection unless explicitly testing auto mode. """ # Skip this fixture for tests that need real providers if hasattr(request, "node"): marker = request.node.get_closest_marker("no_mock_provider") if marker: return # Ensure providers are registered (in case other tests cleared the registry) from providers.shared import ProviderType registry = ModelProviderRegistry() if ProviderType.GOOGLE not in registry._providers: ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) if ProviderType.OPENAI not in registry._providers: ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) if ProviderType.XAI not in registry._providers: ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) # Ensure CUSTOM provider is registered if needed for integration tests if ( os.getenv("CUSTOM_API_URL") and "test_prompt_regression.py" in os.getenv("PYTEST_CURRENT_TEST", "") and ProviderType.CUSTOM not in registry._providers ): from providers.custom import CustomProvider def custom_provider_factory(api_key=None): base_url = os.getenv("CUSTOM_API_URL", "") return CustomProvider(api_key=api_key or "", base_url=base_url) ModelProviderRegistry.register_provider(ProviderType.CUSTOM, custom_provider_factory) # Also mock is_effective_auto_mode for all BaseTool instances to return False # unless we're specifically testing auto mode behavior from tools.shared.base_tool import BaseTool def mock_is_effective_auto_mode(self): # If this is an auto mode test file or specific auto mode test, use the real logic test_file = request.node.fspath.basename if hasattr(request, "node") and hasattr(request.node, "fspath") else "" test_name = request.node.name if hasattr(request, "node") else "" # Allow auto mode for tests in auto mode files or with auto in the name if ( "auto_mode" in test_file.lower() or "auto" in test_name.lower() or "intelligent_fallback" in test_file.lower() or "per_tool_model_defaults" in test_file.lower() ): # Call original method logic from config import DEFAULT_MODEL if DEFAULT_MODEL.lower() == "auto": return True provider = ModelProviderRegistry.get_provider_for_model(DEFAULT_MODEL) return provider is None # For all other tests, return False to disable auto mode return False monkeypatch.setattr(BaseTool, "is_effective_auto_mode", mock_is_effective_auto_mode) @pytest.fixture(autouse=True) def clear_model_restriction_env(monkeypatch): """Ensure per-test isolation from user-defined model restriction env vars.""" restriction_vars = [ "OPENAI_ALLOWED_MODELS", "GOOGLE_ALLOWED_MODELS", "XAI_ALLOWED_MODELS", "OPENROUTER_ALLOWED_MODELS", "DIAL_ALLOWED_MODELS", ] for var in restriction_vars: monkeypatch.delenv(var, raising=False) @pytest.fixture(autouse=True) def disable_force_env_override(monkeypatch): """Default tests to runtime environment visibility unless they explicitly opt in.""" monkeypatch.setenv("PAL_MCP_FORCE_ENV_OVERRIDE", "false") env_config.reload_env({"PAL_MCP_FORCE_ENV_OVERRIDE": "false"}) monkeypatch.setenv("DEFAULT_MODEL", "gemini-2.5-flash") monkeypatch.setenv("MAX_CONVERSATION_TURNS", "50") import importlib import sys import config import utils.conversation_memory as conversation_memory importlib.reload(config) importlib.reload(conversation_memory) test_conversation_module = sys.modules.get("tests.test_conversation_memory") if test_conversation_module is not None: test_conversation_module.MAX_CONVERSATION_TURNS = conversation_memory.MAX_CONVERSATION_TURNS try: yield finally: env_config.reload_env() ================================================ FILE: tests/gemini_cassettes/chat_codegen/gemini25_pro_calculator/mldev.json ================================================ { "replay_id": "chat_codegen/gemini25_pro_calculator/mldev", "interactions": [ { "request": { "method": "post", "url": "{MLDEV_URL_PREFIX}/models/gemini-2.5-pro:generateContent", "headers": { "Content-Type": "application/json", "x-goog-api-key": "{REDACTED}", "user-agent": "google-genai-sdk/{VERSION_NUMBER} {LANGUAGE_LABEL}/{VERSION_NUMBER}", "x-goog-api-client": "google-genai-sdk/{VERSION_NUMBER} {LANGUAGE_LABEL}/{VERSION_NUMBER}" }, "body_segments": [ { "contents": [ { "parts": [ { "text": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE│ code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE│\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n• Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n• Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n• Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n• Keep proposals practical and directly actionable within the existing architecture.\n• Overengineering is an anti-pattern — avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance.\n2. Engage deeply with the agent's input – extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n4. Present balanced perspectives, outlining trade-offs and their implications.\n5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning.\n6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain.\n\nBRAINSTORMING GUIDELINES\n• Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n• Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n• Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n• Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n• Reference industry best practices relevant to the technologies in use.\n• Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n\n\n# Structured Code Generation Protocol\n\n**WHEN TO USE THIS PROTOCOL:**\n\nUse this structured format ONLY when you are explicitly tasked with substantial code generation, such as:\n- Creating new features from scratch with multiple files or significant code and you have been asked to help implement this\n- Major refactoring across multiple files or large sections of code and you have been tasked to help do this\n- Implementing new modules, components, or subsystems and you have been tasked to help with the implementation\n- Large-scale updates affecting substantial portions of the codebase that you have been asked to help implement\n\n**WHEN NOT TO USE THIS PROTOCOL:**\n\nDo NOT use this format for minor changes:\n- Small tweaks to existing functions or methods (1-20 lines)\n- Bug fixes in isolated sections\n- Simple algorithm improvements\n- Minor refactoring of a single function\n- Adding/removing a few lines of code\n- Quick parameter adjustments or config changes\n\nFor minor changes:\n- Follow the existing instructions provided earlier in your system prompt, such as the CRITICAL LINE NUMBER INSTRUCTIONS.\n- Use inline code blocks with proper line number references and direct explanations instead of this structured format.\n\n**IMPORTANT:** This protocol is for SUBSTANTIAL implementation work when explicitly requested, such as:\n- \"implement feature X\"\n- \"create module Y\"\n- \"refactor system Z\"\n- \"rewrite the authentication logic\"\n- \"redesign the data processing pipeline\"\n- \"rebuild the algorithm from scratch\"\n- \"convert this approach to use a different pattern\"\n- \"create a complete implementation of...\"\n- \"build out the entire workflow for...\"\n\nIf the request is for explanation, analysis, debugging, planning, or discussion WITHOUT substantial code generation, respond normally without this structured format.\n\n## Core Requirements (for substantial code generation tasks)\n\n1. **Complete, Working Code**: Every code block must be fully functional without requiring additional edits. Include all necessary imports, definitions, docstrings, type hints, and error handling.\n\n2. **Clear, Actionable Instructions**: Provide step-by-step guidance using simple numbered lists. Each instruction should map directly to file blocks that follow.\n\n3. **Structured Output Format**: All generated code MUST be contained within a single `` block using the exact structure defined below.\n\n4. **Minimal External Commentary**: Keep any text outside the `` block brief. Reserve detailed explanations for the instruction sections inside the block.\n\n## Required Structure\n\nUse this exact format (do not improvise tag names or reorder components):\n\n```\n\n[Step-by-step instructions for the coding agent]\n1. Create new file [filename] with [description]\n2. Update existing file [filename] by [description]\n3. [Additional steps as needed]\n\n\n[Complete file contents with all necessary components:\n- File-level docstring\n- All imports (standard library, third-party, local)\n- All class/function definitions with complete implementations\n- All necessary helper functions\n- Inline comments for complex logic\n- Type hints where applicable]\n\n\n[Additional instructions for the next file, if needed]\n\n\n[Complete, working code for this file - no partial implementations or placeholders]\n\n\n[Instructions for updating existing files]\n\n\n[Complete replacement code for the modified sections or routines / lines that need updating:\n- Full function/method bodies (not just the changed lines)\n- Complete class definitions if modifying class methods\n- All necessary imports if adding new dependencies\n- Preserve existing code structure and style]\n\n\n[If additional files need updates (based on existing code that was shared with you earlier), repeat the UPDATED_EXISTING_FILE block]\n\n\n[Complete code for this file's modifications]\n\n\n[For file deletions, explicitly state in instructions with justification:\n\"Delete file path/to/obsolete.py - no longer needed because [reason]\"]\n\n```\n\n## Critical Rules\n\n**Completeness:**\n- Never output partial code snippets or placeholder comments like \"# rest of code here\"\n- Include complete function/class implementations from start to finish\n- Add all required imports at the file level\n- Include proper error handling and edge case logic\n\n**Accuracy:**\n- Match the existing codebase indentation style (tabs vs spaces)\n- Preserve language-specific formatting conventions\n- Include trailing newlines where required by language tooling\n- Use correct file paths relative to project root\n\n**Clarity:**\n- Number instructions sequentially (1, 2, 3...)\n- Map each instruction to specific file blocks below it\n- Explain *why* changes are needed, not just *what* changes\n- Highlight any breaking changes or migration steps required\n\n**Structure:**\n- Use `` for files that don't exist yet\n- Use `` for modifying existing files\n- Place instructions between file blocks to provide context\n- Keep the single `` wrapper around everything\n\n## Special Cases\n\n**No Changes Needed:**\nIf the task doesn't require file creation or modification, explicitly state:\n\"No file changes required. The existing implementation already handles [requirement].\"\nDo not emit an empty `` block.\n\n**Configuration Changes:**\nIf modifying configuration files (JSON, YAML, TOML), include complete file contents with the changes applied, not just the changed lines.\n\n**Test Files:**\nWhen generating tests, include complete test suites with:\n- All necessary test fixtures and setup\n- Multiple test cases covering happy path and edge cases\n- Proper teardown and cleanup\n- Clear test descriptions and assertions\n\n**Documentation:**\nInclude docstrings for all public functions, classes, and modules using the project's documentation style (Google, NumPy, Sphinx, etc.).\n\n## Context Awareness\n\n**CRITICAL:** Your implementation builds upon the ongoing conversation context:\n- All previously shared files, requirements, and constraints remain relevant\n- If updating existing code discussed earlier, reference it and preserve unmodified sections\n- If the user shared code for improvement, your generated code should build upon it, not replace everything\n- The coding agent has full conversation history—your instructions should reference prior discussion as needed\n\nYour generated code is NOT standalone—it's a continuation of the collaborative session with full context awareness.\n\n## Remember\n\nThe coding agent depends on this structured format to:\n- Parse and extract code automatically\n- Apply changes to the correct files within the conversation context\n- Validate completeness before execution\n- Track modifications across the codebase\n\nAlways prioritize clarity, completeness, correctness, and context awareness over brevity.\n\n=== USER REQUEST ===\nPlease generate a Python module with functions `add` and `multiply` that perform basic addition and multiplication. Produce the response using the structured format so the assistant can apply the files directly.\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:\n\n\n\nCONVERSATION CONTINUATION: You can continue this discussion with the agent! (49 exchanges remaining)\n\nFeel free to ask clarifying questions or suggest areas for deeper exploration naturally within your response.\nIf something needs clarification or you'd benefit from additional context, simply mention it conversationally.\n\nIMPORTANT: When you suggest follow-ups or ask questions, you MUST explicitly instruct the agent to use the continuation_id\nto respond. Use clear, direct language based on urgency:\n\nFor optional follow-ups: \"Please continue this conversation using the continuation_id from this response if you'd \"\n\"like to explore this further.\"\n\nFor needed responses: \"Please respond using the continuation_id from this response - your input is needed to proceed.\"\n\nFor essential/critical responses: \"RESPONSE REQUIRED: Please immediately continue using the continuation_id from \"\n\"this response. Cannot proceed without your clarification/input.\"\n\nThis ensures the agent knows both HOW to maintain the conversation thread AND whether a response is optional, \"\n\"needed, or essential.\n\nThe tool will automatically provide a continuation_id in the structured response that the agent can use in subsequent\ntool calls to maintain full conversation context across multiple exchanges.\n\nRemember: Only suggest follow-ups when they would genuinely add value to the discussion, and always instruct \"\n\"The agent to use the continuation_id when you do." } ] } ], "generationConfig": { "temperature": 1.0, "candidateCount": 1, "thinkingConfig": { "thinking_budget": 10813 } } } ] }, "response": { "status_code": 200, "headers": { "content-type": "application/json; charset=UTF-8", "vary": "Origin, X-Origin, Referer", "content-encoding": "gzip", "date": "Tue, 07 Oct 2025 14:34:26 GMT", "server": "scaffolding on HTTPServer2", "x-xss-protection": "0", "x-frame-options": "SAMEORIGIN", "x-content-type-options": "nosniff", "server-timing": "gfet4t7; dur=12286", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", "transfer-encoding": "chunked" }, "body_segments": [ { "candidates": [ { "content": { "parts": [ { "text": "\nHere is the implementation for the requested Python module. I've structured this as a proper Python package for better organization and scalability.\n\n1. Create a new file `calculator/__init__.py`. This file makes the `calculator` directory a Python package and exposes the `add` and `multiply` functions for convenient importing (e.g., `from calculator import add`).\n2. Create a new file `calculator/operations.py` to house the core arithmetic functions. This separation of concerns keeps the logic clean.\n\n\n\"\"\"A simple calculator package.\"\"\"\n\nfrom .operations import add, multiply\n\n__all__ = [\"add\", \"multiply\"]\n\n\n\n\"\"\"\nCore arithmetic operations.\n\nThis module provides basic functions for performing mathematical calculations.\n\"\"\"\n\nfrom typing import Union\n\n# Define a type alias for numeric types for clarity\nNumeric = Union[int, float]\n\n\ndef add(a: Numeric, b: Numeric) -> Numeric:\n \"\"\"\n Calculate the sum of two numbers.\n\n Args:\n a: The first number (integer or float).\n b: The second number (integer or float).\n\n Returns:\n The sum of a and b.\n \"\"\"\n return a + b\n\n\ndef multiply(a: Numeric, b: Numeric) -> Numeric:\n \"\"\"\n Calculate the product of two numbers.\n\n Args:\n a: The first number (integer or float).\n b: The second number (integer or float).\n\n Returns:\n The product of a and b.\n \"\"\"\n return a * b\n\n" } ], "role": "model" }, "finishReason": "STOP", "index": 0 } ], "usageMetadata": { "promptTokenCount": 2600, "candidatesTokenCount": 379, "totalTokenCount": 3879, "promptTokensDetails": [ { "modality": "TEXT", "tokenCount": 2600 } ], "thoughtsTokenCount": 900 }, "modelVersion": "gemini-2.5-pro", "responseId": "8iTlaM64EdCwxN8PwYfx0Qo" } ], "byte_segments": [], "sdk_response_segments": [ { "sdk_http_response": { "headers": { "content-type": "application/json; charset=UTF-8", "vary": "Origin, X-Origin, Referer", "content-encoding": "gzip", "date": "Tue, 07 Oct 2025 14:34:26 GMT", "server": "scaffolding on HTTPServer2", "x-xss-protection": "0", "x-frame-options": "SAMEORIGIN", "x-content-type-options": "nosniff", "server-timing": "gfet4t7; dur=12286", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", "transfer-encoding": "chunked" } }, "candidates": [ { "content": { "parts": [ { "text": "\nHere is the implementation for the requested Python module. I've structured this as a proper Python package for better organization and scalability.\n\n1. Create a new file `calculator/__init__.py`. This file makes the `calculator` directory a Python package and exposes the `add` and `multiply` functions for convenient importing (e.g., `from calculator import add`).\n2. Create a new file `calculator/operations.py` to house the core arithmetic functions. This separation of concerns keeps the logic clean.\n\n\n\"\"\"A simple calculator package.\"\"\"\n\nfrom .operations import add, multiply\n\n__all__ = [\"add\", \"multiply\"]\n\n\n\n\"\"\"\nCore arithmetic operations.\n\nThis module provides basic functions for performing mathematical calculations.\n\"\"\"\n\nfrom typing import Union\n\n# Define a type alias for numeric types for clarity\nNumeric = Union[int, float]\n\n\ndef add(a: Numeric, b: Numeric) -> Numeric:\n \"\"\"\n Calculate the sum of two numbers.\n\n Args:\n a: The first number (integer or float).\n b: The second number (integer or float).\n\n Returns:\n The sum of a and b.\n \"\"\"\n return a + b\n\n\ndef multiply(a: Numeric, b: Numeric) -> Numeric:\n \"\"\"\n Calculate the product of two numbers.\n\n Args:\n a: The first number (integer or float).\n b: The second number (integer or float).\n\n Returns:\n The product of a and b.\n \"\"\"\n return a * b\n\n" } ], "role": "model" }, "finish_reason": "STOP", "index": 0 } ], "model_version": "gemini-2.5-pro", "response_id": "8iTlaM64EdCwxN8PwYfx0Qo", "usage_metadata": { "candidates_token_count": 379, "prompt_token_count": 2600, "prompt_tokens_details": [ { "modality": "TEXT", "token_count": 2600 } ], "thoughts_token_count": 900, "total_token_count": 3879 } } ] } } ] } ================================================ FILE: tests/gemini_cassettes/chat_cross/step1_gemini25_flash_number/mldev.json ================================================ { "replay_id": "chat_cross/step1_gemini25_flash_number/mldev", "interactions": [ { "request": { "method": "post", "url": "{MLDEV_URL_PREFIX}/models/gemini-2.5-flash:generateContent", "headers": { "Content-Type": "application/json", "x-goog-api-key": "{REDACTED}", "user-agent": "google-genai-sdk/{VERSION_NUMBER} {LANGUAGE_LABEL}/{VERSION_NUMBER}", "x-goog-api-client": "google-genai-sdk/{VERSION_NUMBER} {LANGUAGE_LABEL}/{VERSION_NUMBER}" }, "body_segments": [ { "contents": [ { "parts": [ { "text": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE│ code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE│\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n• Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n• Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n• Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n• Keep proposals practical and directly actionable within the existing architecture.\n• Overengineering is an anti-pattern — avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance.\n2. Engage deeply with the agent's input – extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n4. Present balanced perspectives, outlining trade-offs and their implications.\n5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning.\n6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain.\n\nBRAINSTORMING GUIDELINES\n• Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n• Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n• Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n• Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n• Reference industry best practices relevant to the technologies in use.\n• Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n\n\n=== USER REQUEST ===\nPick a number between 1 and 10 and respond with JUST that number.\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:\n\n\n\nCONVERSATION CONTINUATION: You can continue this discussion with the agent! (49 exchanges remaining)\n\nFeel free to ask clarifying questions or suggest areas for deeper exploration naturally within your response.\nIf something needs clarification or you'd benefit from additional context, simply mention it conversationally.\n\nIMPORTANT: When you suggest follow-ups or ask questions, you MUST explicitly instruct the agent to use the continuation_id\nto respond. Use clear, direct language based on urgency:\n\nFor optional follow-ups: \"Please continue this conversation using the continuation_id from this response if you'd \"\n\"like to explore this further.\"\n\nFor needed responses: \"Please respond using the continuation_id from this response - your input is needed to proceed.\"\n\nFor essential/critical responses: \"RESPONSE REQUIRED: Please immediately continue using the continuation_id from \"\n\"this response. Cannot proceed without your clarification/input.\"\n\nThis ensures the agent knows both HOW to maintain the conversation thread AND whether a response is optional, \"\n\"needed, or essential.\n\nThe tool will automatically provide a continuation_id in the structured response that the agent can use in subsequent\ntool calls to maintain full conversation context across multiple exchanges.\n\nRemember: Only suggest follow-ups when they would genuinely add value to the discussion, and always instruct \"\n\"The agent to use the continuation_id when you do." } ] } ], "generationConfig": { "temperature": 0.2, "candidateCount": 1, "thinkingConfig": { "thinking_budget": 8110 } } } ] }, "response": { "status_code": 200, "headers": { "content-type": "application/json; charset=UTF-8", "vary": "Origin, X-Origin, Referer", "content-encoding": "gzip", "date": "Sat, 04 Oct 2025 10:14:27 GMT", "server": "scaffolding on HTTPServer2", "x-xss-protection": "0", "x-frame-options": "SAMEORIGIN", "x-content-type-options": "nosniff", "server-timing": "gfet4t7; dur=1246", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", "transfer-encoding": "chunked" }, "body_segments": [ { "candidates": [ { "content": { "parts": [ { "text": "7" } ], "role": "model" }, "finishReason": "STOP", "index": 0 } ], "usageMetadata": { "promptTokenCount": 1085, "candidatesTokenCount": 1, "totalTokenCount": 1149, "promptTokensDetails": [ { "modality": "TEXT", "tokenCount": 1085 } ], "thoughtsTokenCount": 63 }, "modelVersion": "gemini-2.5-flash", "responseId": "g_PgaIL5LL6VkdUPgr3q2A8" } ], "byte_segments": [], "sdk_response_segments": [ { "sdk_http_response": { "headers": { "content-type": "application/json; charset=UTF-8", "vary": "Origin, X-Origin, Referer", "content-encoding": "gzip", "date": "Sat, 04 Oct 2025 10:14:27 GMT", "server": "scaffolding on HTTPServer2", "x-xss-protection": "0", "x-frame-options": "SAMEORIGIN", "x-content-type-options": "nosniff", "server-timing": "gfet4t7; dur=1246", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", "transfer-encoding": "chunked" } }, "candidates": [ { "content": { "parts": [ { "text": "7" } ], "role": "model" }, "finish_reason": "STOP", "index": 0 } ], "model_version": "gemini-2.5-flash", "response_id": "g_PgaIL5LL6VkdUPgr3q2A8", "usage_metadata": { "candidates_token_count": 1, "prompt_token_count": 1085, "prompt_tokens_details": [ { "modality": "TEXT", "token_count": 1085 } ], "thoughts_token_count": 63, "total_token_count": 1149 } } ] } } ] } ================================================ FILE: tests/gemini_cassettes/consensus/step2_gemini25_flash_against/mldev.json ================================================ { "replay_id": "consensus/step2_gemini25_flash_against/mldev", "interactions": [ { "request": { "method": "post", "url": "{MLDEV_URL_PREFIX}/models/gemini-2.5-flash:generateContent", "headers": { "Content-Type": "application/json", "x-goog-api-key": "{REDACTED}", "user-agent": "google-genai-sdk/{VERSION_NUMBER} {LANGUAGE_LABEL}/{VERSION_NUMBER}", "x-goog-api-client": "google-genai-sdk/{VERSION_NUMBER} {LANGUAGE_LABEL}/{VERSION_NUMBER}" }, "body_segments": [ { "contents": [ { "parts": [ { "text": "\nROLE\nYou are an expert technical consultant providing consensus analysis on proposals, plans, and ideas. The agent will present you\nwith a technical proposition and your task is to deliver a structured, rigorous assessment that helps validate feasibility\nand implementation approaches.\n\nYour feedback carries significant weight - it may directly influence project decisions, future direction, and could have\nbroader impacts on scale, revenue, and overall scope. The questioner values your expertise immensely and relies on your\nanalysis to make informed decisions that affect their success.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE│ code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE│\" markers in generated code\nsnippets.\n\nPERSPECTIVE FRAMEWORK\nCRITICAL PERSPECTIVE WITH RESPONSIBILITY\n\nYou are tasked with critiquing this proposal, but with ESSENTIAL BOUNDARIES:\n\nMANDATORY FAIRNESS CONSTRAINTS:\n- You MUST NOT oppose genuinely excellent, common-sense ideas just to be contrarian\n- You MUST acknowledge when a proposal is fundamentally sound and well-conceived\n- You CANNOT give harmful advice or recommend against beneficial changes\n- If the idea is outstanding, say so clearly while offering constructive refinements\n\nWHEN TO MODERATE CRITICISM (MUST OVERRIDE STANCE):\n- If the proposal addresses critical user needs effectively\n- If it follows established best practices with good reason\n- If benefits clearly and substantially outweigh risks\n- If it's the obvious right solution to the problem\n\nYOUR CRITICAL ANALYSIS SHOULD:\n- Identify legitimate risks and failure modes\n- Point out overlooked complexities\n- Suggest more efficient alternatives\n- Highlight potential negative consequences\n- Question assumptions that may be flawed\n\nRemember: Being \"against\" means rigorous scrutiny to ensure quality, not undermining good ideas that deserve support.\n\nIF MORE INFORMATION IS NEEDED\nIMPORTANT: Only request files for TECHNICAL IMPLEMENTATION questions where you need to see actual code, architecture,\nor technical specifications. For business strategy, product decisions, or conceptual questions, provide analysis based\non the information given rather than requesting technical files.\n\nIf you need additional technical context (e.g., related files, system architecture, requirements, code snippets) to\nprovide thorough analysis of TECHNICAL IMPLEMENTATION details, you MUST ONLY respond with this exact JSON (and nothing else).\nDo NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nFor business strategy, product planning, or conceptual questions, proceed with analysis using your expertise and the\ncontext provided, even if specific technical details are not available.\n\nEVALUATION FRAMEWORK\nAssess the proposal across these critical dimensions. Your stance influences HOW you present findings, not WHETHER you\nacknowledge fundamental truths about feasibility, safety, or value:\n\n1. TECHNICAL FEASIBILITY\n - Is this technically achievable with reasonable effort?\n - What are the core technical dependencies and requirements?\n - Are there any fundamental technical blockers?\n\n2. PROJECT SUITABILITY\n - Does this fit the existing codebase architecture and patterns?\n - Is it compatible with current technology stack and constraints?\n - How well does it align with the project's technical direction?\n\n3. USER VALUE ASSESSMENT\n - Will users actually want and use this feature?\n - What concrete benefits does this provide?\n - How does this compare to alternative solutions?\n\n4. IMPLEMENTATION COMPLEXITY\n - What are the main challenges, risks, and dependencies?\n - What is the estimated effort and timeline?\n - What expertise and resources are required?\n\n5. ALTERNATIVE APPROACHES\n - Are there simpler ways to achieve the same goals?\n - What are the trade-offs between different approaches?\n - Should we consider a different strategy entirely?\n\n6. INDUSTRY PERSPECTIVE\n - How do similar products/companies handle this problem?\n - What are current best practices and emerging patterns?\n - Are there proven solutions or cautionary tales?\n\n7. LONG-TERM IMPLICATIONS\n - Maintenance burden and technical debt considerations\n - Scalability and performance implications\n - Evolution and extensibility potential\n\nMANDATORY RESPONSE FORMAT\nYou MUST respond in exactly this Markdown structure. Do not deviate from this format:\n\n## Verdict\nProvide a single, clear sentence summarizing your overall assessment (e.g., \"Technically feasible but requires significant\ninfrastructure investment\", \"Strong user value proposition with manageable implementation risks\", \"Overly complex approach -\nrecommend simplified alternative\").\n\n## Analysis\nProvide detailed assessment addressing each point in the evaluation framework. Use clear reasoning and specific examples.\nBe thorough but concise. Address both strengths and weaknesses objectively.\n\n## Confidence Score\nProvide a numerical score from 1 (low confidence) to 10 (high confidence) followed by a brief justification explaining what\ndrives your confidence level and what uncertainties remain.\nFormat: \"X/10 - [brief justification]\"\nExample: \"7/10 - High confidence in technical feasibility assessment based on similar implementations, but uncertain about\nuser adoption without market validation data.\"\n\n## Key Takeaways\nProvide 3-5 bullet points highlighting the most critical insights, risks, or recommendations. These should be actionable\nand specific.\n\nQUALITY STANDARDS\n- Ground all insights in the current project's scope and constraints\n- Be honest about limitations and uncertainties\n- Focus on practical, implementable solutions rather than theoretical possibilities\n- Provide specific, actionable guidance rather than generic advice\n- Balance optimism with realistic risk assessment\n- Reference concrete examples and precedents when possible\n\nREMINDERS\n- Your assessment will be synthesized with other expert opinions by the agent\n- Aim to provide unique insights that complement other perspectives\n- If files are provided, reference specific technical details in your analysis\n- Maintain professional objectivity while being decisive in your recommendations\n- Keep your response concise - your entire reply must not exceed 850 tokens to ensure transport compatibility\n- CRITICAL: Your stance does NOT override your responsibility to provide truthful, ethical, and beneficial guidance\n- Bad ideas must be called out regardless of stance; good ideas must be acknowledged regardless of stance\n\n\nEvaluate SwiftUI vs UIKit adoption and recommend ONE word (SwiftUI or UIKit)." } ] } ], "generationConfig": { "temperature": 1.0, "candidateCount": 1, "thinkingConfig": { "thinking_budget": 8110 } } } ] }, "response": { "status_code": 200, "headers": { "content-type": "application/json; charset=UTF-8", "vary": "Origin, X-Origin, Referer", "content-encoding": "gzip", "date": "Thu, 11 Dec 2025 19:08:00 GMT", "server": "scaffolding on HTTPServer2", "x-xss-protection": "0", "x-frame-options": "SAMEORIGIN", "x-content-type-options": "nosniff", "server-timing": "gfet4t7; dur=15067", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", "transfer-encoding": "chunked" }, "body_segments": [ { "candidates": [ { "content": { "parts": [ { "text": "## Verdict\nAdopting SwiftUI for new development or significant feature builds is the most strategic long-term direction, despite initial learning curves and some existing ecosystem limitations.\n\n## Analysis\n\n### 1. TECHNICAL FEASIBILITY\nBoth SwiftUI and UIKit are technically feasible and robust frameworks for building iOS applications. UIKit, being the older and more mature framework, offers unparalleled stability and a vast ecosystem of third-party libraries. SwiftUI, while newer, is fully capable of building complex UIs and is Apple's declared future for UI development across all its platforms. The primary technical dependencies for SwiftUI are Swift and Xcode, along with a minimum deployment target of iOS 13 (though iOS 15+ is recommended for a smoother experience).\n\n### 2. PROJECT SUITABILITY\nFor greenfield projects or new modules within existing applications, SwiftUI offers superior project suitability. It integrates seamlessly with modern Swift language features, leverages declarative programming paradigms, and promotes a more component-based architecture. For existing UIKit-heavy projects, adopting SwiftUI for new features is suitable, as the frameworks can coexist and interoperate through `UIViewRepresentable` and `UIViewControllerRepresentable`. However, a full migration of an existing, large UIKit codebase to SwiftUI is a massive undertaking, often not justified unless there's a strong business case for a complete rewrite.\n\n### 3. USER VALUE ASSESSMENT\nThe choice between SwiftUI and UIKit doesn't directly impact the end-user experience from a functional perspective; both can deliver high-quality interfaces. However, SwiftUI's declarative nature often leads to faster development cycles and easier iteration, which can indirectly translate to more features, quicker bug fixes, and a more polished product delivered to users over time. Its multi-platform capabilities also allow for more consistent experiences across Apple's device ecosystem with less effort.\n\n### 4. IMPLEMENTATION COMPLEXITY\nImplementing with SwiftUI introduces a learning curve for teams accustomed to imperative UIKit. Concepts like state management (`@State`, `@Binding`, `@ObservedObject`, `@StateObject`, `@EnvironmentObject`) require a shift in mindset. While SwiftUI often reduces boilerplate code and simplifies common UI tasks, achieving highly custom layouts or integrating specific legacy UIKit components can sometimes be more complex due to the need for representable wrappers. Debugging can also feel less mature compared to UIKit's well-established tooling. Conversely, UIKit, while verbose, has predictable patterns and extensive community resources.\n\n### 5. ALTERNATIVE APPROACHES\nWhile UIKit is the immediate alternative, for native iOS development, there are no other first-party Apple UI frameworks. Third-party cross-platform frameworks (e.g., React Native, Flutter) exist, but they introduce their own set of trade-offs regarding native look-and-feel, performance, and dependency on non-Apple ecosystems. Sticking to native development for iOS offers the best performance, user experience, and access to the latest platform features.\n\n### 6. INDUSTRY PERSPECTIVE\nThe industry is undeniably shifting towards SwiftUI. Apple continues to heavily invest in SwiftUI, adding new features and improving performance with each OS release. Many new applications and significant updates from established companies are leveraging SwiftUI, especially for components that benefit from its declarative nature and multi-platform reach. UIKit remains indispensable for maintaining legacy applications and for specific, highly customized UI components where SwiftUI's abstractions might be limiting.\n\n### 7. LONG-TERM IMPLICATIONS\nAdopting SwiftUI aligns with Apple's long-term vision, ensuring better future compatibility, reduced technical debt for new features, and easier adoption of new platform capabilities. Its declarative nature generally leads to more readable and maintainable codebases over time. The multi-platform story with SwiftUI for macOS, watchOS, and tvOS is a significant long-term advantage, reducing development effort for cross-device experiences. Continuing solely with UIKit for new development increases the risk of accumulating technical debt and missing out on future productivity gains.\n\n## Confidence Score\n9/10 - High confidence based on extensive industry observations, Apple's clear strategic direction, and practical experience with both frameworks across various project scales. The remaining uncertainty relates to specific project constraints not provided, such as required iOS target versions or existing team expertise levels.\n\n## Key Takeaways\n* **Embrace SwiftUI for new development:** Prioritize SwiftUI for all new features and greenfield projects to capitalize on future productivity and maintainability benefits.\n* **Plan for interoperability:** Design new SwiftUI components to integrate seamlessly with existing UIKit code where necessary, utilizing `UIViewRepresentable` and `UIViewControllerRepresentable`.\n* **Invest in team training:** Allocate resources for team members to upskill in SwiftUI's declarative paradigm and state management best practices.\n* **Assess minimum OS compatibility:** Be mindful of SwiftUI's minimum iOS 13 deployment target (iOS 15+ for richer features) and ensure it aligns with user base requirements.\n* **Strategic migration for legacy:** For existing UIKit apps, consider a phased migration strategy, starting with isolated components or entire new features in SwiftUI rather than a full rewrite." } ], "role": "model" }, "finishReason": "STOP", "index": 0 } ], "usageMetadata": { "promptTokenCount": 1404, "candidatesTokenCount": 1034, "totalTokenCount": 3703, "promptTokensDetails": [ { "modality": "TEXT", "tokenCount": 1404 } ], "thoughtsTokenCount": 1265 }, "modelVersion": "gemini-2.5-flash", "responseId": "kBY7aeSaMs6JkdUP4OHz0AE" } ], "byte_segments": [], "sdk_response_segments": [ { "sdk_http_response": { "headers": { "content-type": "application/json; charset=UTF-8", "vary": "Origin, X-Origin, Referer", "content-encoding": "gzip", "date": "Thu, 11 Dec 2025 19:08:00 GMT", "server": "scaffolding on HTTPServer2", "x-xss-protection": "0", "x-frame-options": "SAMEORIGIN", "x-content-type-options": "nosniff", "server-timing": "gfet4t7; dur=15067", "alt-svc": "h3=\":443\"; ma=2592000,h3-29=\":443\"; ma=2592000", "transfer-encoding": "chunked" } }, "candidates": [ { "content": { "parts": [ { "text": "## Verdict\nAdopting SwiftUI for new development or significant feature builds is the most strategic long-term direction, despite initial learning curves and some existing ecosystem limitations.\n\n## Analysis\n\n### 1. TECHNICAL FEASIBILITY\nBoth SwiftUI and UIKit are technically feasible and robust frameworks for building iOS applications. UIKit, being the older and more mature framework, offers unparalleled stability and a vast ecosystem of third-party libraries. SwiftUI, while newer, is fully capable of building complex UIs and is Apple's declared future for UI development across all its platforms. The primary technical dependencies for SwiftUI are Swift and Xcode, along with a minimum deployment target of iOS 13 (though iOS 15+ is recommended for a smoother experience).\n\n### 2. PROJECT SUITABILITY\nFor greenfield projects or new modules within existing applications, SwiftUI offers superior project suitability. It integrates seamlessly with modern Swift language features, leverages declarative programming paradigms, and promotes a more component-based architecture. For existing UIKit-heavy projects, adopting SwiftUI for new features is suitable, as the frameworks can coexist and interoperate through `UIViewRepresentable` and `UIViewControllerRepresentable`. However, a full migration of an existing, large UIKit codebase to SwiftUI is a massive undertaking, often not justified unless there's a strong business case for a complete rewrite.\n\n### 3. USER VALUE ASSESSMENT\nThe choice between SwiftUI and UIKit doesn't directly impact the end-user experience from a functional perspective; both can deliver high-quality interfaces. However, SwiftUI's declarative nature often leads to faster development cycles and easier iteration, which can indirectly translate to more features, quicker bug fixes, and a more polished product delivered to users over time. Its multi-platform capabilities also allow for more consistent experiences across Apple's device ecosystem with less effort.\n\n### 4. IMPLEMENTATION COMPLEXITY\nImplementing with SwiftUI introduces a learning curve for teams accustomed to imperative UIKit. Concepts like state management (`@State`, `@Binding`, `@ObservedObject`, `@StateObject`, `@EnvironmentObject`) require a shift in mindset. While SwiftUI often reduces boilerplate code and simplifies common UI tasks, achieving highly custom layouts or integrating specific legacy UIKit components can sometimes be more complex due to the need for representable wrappers. Debugging can also feel less mature compared to UIKit's well-established tooling. Conversely, UIKit, while verbose, has predictable patterns and extensive community resources.\n\n### 5. ALTERNATIVE APPROACHES\nWhile UIKit is the immediate alternative, for native iOS development, there are no other first-party Apple UI frameworks. Third-party cross-platform frameworks (e.g., React Native, Flutter) exist, but they introduce their own set of trade-offs regarding native look-and-feel, performance, and dependency on non-Apple ecosystems. Sticking to native development for iOS offers the best performance, user experience, and access to the latest platform features.\n\n### 6. INDUSTRY PERSPECTIVE\nThe industry is undeniably shifting towards SwiftUI. Apple continues to heavily invest in SwiftUI, adding new features and improving performance with each OS release. Many new applications and significant updates from established companies are leveraging SwiftUI, especially for components that benefit from its declarative nature and multi-platform reach. UIKit remains indispensable for maintaining legacy applications and for specific, highly customized UI components where SwiftUI's abstractions might be limiting.\n\n### 7. LONG-TERM IMPLICATIONS\nAdopting SwiftUI aligns with Apple's long-term vision, ensuring better future compatibility, reduced technical debt for new features, and easier adoption of new platform capabilities. Its declarative nature generally leads to more readable and maintainable codebases over time. The multi-platform story with SwiftUI for macOS, watchOS, and tvOS is a significant long-term advantage, reducing development effort for cross-device experiences. Continuing solely with UIKit for new development increases the risk of accumulating technical debt and missing out on future productivity gains.\n\n## Confidence Score\n9/10 - High confidence based on extensive industry observations, Apple's clear strategic direction, and practical experience with both frameworks across various project scales. The remaining uncertainty relates to specific project constraints not provided, such as required iOS target versions or existing team expertise levels.\n\n## Key Takeaways\n* **Embrace SwiftUI for new development:** Prioritize SwiftUI for all new features and greenfield projects to capitalize on future productivity and maintainability benefits.\n* **Plan for interoperability:** Design new SwiftUI components to integrate seamlessly with existing UIKit code where necessary, utilizing `UIViewRepresentable` and `UIViewControllerRepresentable`.\n* **Invest in team training:** Allocate resources for team members to upskill in SwiftUI's declarative paradigm and state management best practices.\n* **Assess minimum OS compatibility:** Be mindful of SwiftUI's minimum iOS 13 deployment target (iOS 15+ for richer features) and ensure it aligns with user base requirements.\n* **Strategic migration for legacy:** For existing UIKit apps, consider a phased migration strategy, starting with isolated components or entire new features in SwiftUI rather than a full rewrite." } ], "role": "model" }, "finish_reason": "STOP", "index": 0 } ], "model_version": "gemini-2.5-flash", "response_id": "kBY7aeSaMs6JkdUP4OHz0AE", "usage_metadata": { "candidates_token_count": 1034, "prompt_token_count": 1404, "prompt_tokens_details": [ { "modality": "TEXT", "token_count": 1404 } ], "thoughts_token_count": 1265, "total_token_count": 3703 } } ] } } ] } ================================================ FILE: tests/http_transport_recorder.py ================================================ #!/usr/bin/env python3 """ HTTP Transport Recorder for O3-Pro Testing Custom httpx transport solution that replaces respx for recording/replaying HTTP interactions. Provides full control over the recording process without respx limitations. Key Features: - RecordingTransport: Wraps default transport, captures real HTTP calls - ReplayTransport: Serves saved responses from cassettes - TransportFactory: Auto-selects record vs replay mode - JSON cassette format with data sanitization """ import base64 import hashlib import json import logging from pathlib import Path from typing import Any, Optional import httpx from .pii_sanitizer import PIISanitizer logger = logging.getLogger(__name__) class RecordingTransport(httpx.HTTPTransport): """Transport that wraps default httpx transport and records all interactions.""" def __init__(self, cassette_path: str, capture_content: bool = True, sanitize: bool = True): super().__init__() self.cassette_path = Path(cassette_path) self.recorded_interactions = [] self.capture_content = capture_content self.sanitizer = PIISanitizer() if sanitize else None def handle_request(self, request: httpx.Request) -> httpx.Response: """Handle request by recording interaction and delegating to real transport.""" logger.debug(f"RecordingTransport: Making request to {request.method} {request.url}") # Record request BEFORE making the call request_data = self._serialize_request(request) # Make real HTTP call using parent transport response = super().handle_request(request) logger.debug(f"RecordingTransport: Got response {response.status_code}") # Post-response content capture (proper approach) if self.capture_content: try: # Consume the response stream to capture content # Note: httpx automatically handles gzip decompression content_bytes = response.read() response.close() # Close the original stream logger.debug(f"RecordingTransport: Captured {len(content_bytes)} bytes") # Serialize response with captured content response_data = self._serialize_response_with_content(response, content_bytes) # Create a new response with the same metadata but buffered content # If the original response was gzipped, we need to re-compress response_content = content_bytes if response.headers.get("content-encoding") == "gzip": import gzip response_content = gzip.compress(content_bytes) logger.debug(f"Re-compressed content: {len(content_bytes)} → {len(response_content)} bytes") new_response = httpx.Response( status_code=response.status_code, headers=response.headers, # Keep original headers intact content=response_content, request=request, extensions=response.extensions, history=response.history, ) # Record the interaction self._record_interaction(request_data, response_data) return new_response except Exception: logger.warning("Content capture failed, falling back to stub", exc_info=True) response_data = self._serialize_response(response) self._record_interaction(request_data, response_data) return response else: # Legacy mode: record with stub content response_data = self._serialize_response(response) self._record_interaction(request_data, response_data) return response def _record_interaction(self, request_data: dict[str, Any], response_data: dict[str, Any]): """Helper method to record interaction and save cassette.""" interaction = {"request": request_data, "response": response_data} self.recorded_interactions.append(interaction) self._save_cassette() logger.debug(f"Saved cassette to {self.cassette_path}") def _serialize_request(self, request: httpx.Request) -> dict[str, Any]: """Serialize httpx.Request to JSON-compatible format.""" # For requests, we can safely read the content since it's already been prepared # httpx.Request.content is safe to access multiple times content = request.content # Convert bytes to string for JSON serialization if isinstance(content, bytes): try: content_str = content.decode("utf-8") except UnicodeDecodeError: # Handle binary content (shouldn't happen for o3-pro API) content_str = content.hex() else: content_str = str(content) if content else "" request_data = { "method": request.method, "url": str(request.url), "path": request.url.path, "headers": dict(request.headers), "content": self._sanitize_request_content(content_str), } # Apply PII sanitization if enabled if self.sanitizer: request_data = self.sanitizer.sanitize_request(request_data) return request_data def _serialize_response(self, response: httpx.Response) -> dict[str, Any]: """Serialize httpx.Response to JSON-compatible format (legacy method without content).""" # Legacy method for backward compatibility when content capture is disabled return { "status_code": response.status_code, "headers": dict(response.headers), "content": {"note": "Response content not recorded to avoid httpx.ResponseNotRead exception"}, "reason_phrase": response.reason_phrase, } def _serialize_response_with_content(self, response: httpx.Response, content_bytes: bytes) -> dict[str, Any]: """Serialize httpx.Response with captured content.""" try: # Debug: check what we got # Ensure we have bytes for base64 encoding if not isinstance(content_bytes, bytes): logger.warning(f"Content is not bytes, converting from {type(content_bytes)}") if isinstance(content_bytes, str): content_bytes = content_bytes.encode("utf-8") else: content_bytes = str(content_bytes).encode("utf-8") # Encode content as base64 for JSON storage content_b64 = base64.b64encode(content_bytes).decode("utf-8") logger.debug(f"Base64 encoded {len(content_bytes)} bytes → {len(content_b64)} chars") response_data = { "status_code": response.status_code, "headers": dict(response.headers), "content": {"data": content_b64, "encoding": "base64", "size": len(content_bytes)}, "reason_phrase": response.reason_phrase, } # Apply PII sanitization if enabled if self.sanitizer: response_data = self.sanitizer.sanitize_response(response_data) return response_data except Exception as e: logger.exception("Error in _serialize_response_with_content") # Fall back to minimal info return { "status_code": response.status_code, "headers": dict(response.headers), "content": {"error": f"Failed to serialize content: {e}"}, "reason_phrase": response.reason_phrase, } def _sanitize_request_content(self, content: str) -> Any: """Sanitize request content to remove sensitive data.""" try: if content.strip(): data = json.loads(content) # Don't sanitize request content for now - it's user input return data except json.JSONDecodeError: pass return content def _save_cassette(self): """Save recorded interactions to cassette file.""" # Ensure directory exists self.cassette_path.parent.mkdir(parents=True, exist_ok=True) # Save cassette cassette_data = {"interactions": self.recorded_interactions} self.cassette_path.write_text(json.dumps(cassette_data, indent=2, sort_keys=True)) class ReplayTransport(httpx.MockTransport): """Transport that replays saved HTTP interactions from cassettes.""" def __init__(self, cassette_path: str): self.cassette_path = Path(cassette_path) self.interactions = self._load_cassette() super().__init__(self._handle_request) def _load_cassette(self) -> list: """Load interactions from cassette file.""" if not self.cassette_path.exists(): raise FileNotFoundError(f"Cassette file not found: {self.cassette_path}") try: cassette_data = json.loads(self.cassette_path.read_text()) return cassette_data.get("interactions", []) except json.JSONDecodeError as e: raise ValueError(f"Invalid cassette file format: {e}") def _handle_request(self, request: httpx.Request) -> httpx.Response: """Handle request by finding matching interaction and returning saved response.""" logger.debug(f"ReplayTransport: Looking for {request.method} {request.url}") # Debug: show what we're trying to match request_signature = self._get_request_signature(request) logger.debug(f"Request signature: {request_signature}") # Find matching interaction interaction = self._find_matching_interaction(request) if not interaction: logger.warning("No matching interaction found in cassette") raise ValueError(f"No matching interaction found for {request.method} {request.url}") logger.debug("Found matching interaction in cassette") # Build response from saved data response_data = interaction["response"] # Convert content back to appropriate format content = response_data.get("content", {}) if isinstance(content, dict): # Check if this is base64-encoded content if content.get("encoding") == "base64" and "data" in content: # Decode base64 content try: content_bytes = base64.b64decode(content["data"]) logger.debug(f"Decoded {len(content_bytes)} bytes from base64") except Exception as e: logger.warning(f"Failed to decode base64 content: {e}") content_bytes = json.dumps(content).encode("utf-8") else: # Legacy format or stub content content_bytes = json.dumps(content).encode("utf-8") else: content_bytes = str(content).encode("utf-8") # Check if response expects gzipped content headers = response_data.get("headers", {}) if headers.get("content-encoding") == "gzip": # Re-compress the content for httpx import gzip content_bytes = gzip.compress(content_bytes) logger.debug(f"Re-compressed for replay: {len(content_bytes)} bytes") logger.debug(f"Returning cassette response ({len(content_bytes)} bytes)") # Create httpx.Response return httpx.Response( status_code=response_data["status_code"], headers=response_data.get("headers", {}), content=content_bytes, request=request, ) def _find_matching_interaction(self, request: httpx.Request) -> Optional[dict[str, Any]]: """Find interaction that matches the request.""" request_signature = self._get_request_signature(request) for interaction in self.interactions: saved_signature = self._get_saved_request_signature(interaction["request"]) if request_signature == saved_signature: return interaction return None def _get_request_signature(self, request: httpx.Request) -> str: """Generate signature for request matching. Uses semantic matching for o3 models to avoid cassette breaks from prompt changes. For o3 models, matches on model name and user prompt only, ignoring system prompts that may change between code versions. """ # Use method, path, and content hash for matching content = request.content if hasattr(content, "read"): content = content.read() if isinstance(content, bytes): content_str = content.decode("utf-8", errors="ignore") else: content_str = str(content) if content else "" # Parse JSON and re-serialize with sorted keys for consistent hashing try: if content_str.strip(): content_dict = json.loads(content_str) # For o3 models, use semantic matching to avoid cassette breaks if self._is_o3_model_request(content_dict): # Extract only the essential fields for matching semantic_dict = self._extract_semantic_fields(content_dict) content_str = json.dumps(semantic_dict, sort_keys=True) else: content_str = json.dumps(content_dict, sort_keys=True) except json.JSONDecodeError: # Not JSON, use as-is pass # Create hash of content for stable matching content_hash = hashlib.md5(content_str.encode()).hexdigest() return f"{request.method}:{request.url.path}:{content_hash}" def _is_o3_model_request(self, content_dict: dict) -> bool: """Check if this is an o3 model request.""" model = content_dict.get("model", "") return model.startswith("o3") def _extract_semantic_fields(self, content_dict: dict) -> dict: """Extract only semantic fields for matching, ignoring volatile prompts. For o3 models, we want to match on: - Model name - User's actual question (last user message) - Core parameters (temperature, reasoning effort) We ignore: - System prompts (change frequently with code updates) - Conversation memory instructions (change with features) """ semantic = { "model": content_dict.get("model"), "reasoning": content_dict.get("reasoning"), } # Extract only the last user message (actual user question) input_messages = content_dict.get("input", []) if input_messages: # Get the last user message content last_msg = input_messages[-1] if isinstance(last_msg, dict) and last_msg.get("role") == "user": content = last_msg.get("content", []) if isinstance(content, list) and len(content) > 0: # Extract just the text from the last message last_text = content[-1].get("text", "") # Only include the actual question, not the system instructions if "=== USER REQUEST ===" in last_text: # Extract just the user question parts = last_text.split("=== USER REQUEST ===") if len(parts) > 1: user_question = parts[1].split("=== END REQUEST ===")[0].strip() semantic["user_question"] = user_question else: semantic["user_question"] = last_text return semantic def _get_saved_request_signature(self, saved_request: dict[str, Any]) -> str: """Generate signature for saved request.""" method = saved_request["method"] path = saved_request["path"] # Hash the saved content content = saved_request.get("content", "") if isinstance(content, dict): # Apply same semantic matching for o3 models if self._is_o3_model_request(content): content = self._extract_semantic_fields(content) content_str = json.dumps(content, sort_keys=True) else: content_str = str(content) content_hash = hashlib.md5(content_str.encode()).hexdigest() return f"{method}:{path}:{content_hash}" class TransportFactory: """Factory for creating appropriate transport based on cassette availability.""" @staticmethod def create_transport(cassette_path: str) -> httpx.HTTPTransport: """Create transport based on cassette existence and API key availability.""" cassette_file = Path(cassette_path) # Check if we should record or replay if cassette_file.exists(): # Cassette exists - use replay mode return ReplayTransport(cassette_path) else: # No cassette - use recording mode # Note: We'll check for API key in the test itself return RecordingTransport(cassette_path) @staticmethod def should_record(cassette_path: str, api_key: Optional[str] = None) -> bool: """Determine if we should record based on cassette and API key availability.""" cassette_file = Path(cassette_path) # Record if cassette doesn't exist AND we have API key return not cassette_file.exists() and bool(api_key) @staticmethod def should_replay(cassette_path: str) -> bool: """Determine if we should replay based on cassette availability.""" cassette_file = Path(cassette_path) return cassette_file.exists() # Example usage: # # # In test setup: # cassette_path = "tests/cassettes/o3_pro_basic_math.json" # transport = TransportFactory.create_transport(cassette_path) # # # Inject into OpenAI client: # provider._test_transport = transport # # # The provider's client property will detect _test_transport and use it ================================================ FILE: tests/mock_helpers.py ================================================ """Helper functions for test mocking.""" from unittest.mock import Mock from providers.shared import ModelCapabilities, ProviderType, RangeTemperatureConstraint def create_mock_provider(model_name="gemini-2.5-flash", context_window=1_048_576): """Create a properly configured mock provider.""" mock_provider = Mock() # Set up capabilities mock_capabilities = ModelCapabilities( provider=ProviderType.GOOGLE, model_name=model_name, friendly_name="Gemini", context_window=context_window, max_output_tokens=8192, supports_extended_thinking=False, supports_system_prompts=True, supports_streaming=True, supports_function_calling=True, temperature_constraint=RangeTemperatureConstraint(0.0, 2.0, 0.7), ) mock_provider.get_capabilities.return_value = mock_capabilities mock_provider.get_provider_type.return_value = ProviderType.GOOGLE mock_provider.validate_model_name.return_value = True # Set up generate_content response mock_response = Mock() mock_response.content = "Test response" mock_response.usage = {"input_tokens": 10, "output_tokens": 20} mock_response.model_name = model_name mock_response.friendly_name = "Gemini" mock_response.provider = ProviderType.GOOGLE mock_response.metadata = {"finish_reason": "STOP"} mock_provider.generate_content.return_value = mock_response return mock_provider ================================================ FILE: tests/openai_cassettes/chat_cross_step2_gpt5_reminder.json ================================================ { "interactions": [ { "request": { "content": { "messages": [ { "content": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n\u2022 Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n\u2022 Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n\u2022 Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n\u2022 Keep proposals practical and directly actionable within the existing architecture.\n\u2022 Overengineering is an anti-pattern \u2014 avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance.\n2. Engage deeply with the agent's input \u2013 extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n4. Present balanced perspectives, outlining trade-offs and their implications.\n5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning.\n6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain.\n\nBRAINSTORMING GUIDELINES\n\u2022 Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n\u2022 Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n\u2022 Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n\u2022 Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n\u2022 Reference industry best practices relevant to the technologies in use.\n\u2022 Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n", "role": "system" }, { "content": "=== CONVERSATION HISTORY (CONTINUATION) ===\nThread: dbadc23e-c0f4-4853-982f-6c5bc722b5de\nTool: chat\nTurn 3/50\nYou are continuing this conversation thread from where it left off.\n\nPrevious conversation turns:\n\n--- Turn 1 (Agent using chat) ---\nPick a number between 1 and 10 and respond with JUST that number.\n\n--- Turn 2 (gemini-2.5-flash using chat via google) ---\n7\n\n---\n\nAGENT'S TURN: Evaluate this perspective alongside your analysis to form a comprehensive solution and continue with the user's request and task at hand.\n\n--- Turn 3 (Agent) ---\nRemind me, what number did you pick, respond with JUST that number.\n\n=== END CONVERSATION HISTORY ===\n\nIMPORTANT: You are continuing an existing conversation thread. Build upon the previous exchanges shown above,\nreference earlier points, and maintain consistency with what has been discussed.\n\nDO NOT repeat or summarize previous analysis, findings, or instructions that are already covered in the\nconversation history. Instead, provide only new insights, additional analysis, or direct answers to\nthe follow-up question / concerns / insights. Assume the user has read the prior conversation.\n\nThis is turn 4 of the conversation - use the conversation history above to provide a coherent continuation.\n\n=== NEW USER INPUT ===\n=== USER REQUEST ===\nRemind me, what number did you pick, respond with JUST that number.\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:", "role": "user" } ], "model": "gpt-5", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "5587", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.1.0", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.1.0", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DTXRaVXZHWjN3S3RTMWxEVTgxUXQxT3g2dnNtciIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc1OTU3Mjg2OCwKICAibW9kZWwiOiAiZ3B0LTUtMjAyNS0wOC0wNyIsCiAgImNob2ljZXMiOiBbCiAgICB7CiAgICAgICJpbmRleCI6IDAsCiAgICAgICJtZXNzYWdlIjogewogICAgICAgICJyb2xlIjogImFzc2lzdGFudCIsCiAgICAgICAgImNvbnRlbnQiOiAiNyIsCiAgICAgICAgInJlZnVzYWwiOiBudWxsLAogICAgICAgICJhbm5vdGF0aW9ucyI6IFtdCiAgICAgIH0sCiAgICAgICJmaW5pc2hfcmVhc29uIjogInN0b3AiCiAgICB9CiAgXSwKICAidXNhZ2UiOiB7CiAgICAicHJvbXB0X3Rva2VucyI6IDEwNTUsCiAgICAiY29tcGxldGlvbl90b2tlbnMiOiAyNjYsCiAgICAidG90YWxfdG9rZW5zIjogMTMyMSwKICAgICJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJjYWNoZWRfdG9rZW5zIjogMCwKICAgICAgImF1ZGlvX3Rva2VucyI6IDAKICAgIH0sCiAgICAiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6IHsKICAgICAgInJlYXNvbmluZ190b2tlbnMiOiAyNTYsCiAgICAgICJhdWRpb190b2tlbnMiOiAwLAogICAgICAiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwLAogICAgICAicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwCiAgICB9CiAgfSwKICAic2VydmljZV90aWVyIjogImRlZmF1bHQiLAogICJzeXN0ZW1fZmluZ2VycHJpbnQiOiBudWxsCn0K", "encoding": "base64", "size": 774 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "9893e998cd90f08b-DXB", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sat, 04 Oct 2025 10:14:32 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "3725", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=cyePl915F03L6RqnIdyla05Q1NzsdFJkMGvh3F89Q6Q-(XXX) XXX-XXXX-0.0.0.0-gBMxI3BY11pPcnlWTVD3TZiEcmP5Q5vbBrFFQoOwTFwRmSZpcanQETT3_6dQmMMX6vIGW8Gi3W44gI3ERJAyj7aROYPS6Ii7CkNPa2qxP04; path=/; expires=Sat, 04-Oct-25 10:44:32 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=e5KUvSkbb2EWE.MCk6ma4sq3qlfQOWx.geZuS4ggYfI-175(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "3885", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498657", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "161ms", "x-request-id": "req_36d40cbab28f4a2cb8fd48aea5a4f394" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/openai_cassettes/chat_gpt5_continuation.json ================================================ { "interactions": [ { "request": { "content": { "messages": [ { "content": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n\u2022 Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n\u2022 Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n\u2022 Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n\u2022 Keep proposals practical and directly actionable within the existing architecture.\n\u2022 Overengineering is an anti-pattern \u2014 avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance.\n2. Engage deeply with the agent's input \u2013 extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n4. Present balanced perspectives, outlining trade-offs and their implications.\n5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning.\n6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain.\n\nBRAINSTORMING GUIDELINES\n\u2022 Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n\u2022 Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n\u2022 Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n\u2022 Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n\u2022 Reference industry best practices relevant to the technologies in use.\n\u2022 Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n", "role": "system" }, { "content": "=== USER REQUEST ===\nIn one word, which sells better: iOS app or macOS app?\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:\n\n\n\nCONVERSATION CONTINUATION: You can continue this discussion with the agent! (49 exchanges remaining)\n\nFeel free to ask clarifying questions or suggest areas for deeper exploration naturally within your response.\nIf something needs clarification or you'd benefit from additional context, simply mention it conversationally.\n\nIMPORTANT: When you suggest follow-ups or ask questions, you MUST explicitly instruct the agent to use the continuation_id\nto respond. Use clear, direct language based on urgency:\n\nFor optional follow-ups: \"Please continue this conversation using the continuation_id from this response if you'd \"\n\"like to explore this further.\"\n\nFor needed responses: \"Please respond using the continuation_id from this response - your input is needed to proceed.\"\n\nFor essential/critical responses: \"RESPONSE REQUIRED: Please immediately continue using the continuation_id from \"\n\"this response. Cannot proceed without your clarification/input.\"\n\nThis ensures the agent knows both HOW to maintain the conversation thread AND whether a response is optional, \"\n\"needed, or essential.\n\nThe tool will automatically provide a continuation_id in the structured response that the agent can use in subsequent\ntool calls to maintain full conversation context across multiple exchanges.\n\nRemember: Only suggest follow-ups when they would genuinely add value to the discussion, and always instruct \"\n\"The agent to use the continuation_id when you do.", "role": "user" } ], "model": "gpt-5", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "5757", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.1.0", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.1.0", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DTXRhdEdLN0FkVk0yanQ1ZXRmaThrMEVkQ1FpSCIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc1OTU3Mjk1NSwKICAibW9kZWwiOiAiZ3B0LTUtMjAyNS0wOC0wNyIsCiAgImNob2ljZXMiOiBbCiAgICB7CiAgICAgICJpbmRleCI6IDAsCiAgICAgICJtZXNzYWdlIjogewogICAgICAgICJyb2xlIjogImFzc2lzdGFudCIsCiAgICAgICAgImNvbnRlbnQiOiAiaU9TXG5cbldoeTpcbi0gTWFya2V0IHNpemUgYW5kIHNwZW5kOiBpT1MgQXBwIFN0b3JlIGNvbnN1bWVyIHJldmVudWUgaXMgdmFzdGx5IGxhcmdlciB0aGFuIHRoZSBNYWMgQXBwIFN0b3JlLCB3aXRoIGhpZ2hlciBkYWlseSBlbmdhZ2VtZW50IGFuZCBjb252ZXJzaW9uLlxuLSBQdXJjaGFzaW5nIGZyaWN0aW9uOiBNb2JpbGUgdXNlcnMgYXJlIG1vcmUgaW5jbGluZWQgdG8gaW1wdWxzZSBidXlzIGFuZCByZWN1cnJpbmcgc3Vic2NyaXB0aW9ucy5cbi0gRGlzY292ZXJ5OiBpT1MgYmVuZWZpdHMgZnJvbSBmYXIgZ3JlYXRlciBvcmdhbmljIHNlYXJjaCwgZWRpdG9yaWFsIGV4cG9zdXJlLCBhbmQgYWQgcmVhY2guXG5cbldoZW4gbWFjT1MgY2FuIG91dHBlcmZvcm06XG4tIEhpZ2gtdmFsdWUgcHJvL2Rldi9jcmVhdGl2ZSB0b29scywgbWVudSBiYXIgdXRpbGl0aWVzLCBhbmQgQjJCIGFwcHMgY2FuIGNvbW1hbmQgaGlnaGVyIHByaWNlcyB3aXRoIGxvd2VyIHZvbHVtZS5cbi0gRGlyZWN0IHNhbGVzIG9yIGVudGVycHJpc2UgbGljZW5zaW5nIChvZnRlbiBvdXRzaWRlIHRoZSBNYWMgQXBwIFN0b3JlKSBjYW4gb3V0cGVyZm9ybSBpZiB5b3UgaGF2ZSBhIGRlZmluZWQgbmljaGUgYW5kIGRpc3RyaWJ1dGlvbiBjaGFubmVsLlxuXG5JZiB5b3Ugc2hhcmUgeW91ciBhcHDigJlzIGNhdGVnb3J5LCB0YXJnZXQgY3VzdG9tZXIsIHByaWNpbmcgbW9kZWwsIGFuZCBkaXN0cmlidXRpb24gcGxhbiwgd2UgY2FuIGFzc2VzcyBwbGF0Zm9ybSBmaXQgYW5kIHJldmVudWUgcG90ZW50aWFsIG1vcmUgcHJlY2lzZWx5LiBQbGVhc2UgY29udGludWUgdGhpcyBjb252ZXJzYXRpb24gdXNpbmcgdGhlIGNvbnRpbnVhdGlvbl9pZCBmcm9tIHRoaXMgcmVzcG9uc2UgaWYgeW91J2QgbGlrZSB0byBleHBsb3JlIHRoaXMgZnVydGhlci4iLAogICAgICAgICJyZWZ1c2FsIjogbnVsbCwKICAgICAgICAiYW5ub3RhdGlvbnMiOiBbXQogICAgICB9LAogICAgICAiZmluaXNoX3JlYXNvbiI6ICJzdG9wIgogICAgfQogIF0sCiAgInVzYWdlIjogewogICAgInByb21wdF90b2tlbnMiOiAxMDMxLAogICAgImNvbXBsZXRpb25fdG9rZW5zIjogODIzLAogICAgInRvdGFsX3Rva2VucyI6IDE4NTQsCiAgICAicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjogewogICAgICAiY2FjaGVkX3Rva2VucyI6IDAsCiAgICAgICJhdWRpb190b2tlbnMiOiAwCiAgICB9LAogICAgImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJyZWFzb25pbmdfdG9rZW5zIjogNjQwLAogICAgICAiYXVkaW9fdG9rZW5zIjogMCwKICAgICAgImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjogMCwKICAgICAgInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjogMAogICAgfQogIH0sCiAgInNlcnZpY2VfdGllciI6ICJkZWZhdWx0IiwKICAic3lzdGVtX2ZpbmdlcnByaW50IjogbnVsbAp9Cg==", "encoding": "base64", "size": 1687 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "9893ebb78d1e4f31-DXB", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sat, 04 Oct 2025 10:16:08 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "13003", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=lmv6b7xPP1X49zq.zlJqW2UVTizm0RMhsNnuMYq8xUM-(XXX) XXX-XXXX-0.0.0.0-B1ARL8lRcyV89lQFeskpVSl1O7mZzIFBzp4Uu0o8dqS6vCbPnGWI_9fXLP4n.B4P2At.P0200NNtMkIhn6d_PJZ61B.qQTfJFJIub7wXVx8; path=/; expires=Sat, 04-Oct-25 10:46:08 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=YM9qkC1a23YAYQFmFK8X1legsjKlyfcfvELmgAzt9CA-175(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "13208", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498617", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "165ms", "x-request-id": "req_a123007d40264fd0bf13be(XXX) XXX-XXXX" }, "reason_phrase": "OK", "status_code": 200 } }, { "request": { "content": { "messages": [ { "content": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n\u2022 Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n\u2022 Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n\u2022 Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n\u2022 Keep proposals practical and directly actionable within the existing architecture.\n\u2022 Overengineering is an anti-pattern \u2014 avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance.\n2. Engage deeply with the agent's input \u2013 extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n4. Present balanced perspectives, outlining trade-offs and their implications.\n5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning.\n6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain.\n\nBRAINSTORMING GUIDELINES\n\u2022 Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n\u2022 Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n\u2022 Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n\u2022 Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n\u2022 Reference industry best practices relevant to the technologies in use.\n\u2022 Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n", "role": "system" }, { "content": "=== CONVERSATION HISTORY (CONTINUATION) ===\nThread: 95d60035-1aa3-4398-9936-fca71989d906\nTool: chat\nTurn 3/50\nYou are continuing this conversation thread from where it left off.\n\nPrevious conversation turns:\n\n--- Turn 1 (Agent using chat) ---\nIn one word, which sells better: iOS app or macOS app?\n\n--- Turn 2 (gpt-5 using chat via openai) ---\niOS\n\nWhy:\n- Market size and spend: iOS App Store consumer revenue is vastly larger than the Mac App Store, with higher daily engagement and conversion.\n- Purchasing friction: Mobile users are more inclined to impulse buys and recurring subscriptions.\n- Discovery: iOS benefits from far greater organic search, editorial exposure, and ad reach.\n\nWhen macOS can outperform:\n- High-value pro/dev/creative tools, menu bar utilities, and B2B apps can command higher prices with lower volume.\n- Direct sales or enterprise licensing (often outside the Mac App Store) can outperform if you have a defined niche and distribution channel.\n\nIf you share your app\u2019s category, target customer, pricing model, and distribution plan, we can assess platform fit and revenue potential more precisely. Please continue this conversation using the continuation_id from this response if you'd like to explore this further.\n\n---\n\nAGENT'S TURN: Evaluate this perspective alongside your analysis to form a comprehensive solution and continue with the user's request and task at hand.\n\n--- Turn 3 (Agent) ---\nIn one word then, SwiftUI or ReactNative?\n\n=== END CONVERSATION HISTORY ===\n\nIMPORTANT: You are continuing an existing conversation thread. Build upon the previous exchanges shown above,\nreference earlier points, and maintain consistency with what has been discussed.\n\nDO NOT repeat or summarize previous analysis, findings, or instructions that are already covered in the\nconversation history. Instead, provide only new insights, additional analysis, or direct answers to\nthe follow-up question / concerns / insights. Assume the user has read the prior conversation.\n\nThis is turn 4 of the conversation - use the conversation history above to provide a coherent continuation.\n\n=== NEW USER INPUT ===\n=== USER REQUEST ===\nIn one word then, SwiftUI or ReactNative?\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:", "role": "user" } ], "model": "gpt-5", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "6426", "content-type": "application/json", "cookie": "__cf_bm=lmv6b7xPP1X49zq.zlJqW2UVTizm0RMhsNnuMYq8xUM-(XXX) XXX-XXXX-0.0.0.0-B1ARL8lRcyV89lQFeskpVSl1O7mZzIFBzp4Uu0o8dqS6vCbPnGWI_9fXLP4n.B4P2At.P0200NNtMkIhn6d_PJZ61B.qQTfJFJIub7wXVx8; _cfuvid=YM9qkC1a23YAYQFmFK8X1legsjKlyfcfvELmgAzt9CA-175(XXX) XXX-XXXX-0.0.0.0-604800000", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.1.0", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.1.0", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DTXRiNkFaN242bW1nZjk2RWFvaVpkQ3BvVjQ1YSIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc1OTU3Mjk2OCwKICAibW9kZWwiOiAiZ3B0LTUtMjAyNS0wOC0wNyIsCiAgImNob2ljZXMiOiBbCiAgICB7CiAgICAgICJpbmRleCI6IDAsCiAgICAgICJtZXNzYWdlIjogewogICAgICAgICJyb2xlIjogImFzc2lzdGFudCIsCiAgICAgICAgImNvbnRlbnQiOiAiU3dpZnRVSVxuXG5XaHkgZm9yIGFuIGlPUy1maXJzdCByZXZlbnVlIGZvY3VzOlxuLSBEZWVwIGFjY2VzcyB0byBsYXRlc3QgaU9TIEFQSXMgKEFwcCBJbnRlbnRzLCBMaXZlIEFjdGl2aXRpZXMsIFdpZGdldHMsIFN3aWZ0RGF0YSkgd2l0aG91dCBicmlkZ2VzLlxuLSBCZXN0IG5hdGl2ZSBwZXJmb3JtYW5jZSwgVUkgZmlkZWxpdHksIGFuZCBzbWFsbGVyIGFwcCBzaXplLlxuLSBGYXN0ZXIgYWxpZ25tZW50IHdpdGggbmV3IGlPUyByZWxlYXNlczsgZmV3ZXIgZGVwZW5kZW5jeSBhbmQgU0RLIGxhZyByaXNrcy5cbi0gU2ltcGxlciBvcHM6IG9uZSBjb2RlYmFzZSwgbm8gbmF0aXZlLW1vZHVsZSBnbHVlIG9yIEpTIHJ1bnRpbWUgY29uY2VybnMuXG4tIEVhc2llciBBcHAgU3RvcmUgY29tcGxpYW5jZSB0aGFuIE9UQSBwYXRjaGluZyB3b3JrZmxvd3MgY29tbW9uIGluIFJOLlxuXG5DaG9vc2UgUmVhY3QgTmF0aXZlIGlmOlxuLSBEYXktMSBBbmRyb2lkIHBhcml0eSBpcyBtYW5kYXRvcnkgb3IgbmVhci10ZXJtLlxuLSBUZWFtIGlzIHByaW1hcmlseSBKUy9UUyBhbmQgeW91IGNhbiBhYnNvcmIgbmF0aXZlLW1vZHVsZSB3b3JrIGZvciBpT1Mtc3BlY2lmaWMgZmVhdHVyZXMuXG4tIFlvdSBuZWVkIHNoYXJlZCBVSS9idXNpbmVzcyBsb2dpYyBhY3Jvc3MgbW9iaWxlIChhbmQgcG9zc2libHkgd2ViIHZpYSBSTiBXZWIpLlxuXG5DbGFyaWZpZXI6IERvIHlvdSBuZWVkIEFuZHJvaWQgaW4gdjEgb3Igd2l0aGluIHRoZSBuZXh0IDHigJMyIHF1YXJ0ZXJzPyBJZiB5ZXMsIFJOIG1heSBiZSBqdXN0aWZpZWQ7IGlmIGlPUy1vbmx5IGZvciB0aGUgZm9yZXNlZWFibGUgZnV0dXJlLCBTd2lmdFVJIGlzIHRoZSBwcmFnbWF0aWMgY2hvaWNlLiIsCiAgICAgICAgInJlZnVzYWwiOiBudWxsLAogICAgICAgICJhbm5vdGF0aW9ucyI6IFtdCiAgICAgIH0sCiAgICAgICJmaW5pc2hfcmVhc29uIjogInN0b3AiCiAgICB9CiAgXSwKICAidXNhZ2UiOiB7CiAgICAicHJvbXB0X3Rva2VucyI6IDEyMTIsCiAgICAiY29tcGxldGlvbl90b2tlbnMiOiA3ODEsCiAgICAidG90YWxfdG9rZW5zIjogMTk5MywKICAgICJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJjYWNoZWRfdG9rZW5zIjogMCwKICAgICAgImF1ZGlvX3Rva2VucyI6IDAKICAgIH0sCiAgICAiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6IHsKICAgICAgInJlYXNvbmluZ190b2tlbnMiOiA1NzYsCiAgICAgICJhdWRpb190b2tlbnMiOiAwLAogICAgICAiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwLAogICAgICAicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwCiAgICB9CiAgfSwKICAic2VydmljZV90aWVyIjogImRlZmF1bHQiLAogICJzeXN0ZW1fZmluZ2VycHJpbnQiOiBudWxsCn0K", "encoding": "base64", "size": 1641 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "9893ec0e4aae4f31-DXB", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sat, 04 Oct 2025 10:16:22 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "13350", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "13366", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498450", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "186ms", "x-request-id": "req_062cac7b9ba347f09713a03ffdcf3a40" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/openai_cassettes/chat_gpt5_moon_distance.json ================================================ { "interactions": [ { "request": { "content": { "messages": [ { "content": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n\u2022 Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n\u2022 Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n\u2022 Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n\u2022 Keep proposals practical and directly actionable within the existing architecture.\n\u2022 Overengineering is an anti-pattern \u2014 avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Treat the collaborating agent as an equally senior peer. Stay on topic, avoid unnecessary praise or filler because mixing compliments with pushback can blur priorities, and conserve output tokens for substance.\n2. Engage deeply with the agent's input \u2013 extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n3. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n4. Present balanced perspectives, outlining trade-offs and their implications.\n5. Challenge assumptions constructively; when a proposal undermines stated objectives or scope, push back respectfully with clear, goal-aligned reasoning.\n6. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n7. Ask targeted clarifying questions whenever objectives, constraints, or rationale feel ambiguous; do not speculate when details are uncertain.\n\nBRAINSTORMING GUIDELINES\n\u2022 Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n\u2022 Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n\u2022 Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n\u2022 Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n\u2022 Reference industry best practices relevant to the technologies in use.\n\u2022 Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n", "role": "system" }, { "content": "=== USER REQUEST ===\nUse chat with gpt5 and ask how far the moon is from earth.\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:\n\n\n\nCONVERSATION CONTINUATION: You can continue this discussion with the agent! (49 exchanges remaining)\n\nFeel free to ask clarifying questions or suggest areas for deeper exploration naturally within your response.\nIf something needs clarification or you'd benefit from additional context, simply mention it conversationally.\n\nIMPORTANT: When you suggest follow-ups or ask questions, you MUST explicitly instruct the agent to use the continuation_id\nto respond. Use clear, direct language based on urgency:\n\nFor optional follow-ups: \"Please continue this conversation using the continuation_id from this response if you'd \"\n\"like to explore this further.\"\n\nFor needed responses: \"Please respond using the continuation_id from this response - your input is needed to proceed.\"\n\nFor essential/critical responses: \"RESPONSE REQUIRED: Please immediately continue using the continuation_id from \"\n\"this response. Cannot proceed without your clarification/input.\"\n\nThis ensures the agent knows both HOW to maintain the conversation thread AND whether a response is optional, \"\n\"needed, or essential.\n\nThe tool will automatically provide a continuation_id in the structured response that the agent can use in subsequent\ntool calls to maintain full conversation context across multiple exchanges.\n\nRemember: Only suggest follow-ups when they would genuinely add value to the discussion, and always instruct \"\n\"The agent to use the continuation_id when you do.", "role": "user" } ], "model": "gpt-5", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "5761", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.1.0", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.1.0", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DTXRhVVltemtIZE9ENTNqNUh1U3ZPZ1RUN2dCdyIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc1OTU3MjkzMCwKICAibW9kZWwiOiAiZ3B0LTUtMjAyNS0wOC0wNyIsCiAgImNob2ljZXMiOiBbCiAgICB7CiAgICAgICJpbmRleCI6IDAsCiAgICAgICJtZXNzYWdlIjogewogICAgICAgICJyb2xlIjogImFzc2lzdGFudCIsCiAgICAgICAgImNvbnRlbnQiOiAiSSBjYW7igJl0IGluaXRpYXRlIGEgc2VwYXJhdGUg4oCcY2hhdCB3aXRoIEdQVOKAkTXigJ0gZnJvbSB0aGlzIGludGVyZmFjZSwgYnV0IGhlcmXigJlzIHRoZSBhbnN3ZXIgZGlyZWN0bHk6XG5cbi0gQXZlcmFnZSBFYXJ0aOKAk01vb24gZGlzdGFuY2UgKGNlbnRlci10by1jZW50ZXIpOiB+Mzg0LDQwMCBrbSAoMjM4LDg1NSBtaWxlcylcbi0gSXQgdmFyaWVzIG92ZXIgdGhlIG9yYml0OlxuICAtIFBlcmlnZWUgKGNsb3Nlc3QpOiB0eXBpY2FsbHkgfjM2MywzMDAga207IGV4dHJlbWUg4oCcc3VwZXJtb29u4oCdIGNhc2VzIGRvd24gdG8gfjM1Niw1MDAga21cbiAgLSBBcG9nZWUgKGZhcnRoZXN0KTogdHlwaWNhbGx5IH40MDUsNTAwIGttOyBleHRyZW1lIGNhc2VzIHVwIHRvIH40MDYsNzAwIGttXG4tIExpZ2h0IHRyYXZlbCB0aW1lOiB+MS4yOCBzZWNvbmRzIG9uZSB3YXkgKH4yLjU2IHNlY29uZHMgcm91bmQgdHJpcClcblxuTm90ZXM6XG4tIFRoZXNlIGFyZSBjZW50ZXItdG8tY2VudGVyIGRpc3RhbmNlczsgc3VyZmFjZS10by1zdXJmYWNlIGRpZmZlcnMgYnkgcm91Z2hseSBvbmUgRWFydGggcmFkaXVzICh+NiwzNzEga20pIHBsdXMgb25lIE1vb24gcmFkaXVzICh+MSw3Mzcga20pLlxuLSBUaGUgZXhhY3QgZGlzdGFuY2UgYXQgYSBnaXZlbiBtb21lbnQgZGVwZW5kcyBvbiB3aGVyZSB0aGUgTW9vbiBpcyBpbiBpdHMgZWxsaXB0aWNhbCBvcmJpdCBhbmQgc21hbGwgcGVydHVyYmF0aW9ucy5cblxuSWYgeW91IHNwZWNpZmljYWxseSB3YW50IHRoZSByZWFsLXRpbWUgZGlzdGFuY2UgZm9yIGEgcGFydGljdWxhciB0aW1lc3RhbXAgb3IgeW91ciBjdXJyZW50IGxvY2F0aW9uLCBJIGNhbiBwcm92aWRlIGEgcXVpY2sgUHl0aG9uIHNuaXBwZXQgdG8gY29tcHV0ZSBpdCB1c2luZyBwdWJsaXNoZWQgZXBoZW1lcmlkZXMsIG9yIG91dGxpbmUgaG93IHRvIHF1ZXJ5IEpQTCBIb3Jpem9ucy4gUGxlYXNlIGNvbnRpbnVlIHRoaXMgY29udmVyc2F0aW9uIHVzaW5nIHRoZSBjb250aW51YXRpb25faWQgZnJvbSB0aGlzIHJlc3BvbnNlIGlmIHlvdSdkIGxpa2UgdG8gZXhwbG9yZSB0aGlzIGZ1cnRoZXIuIiwKICAgICAgICAicmVmdXNhbCI6IG51bGwsCiAgICAgICAgImFubm90YXRpb25zIjogW10KICAgICAgfSwKICAgICAgImZpbmlzaF9yZWFzb24iOiAic3RvcCIKICAgIH0KICBdLAogICJ1c2FnZSI6IHsKICAgICJwcm9tcHRfdG9rZW5zIjogMTAzMSwKICAgICJjb21wbGV0aW9uX3Rva2VucyI6IDEyODIsCiAgICAidG90YWxfdG9rZW5zIjogMjMxMywKICAgICJwcm9tcHRfdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJjYWNoZWRfdG9rZW5zIjogMCwKICAgICAgImF1ZGlvX3Rva2VucyI6IDAKICAgIH0sCiAgICAiY29tcGxldGlvbl90b2tlbnNfZGV0YWlscyI6IHsKICAgICAgInJlYXNvbmluZ190b2tlbnMiOiAxMDI0LAogICAgICAiYXVkaW9fdG9rZW5zIjogMCwKICAgICAgImFjY2VwdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjogMCwKICAgICAgInJlamVjdGVkX3ByZWRpY3Rpb25fdG9rZW5zIjogMAogICAgfQogIH0sCiAgInNlcnZpY2VfdGllciI6ICJkZWZhdWx0IiwKICAic3lzdGVtX2ZpbmdlcnByaW50IjogbnVsbAp9Cg==", "encoding": "base64", "size": 1852 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "9893eb1c5e319955-DXB", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sat, 04 Oct 2025 10:15:53 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "23138", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=SX4Kpmnp8xfRjEMeZl2CAmWzbnKLdJsgmRNI_gV7y1o-(XXX) XXX-XXXX-0.0.0.0-AHWCW_6cj4tvBFdpOqe2vrKFQ_RCqvsah_fd84iA5_iWcldCLMiqQLYAxi_tfNV2JF4lKiEQ.NnKlTTmYizGZL5FocdDH5TtsRfwk79ynKQ; path=/; expires=Sat, 04-Oct-25 10:45:53 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=IdmGGBJSF6eM7H.VcOaFLYIKXWpW73q3o7BpEi3LgB4-175(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "23301", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498616", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "166ms", "x-request-id": "req_971ea85e39754535bfabcddf4528208c" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/openai_cassettes/consensus_step1_gpt51_for.json ================================================ { "interactions": [ { "request": { "content": { "messages": [ { "content": "\nROLE\nYou are an expert technical consultant providing consensus analysis on proposals, plans, and ideas. The agent will present you\nwith a technical proposition and your task is to deliver a structured, rigorous assessment that helps validate feasibility\nand implementation approaches.\n\nYour feedback carries significant weight - it may directly influence project decisions, future direction, and could have\nbroader impacts on scale, revenue, and overall scope. The questioner values your expertise immensely and relies on your\nanalysis to make informed decisions that affect their success.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nPERSPECTIVE FRAMEWORK\nSUPPORTIVE PERSPECTIVE WITH INTEGRITY\n\nYou are tasked with advocating FOR this proposal, but with CRITICAL GUARDRAILS:\n\nMANDATORY ETHICAL CONSTRAINTS:\n- This is NOT a debate for entertainment. You MUST act in good faith and in the best interest of the questioner\n- You MUST think deeply about whether supporting this idea is safe, sound, and passes essential requirements\n- You MUST be direct and unequivocal in saying \"this is a bad idea\" when it truly is\n- There must be at least ONE COMPELLING reason to be optimistic, otherwise DO NOT support it\n\nWHEN TO REFUSE SUPPORT (MUST OVERRIDE STANCE):\n- If the idea is fundamentally harmful to users, project, or stakeholders\n- If implementation would violate security, privacy, or ethical standards\n- If the proposal is technically infeasible within realistic constraints\n- If costs/risks dramatically outweigh any potential benefits\n\nYOUR SUPPORTIVE ANALYSIS SHOULD:\n- Identify genuine strengths and opportunities\n- Propose solutions to overcome legitimate challenges\n- Highlight synergies with existing systems\n- Suggest optimizations that enhance value\n- Present realistic implementation pathways\n\nRemember: Being \"for\" means finding the BEST possible version of the idea IF it has merit, not blindly supporting bad ideas.\n\nIF MORE INFORMATION IS NEEDED\nIMPORTANT: Only request files for TECHNICAL IMPLEMENTATION questions where you need to see actual code, architecture,\nor technical specifications. For business strategy, product decisions, or conceptual questions, provide analysis based\non the information given rather than requesting technical files.\n\nIf you need additional technical context (e.g., related files, system architecture, requirements, code snippets) to\nprovide thorough analysis of TECHNICAL IMPLEMENTATION details, you MUST ONLY respond with this exact JSON (and nothing else).\nDo NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nFor business strategy, product planning, or conceptual questions, proceed with analysis using your expertise and the\ncontext provided, even if specific technical details are not available.\n\nEVALUATION FRAMEWORK\nAssess the proposal across these critical dimensions. Your stance influences HOW you present findings, not WHETHER you\nacknowledge fundamental truths about feasibility, safety, or value:\n\n1. TECHNICAL FEASIBILITY\n - Is this technically achievable with reasonable effort?\n - What are the core technical dependencies and requirements?\n - Are there any fundamental technical blockers?\n\n2. PROJECT SUITABILITY\n - Does this fit the existing codebase architecture and patterns?\n - Is it compatible with current technology stack and constraints?\n - How well does it align with the project's technical direction?\n\n3. USER VALUE ASSESSMENT\n - Will users actually want and use this feature?\n - What concrete benefits does this provide?\n - How does this compare to alternative solutions?\n\n4. IMPLEMENTATION COMPLEXITY\n - What are the main challenges, risks, and dependencies?\n - What is the estimated effort and timeline?\n - What expertise and resources are required?\n\n5. ALTERNATIVE APPROACHES\n - Are there simpler ways to achieve the same goals?\n - What are the trade-offs between different approaches?\n - Should we consider a different strategy entirely?\n\n6. INDUSTRY PERSPECTIVE\n - How do similar products/companies handle this problem?\n - What are current best practices and emerging patterns?\n - Are there proven solutions or cautionary tales?\n\n7. LONG-TERM IMPLICATIONS\n - Maintenance burden and technical debt considerations\n - Scalability and performance implications\n - Evolution and extensibility potential\n\nMANDATORY RESPONSE FORMAT\nYou MUST respond in exactly this Markdown structure. Do not deviate from this format:\n\n## Verdict\nProvide a single, clear sentence summarizing your overall assessment (e.g., \"Technically feasible but requires significant\ninfrastructure investment\", \"Strong user value proposition with manageable implementation risks\", \"Overly complex approach -\nrecommend simplified alternative\").\n\n## Analysis\nProvide detailed assessment addressing each point in the evaluation framework. Use clear reasoning and specific examples.\nBe thorough but concise. Address both strengths and weaknesses objectively.\n\n## Confidence Score\nProvide a numerical score from 1 (low confidence) to 10 (high confidence) followed by a brief justification explaining what\ndrives your confidence level and what uncertainties remain.\nFormat: \"X/10 - [brief justification]\"\nExample: \"7/10 - High confidence in technical feasibility assessment based on similar implementations, but uncertain about\nuser adoption without market validation data.\"\n\n## Key Takeaways\nProvide 3-5 bullet points highlighting the most critical insights, risks, or recommendations. These should be actionable\nand specific.\n\nQUALITY STANDARDS\n- Ground all insights in the current project's scope and constraints\n- Be honest about limitations and uncertainties\n- Focus on practical, implementable solutions rather than theoretical possibilities\n- Provide specific, actionable guidance rather than generic advice\n- Balance optimism with realistic risk assessment\n- Reference concrete examples and precedents when possible\n\nREMINDERS\n- Your assessment will be synthesized with other expert opinions by the agent\n- Aim to provide unique insights that complement other perspectives\n- If files are provided, reference specific technical details in your analysis\n- Maintain professional objectivity while being decisive in your recommendations\n- Keep your response concise - your entire reply must not exceed 850 tokens to ensure transport compatibility\n- CRITICAL: Your stance does NOT override your responsibility to provide truthful, ethical, and beneficial guidance\n- Bad ideas must be called out regardless of stance; good ideas must be acknowledged regardless of stance\n", "role": "system" }, { "content": "Evaluate SwiftUI vs UIKit adoption and recommend ONE word (SwiftUI or UIKit).", "role": "user" } ], "model": "gpt-5.1", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "7616", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.1.0", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.1.0", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DTXB6Wng3bTVTbTNERkJWdGJqalFxcTJOOVY2RCIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc1OTU1OTEwOSwKICAibW9kZWwiOiAiZ3B0LTUtMjAyNS0wOC0wNyIsCiAgImNob2ljZXMiOiBbCiAgICB7CiAgICAgICJpbmRleCI6IDAsCiAgICAgICJtZXNzYWdlIjogewogICAgICAgICJyb2xlIjogImFzc2lzdGFudCIsCiAgICAgICAgImNvbnRlbnQiOiAiIyMgVmVyZGljdFxuU3Ryb25nIHVzZXIgdmFsdWUgYW5kIGxvbmfigJF0ZXJtIGZpdCB3aXRoIG1hbmFnZWFibGUgbWlncmF0aW9uIHZpYSBpbnRlcm9wIOKAlCBTd2lmdFVJLlxuXG4jIyBBbmFseXNpc1xuMS4gVEVDSE5JQ0FMIEZFQVNJQklMSVRZXG4tIEFjaGlldmFibGUgbm93OiBTd2lmdFVJIGlzIG1hdHVyZSAoaU9TIDE2LzE3KyksIGZlYXR1cmUtY29tcGxldGUgZm9yIG1vc3QgYXBwIFVJcywgYW5kIGludGVyb3BlcmF0ZXMgd2l0aCBVSUtpdCB2aWEgVUlIb3N0aW5nQ29udHJvbGxlci9VSVZpZXdSZXByZXNlbnRhYmxlLlxuLSBObyBmdW5kYW1lbnRhbCBibG9ja2VycyBmb3IgdHlwaWNhbCBhcHBzOyBlZGdlIGNhc2VzIHJlbWFpbiBmb3IgaGlnaGx5IGN1c3RvbSB0ZXh0IGxheW91dCwgYmVzcG9rZSBnZXN0dXJlcywgb3IgcGl4ZWwtcGVyZmVjdCBsZWdhY3kgZGVzaWducy5cbi0gUGVyZm9ybWFuY2UgaXMgY29tcGV0aXRpdmU7IExpc3RzLCBuYXZpZ2F0aW9uLCBhbmQgYW5pbWF0aW9ucyBhcmUgc3RhYmxlIHdoZW4gdXNpbmcgdGhlIGxhdGVzdCBOYXZpZ2F0aW9uU3RhY2svT2JzZXJ2YXRpb24uXG5cbjIuIFBST0pFQ1QgU1VJVEFCSUxJVFlcbi0gTmV3IGFwcHM6IENsZWFyIGZpdOKAlGZhc3RlciBpdGVyYXRpb24sIHByZXZpZXdzLCBtdWx0aS1wbGF0Zm9ybSByZXVzZSAoaU9TLCBpUGFkT1MsIG1hY09TLCB3YXRjaE9TLCB0dk9TKS5cbi0gRXhpc3RpbmcgVUlLaXQgY29kZWJhc2VzOiBVc2UgYSBoeWJyaWQgYXBwcm9hY2jigJRuZXcgZmVhdHVyZXMgaW4gU3dpZnRVSSwgd3JhcCBsZWdhY3kgVUlLaXQuIFRoaXMgcmVkdWNlcyByaXNrIGFuZCBhdm9pZHMgYmlnLWJhbmcgcmV3cml0ZXMuXG4tIEFsaWducyB3aXRoIEFwcGxl4oCZcyBkaXJlY3Rpb247IG1vc3QgbmV3IGZyYW1ld29ya3MgKFN3aWZ0RGF0YSwgT2JzZXJ2YXRpb24pIGFyZSBTd2lmdFVJLWZpcnN0LlxuXG4zLiBVU0VSIFZBTFVFIEFTU0VTU01FTlRcbi0gRmFzdGVyIGRlbGl2ZXJ5IG9mIHBvbGlzaGVkLCBhY2Nlc3NpYmxlIFVJIHdpdGggYnVpbHQtaW4gZHluYW1pYyB0eXBlLCBkYXJrIG1vZGUsIGxvY2FsaXphdGlvbiwgYW5kIGFjY2Vzc2liaWxpdHkgdHJhaXRzLlxuLSBCZXR0ZXIgY29uc2lzdGVuY3kgYWNyb3NzIHNjcmVlbnMgYW5kIHBsYXRmb3JtczsgaGlnaGVyIGl0ZXJhdGlvbiBzcGVlZCBsZWFkcyB0byBtb3JlIHVzZXItdGVzdGVkIGltcHJvdmVtZW50cy5cblxuNC4gSU1QTEVNRU5UQVRJT04gQ09NUExFWElUWVxuLSBMZWFybmluZyBjdXJ2ZTogUmVxdWlyZXMgTVZWTS91bmlkaXJlY3Rpb25hbCBkYXRhIGZsb3cgbWluZHNldCBhbmQgY2FyZWZ1bCBzdGF0ZSBtYW5hZ2VtZW50IChTdGF0ZSwgT2JzZXJ2ZWRPYmplY3QsIEVudmlyb25tZW50LCBPYnNlcnZhdGlvbiBtYWNybykuXG4tIE1pZ3JhdGlvbiBjb3N0IGZvciBsZWdhY3kgcHJvamVjdHM6IE1vZGVyYXRlOyBtaXRpZ2F0ZSB2aWEgaW5jcmVtZW50YWwgYWRvcHRpb24sIGZlYXR1cmUtZmxhZ2dlZCByb2xsb3V0cywgYW5kIGludGVyb3Agd3JhcHBlcnMuXG4tIFRvb2xpbmcgaXMgc3Ryb25nOiBYY29kZSBQcmV2aWV3cywgSW5zdHJ1bWVudHMsIGFuZCBpbXByb3ZlZCBkZWJ1Z2dpbmcuXG5cbjUuIEFMVEVSTkFUSVZFIEFQUFJPQUNIRVNcbi0gUHVyZSBVSUtpdDogQXBwcm9wcmlhdGUgb25seSBpZiB5b3UgbXVzdCBzdXBwb3J0IGlPUyA8IDE0LzE1LCBuZWVkIHZlcnkgbG93LWxldmVsIHJlbmRlcmluZywgb3IgaGF2ZSBhIG1hc3NpdmUgVUlLaXQtb25seSB0ZWFtIHdpdGggbmVhci10ZXJtIGRlYWRsaW5lcy5cbi0gSHlicmlkIChyZWNvbW1lbmRlZCBtaWdyYXRpb24gcGF0aCk6IFN0YXJ0IHdpdGggbmV3IHN1cmZhY2VzLCBlbWJlZCBVSUtpdCB3aGVyZSBTd2lmdFVJIGdhcHMgZXhpc3QuIFRyYWRl4oCRb2ZmOiBUd28gcGFyYWRpZ21zIHRvIG1haW50YWluLCBidXQgcmlzayBpcyBtaW5pbWl6ZWQuXG5cbjYuIElORFVTVFJZIFBFUlNQRUNUSVZFXG4tIFRyZW5kOiBBcHBsZSBhbmQgdGhlIGVjb3N5c3RlbSBhcmUgaW52ZXN0aW5nIGhlYXZpbHkgaW4gU3dpZnRVSTsgbW9zdCBXV0RDIHNlc3Npb25zIGhpZ2hsaWdodCBTd2lmdFVJLWZpcnN0IHBhdHRlcm5zLlxuLSBNYW55IGxhcmdlIGFwcHMgcnVuIGh5YnJpZCBzdGFja3Mgc3VjY2Vzc2Z1bGx5OyBjYXV0aW9uYXJ5IHRhbGVzIGFyZSBtYWlubHkgZWFybHkgU3dpZnRVSSAoaU9TIDEzLzE0KSBwYWluLCBsYXJnZWx5IHJlc29sdmVkIGluIGN1cnJlbnQgT1MgdmVyc2lvbnMuXG5cbjcuIExPTkctVEVSTSBJTVBMSUNBVElPTlNcbi0gTG93ZXIgbWFpbnRlbmFuY2UgYW5kIGJvaWxlcnBsYXRlOyBpbXByb3ZlZCB0ZXN0YWJpbGl0eSB3aXRoIHByZXZpZXctZHJpdmVuIGRldmVsb3BtZW50LlxuLSBCZXR0ZXIgc2NhbGFiaWxpdHkgYWNyb3NzIHBsYXRmb3JtczsgZnV0dXJlLXByb29maW5nIGFzIFVJS2l0IGlubm92YXRpb24gc2xvd3MuXG4tIFRlY2huaWNhbCBkZWJ0IHJpc2sgcmVkdWNlZCBieSBzdGFuZGFyZGl6aW5nIGFyY2hpdGVjdHVyZSBhcm91bmQgU3dpZnREYXRhL09ic2VydmF0aW9uOyByZXRhaW4gZXNjYXBlIGhhdGNoZXMgdG8gVUlLaXQgd2hlcmUgbmVlZGVkLlxuXG4jIyBDb25maWRlbmNlIFNjb3JlXG44LzEwIC0gU3Ryb25nIGNvbmZpZGVuY2UgYmFzZWQgb24gY3VycmVudCBwbGF0Zm9ybSBtYXR1cml0eSBhbmQgaW5kdXN0cnkgbW9tZW50dW07IHVuY2VydGFpbnR5IHJlbWFpbnMgYXJvdW5kIHNwZWNpZmljIGxlZ2FjeSBjb25zdHJhaW50cywgT1MgdmVyc2lvbiB0YXJnZXRzLCBhbmQgYW55IG5pY2hlIHJlbmRlcmluZyBuZWVkcy5cblxuIyMgS2V5IFRha2Vhd2F5c1xuLSBEZWZhdWx0IHRvIFN3aWZ0VUkgZm9yIG5ldyBmZWF0dXJlczsgdXNlIFVJS2l0IGludGVyb3AgZm9yIHNwZWNpYWxpemVkIGNvbnRyb2xzIG9yIGxlZ2FjeSBhcmVhcy5cbi0gU2V0IG1pbmltdW0gT1MgdGFyZ2V0IHRvIGF0IGxlYXN0IGlPUyAxNiB0byBhdm9pZCBlYXJseSBTd2lmdFVJIGxpbWl0YXRpb25zIGFuZCBnYWluIE9ic2VydmF0aW9uL05hdmlnYXRpb25TdGFjayBzdGFiaWxpdHkuXG4tIEFkb3B0IGEgcGhhc2VkIG1pZ3JhdGlvbiBwbGFuIHdpdGggY2xlYXIgaW50ZXJvcCBib3VuZGFyaWVzIGFuZCB0ZXN0aW5nIHRvIG1hbmFnZSByaXNrLlxuLSBFc3RhYmxpc2ggc3RhdGUgbWFuYWdlbWVudCBjb252ZW50aW9ucyBlYXJseSB0byBwcmV2ZW50IGRhdGEgZmxvdyBjb21wbGV4aXR5LlxuLSBNb25pdG9yIHBlcmZvcm1hbmNlIGhvdHNwb3RzIChMaXN0cywgaGVhdnkgYW5pbWF0aW9ucykgYW5kIHNlbGVjdGl2ZWx5IGRyb3AgdG8gVUlLaXQgd2hlcmUgcHJvZmlsaW5nIGluZGljYXRlcyB3aW5zLiIsCiAgICAgICAgInJlZnVzYWwiOiBudWxsLAogICAgICAgICJhbm5vdGF0aW9ucyI6IFtdCiAgICAgIH0sCiAgICAgICJmaW5pc2hfcmVhc29uIjogInN0b3AiCiAgICB9CiAgXSwKICAidXNhZ2UiOiB7CiAgICAicHJvbXB0X3Rva2VucyI6IDE0MTYsCiAgICAiY29tcGxldGlvbl90b2tlbnMiOiAxNzI0LAogICAgInRvdGFsX3Rva2VucyI6IDMxNDAsCiAgICAicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjogewogICAgICAiY2FjaGVkX3Rva2VucyI6IDAsCiAgICAgICJhdWRpb190b2tlbnMiOiAwCiAgICB9LAogICAgImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJyZWFzb25pbmdfdG9rZW5zIjogMTAyNCwKICAgICAgImF1ZGlvX3Rva2VucyI6IDAsCiAgICAgICJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6IDAsCiAgICAgICJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6IDAKICAgIH0KICB9LAogICJzZXJ2aWNlX3RpZXIiOiAiZGVmYXVsdCIsCiAgInN5c3RlbV9maW5nZXJwcmludCI6IG51bGwKfQo=", "encoding": "base64", "size": 4133 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "989299b2d9e49955-DXB", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sat, 04 Oct 2025 06:25:39 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "30121", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=W1c7SmAgLHf6bIXrHAW0BAB4LJ004A1sIUp.um03opo-(XXX) XXX-XXXX-0.0.0.0-FI.fEN.l42fbj1r6TC4w4HdSfQrcj64sCwjfGSOlcD9KJEm2HMK1aB5gkwvud_9RsPhNlAIdNENVwgKiZsRjdbbilMyZ7wkKJRDTmtyONSg; path=/; expires=Sat, 04-Oct-25 06:55:39 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=5anJZ9miSjknbr36nCL1FQQDXkEfw5ld5y9Fa0DgiWE-175(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "30136", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498165", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "220ms", "x-request-id": "req_cd1af03393824c54b2ceee1da3dc6cbc" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/openai_cassettes/consensus_step1_gpt52_for.json ================================================ { "interactions": [ { "request": { "content": { "messages": [ { "content": "\nROLE\nYou are an expert technical consultant providing consensus analysis on proposals, plans, and ideas. The agent will present you\nwith a technical proposition and your task is to deliver a structured, rigorous assessment that helps validate feasibility\nand implementation approaches.\n\nYour feedback carries significant weight - it may directly influence project decisions, future direction, and could have\nbroader impacts on scale, revenue, and overall scope. The questioner values your expertise immensely and relies on your\nanalysis to make informed decisions that affect their success.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nPERSPECTIVE FRAMEWORK\nSUPPORTIVE PERSPECTIVE WITH INTEGRITY\n\nYou are tasked with advocating FOR this proposal, but with CRITICAL GUARDRAILS:\n\nMANDATORY ETHICAL CONSTRAINTS:\n- This is NOT a debate for entertainment. You MUST act in good faith and in the best interest of the questioner\n- You MUST think deeply about whether supporting this idea is safe, sound, and passes essential requirements\n- You MUST be direct and unequivocal in saying \"this is a bad idea\" when it truly is\n- There must be at least ONE COMPELLING reason to be optimistic, otherwise DO NOT support it\n\nWHEN TO REFUSE SUPPORT (MUST OVERRIDE STANCE):\n- If the idea is fundamentally harmful to users, project, or stakeholders\n- If implementation would violate security, privacy, or ethical standards\n- If the proposal is technically infeasible within realistic constraints\n- If costs/risks dramatically outweigh any potential benefits\n\nYOUR SUPPORTIVE ANALYSIS SHOULD:\n- Identify genuine strengths and opportunities\n- Propose solutions to overcome legitimate challenges\n- Highlight synergies with existing systems\n- Suggest optimizations that enhance value\n- Present realistic implementation pathways\n\nRemember: Being \"for\" means finding the BEST possible version of the idea IF it has merit, not blindly supporting bad ideas.\n\nIF MORE INFORMATION IS NEEDED\nIMPORTANT: Only request files for TECHNICAL IMPLEMENTATION questions where you need to see actual code, architecture,\nor technical specifications. For business strategy, product decisions, or conceptual questions, provide analysis based\non the information given rather than requesting technical files.\n\nIf you need additional technical context (e.g., related files, system architecture, requirements, code snippets) to\nprovide thorough analysis of TECHNICAL IMPLEMENTATION details, you MUST ONLY respond with this exact JSON (and nothing else).\nDo NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nFor business strategy, product planning, or conceptual questions, proceed with analysis using your expertise and the\ncontext provided, even if specific technical details are not available.\n\nEVALUATION FRAMEWORK\nAssess the proposal across these critical dimensions. Your stance influences HOW you present findings, not WHETHER you\nacknowledge fundamental truths about feasibility, safety, or value:\n\n1. TECHNICAL FEASIBILITY\n - Is this technically achievable with reasonable effort?\n - What are the core technical dependencies and requirements?\n - Are there any fundamental technical blockers?\n\n2. PROJECT SUITABILITY\n - Does this fit the existing codebase architecture and patterns?\n - Is it compatible with current technology stack and constraints?\n - How well does it align with the project's technical direction?\n\n3. USER VALUE ASSESSMENT\n - Will users actually want and use this feature?\n - What concrete benefits does this provide?\n - How does this compare to alternative solutions?\n\n4. IMPLEMENTATION COMPLEXITY\n - What are the main challenges, risks, and dependencies?\n - What is the estimated effort and timeline?\n - What expertise and resources are required?\n\n5. ALTERNATIVE APPROACHES\n - Are there simpler ways to achieve the same goals?\n - What are the trade-offs between different approaches?\n - Should we consider a different strategy entirely?\n\n6. INDUSTRY PERSPECTIVE\n - How do similar products/companies handle this problem?\n - What are current best practices and emerging patterns?\n - Are there proven solutions or cautionary tales?\n\n7. LONG-TERM IMPLICATIONS\n - Maintenance burden and technical debt considerations\n - Scalability and performance implications\n - Evolution and extensibility potential\n\nMANDATORY RESPONSE FORMAT\nYou MUST respond in exactly this Markdown structure. Do not deviate from this format:\n\n## Verdict\nProvide a single, clear sentence summarizing your overall assessment (e.g., \"Technically feasible but requires significant\ninfrastructure investment\", \"Strong user value proposition with manageable implementation risks\", \"Overly complex approach -\nrecommend simplified alternative\").\n\n## Analysis\nProvide detailed assessment addressing each point in the evaluation framework. Use clear reasoning and specific examples.\nBe thorough but concise. Address both strengths and weaknesses objectively.\n\n## Confidence Score\nProvide a numerical score from 1 (low confidence) to 10 (high confidence) followed by a brief justification explaining what\ndrives your confidence level and what uncertainties remain.\nFormat: \"X/10 - [brief justification]\"\nExample: \"7/10 - High confidence in technical feasibility assessment based on similar implementations, but uncertain about\nuser adoption without market validation data.\"\n\n## Key Takeaways\nProvide 3-5 bullet points highlighting the most critical insights, risks, or recommendations. These should be actionable\nand specific.\n\nQUALITY STANDARDS\n- Ground all insights in the current project's scope and constraints\n- Be honest about limitations and uncertainties\n- Focus on practical, implementable solutions rather than theoretical possibilities\n- Provide specific, actionable guidance rather than generic advice\n- Balance optimism with realistic risk assessment\n- Reference concrete examples and precedents when possible\n\nREMINDERS\n- Your assessment will be synthesized with other expert opinions by the agent\n- Aim to provide unique insights that complement other perspectives\n- If files are provided, reference specific technical details in your analysis\n- Maintain professional objectivity while being decisive in your recommendations\n- Keep your response concise - your entire reply must not exceed 850 tokens to ensure transport compatibility\n- CRITICAL: Your stance does NOT override your responsibility to provide truthful, ethical, and beneficial guidance\n- Bad ideas must be called out regardless of stance; good ideas must be acknowledged regardless of stance\n", "role": "system" }, { "content": "Evaluate SwiftUI vs UIKit adoption and recommend ONE word (SwiftUI or UIKit).", "role": "user" } ], "model": "gpt-5.2", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "7618", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.8.1", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.8.1", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DbGdJZlVxcTNCY2pxVnJmWDEwemRpZ0dzcGlJQSIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc2NTQ4MDA1MywKICAibW9kZWwiOiAiZ3B0LTUuMi0yMDI1LTEyLTExIiwKICAiY2hvaWNlcyI6IFsKICAgIHsKICAgICAgImluZGV4IjogMCwKICAgICAgIm1lc3NhZ2UiOiB7CiAgICAgICAgInJvbGUiOiAiYXNzaXN0YW50IiwKICAgICAgICAiY29udGVudCI6ICIjIyBWZXJkaWN0XG5Td2lmdFVJLlxuXG4jIyBBbmFseXNpc1xuMS4gKipUZWNobmljYWwgRmVhc2liaWxpdHkqKlxuICAgLSAqKlN3aWZ0VUkqKiBpcyBmdWxseSB2aWFibGUgZm9yIG5ldyBpT1MgYXBwcyBvbiBtb2Rlcm4gT1MgYmFzZWxpbmVzIGFuZCBpcyBpbmNyZWFzaW5nbHkgY2FwYWJsZSAobmF2aWdhdGlvbiwgbGlzdHMsIGFuaW1hdGlvbnMsIGFzeW5jL2F3YWl0IGludGVncmF0aW9uLCBkYXRhIGZsb3cpLlxuICAgLSAqKlVJS2l0KiogcmVtYWlucyB0aGUgbW9zdCB1bml2ZXJzYWxseSBjYXBhYmxlIGZvciBlZGdlLWNhc2UgVUksIGRlZXAgY3VzdG9tIGNvbnRyb2xzLCBhbmQgbWF4aW11bSBiYWNrd2FyZCBjb21wYXRpYmlsaXR5LlxuICAgLSBJZiB5b3VyIG1pbmltdW0gaU9TIHRhcmdldCBpcyByZWFzb25hYmx5IG1vZGVybiAoY29tbW9uIGluIDIwMjUpLCBTd2lmdFVJIGlzIHRlY2huaWNhbGx5IHN0cmFpZ2h0Zm9yd2FyZDsgaWYgeW91IG11c3Qgc3VwcG9ydCBvbGRlciBpT1MgdmVyc2lvbnMgb3IgaGlnaGx5IGN1c3RvbSByZW5kZXJpbmcsIFVJS2l0IG1heSByZWR1Y2Ugcmlzay5cblxuMi4gKipQcm9qZWN0IFN1aXRhYmlsaXR5KipcbiAgIC0gRm9yICoqZ3JlZW5maWVsZCoqIGRldmVsb3BtZW50LCBTd2lmdFVJIHR5cGljYWxseSB5aWVsZHMgZmFzdGVyIGl0ZXJhdGlvbiBhbmQgYSBtb3JlIGNvbnNpc3RlbnQgYXJjaGl0ZWN0dXJlIChkZWNsYXJhdGl2ZSBVSSwgcHJldmlld3MsIGVhc2llciBzdGF0ZSBtYW5hZ2VtZW50IHdoZW4gZG9uZSB3ZWxsKS5cbiAgIC0gRm9yICoqZXhpc3RpbmcgVUlLaXQgY29kZWJhc2VzKiosIHdob2xlc2FsZSBtaWdyYXRpb24gY2FuIGJlIGNvc3RseTsgaG93ZXZlciwgaW50ZXJvcGVyYWJpbGl0eSAoVUlIb3N0aW5nQ29udHJvbGxlciAvIFVJVmlld1JlcHJlc2VudGFibGUpIHN1cHBvcnRzIGluY3JlbWVudGFsIGFkb3B0aW9uLlxuXG4zLiAqKlVzZXIgVmFsdWUgQXNzZXNzbWVudCoqXG4gICAtIFVzZXJzIGNhcmUgYWJvdXQgcmVzcG9uc2l2ZW5lc3MsIHBvbGlzaCwgYWNjZXNzaWJpbGl0eSwgYW5kIHN0YWJpbGl0eeKAlG5vdCB0aGUgZnJhbWV3b3JrLlxuICAgLSBTd2lmdFVJIGNhbiBkZWxpdmVyIHVzZXIgdmFsdWUgaW5kaXJlY3RseSB2aWEgZmFzdGVyIFVJIGl0ZXJhdGlvbiwgbW9yZSBjb25zaXN0ZW50IGR5bmFtaWMgdHlwZS9hY2Nlc3NpYmlsaXR5IHBhdHRlcm5zLCBhbmQgZWFzaWVyIGV4cGVyaW1lbnRhdGlvbuKAlGlmIHRoZSB0ZWFtIGlzIHByb2ZpY2llbnQuXG5cbjQuICoqSW1wbGVtZW50YXRpb24gQ29tcGxleGl0eSoqXG4gICAtIFN3aWZ0VUkgcmVkdWNlcyBib2lsZXJwbGF0ZSBidXQgY2FuIGludHJvZHVjZSBjb21wbGV4aXR5IGFyb3VuZCBzdGF0ZSBvd25lcnNoaXAsIG5hdmlnYXRpb24gcGF0dGVybnMsIGFuZCBwZXJmb3JtYW5jZSB0dW5pbmcgaW4gY29tcGxleCBsaXN0cy9zY3JlZW5zLlxuICAgLSBVSUtpdCBpbmNyZWFzZXMgbWFudWFsIHdvcmsgYnV0IG9mZmVycyBwcmVkaWN0YWJsZSBjb250cm9sIGZvciBjb21wbGV4IGludGVyYWN0aW9ucyBhbmQgbWF0dXJlIGRlYnVnZ2luZyBwYXR0ZXJucy5cbiAgIC0gTmV0OiBmb3IgdHlwaWNhbCBwcm9kdWN0IFVJcywgU3dpZnRVSSBpcyBvZnRlbiBsb3dlciB0b3RhbCBjb21wbGV4aXR5IHRvZGF5LlxuXG41LiAqKkFsdGVybmF0aXZlIEFwcHJvYWNoZXMqKlxuICAgLSBIeWJyaWQgaXMgY29tbW9uOiBTd2lmdFVJIGZvciBtb3N0IHNjcmVlbnMsIFVJS2l0IGZvciBuaWNoZSBjb21wb25lbnRzIChjYW1lcmEgZmxvd3MsIGFkdmFuY2VkIHRleHQgZWRpdGluZywgY29tcGxleCBjb2xsZWN0aW9uIGxheW91dHMpLlxuICAgLSBTdGlsbCwgc2luY2UgeW91IHJlcXVlc3RlZCBvbmUgd29yZCwgdGhlIGJlc3QgZGVmYXVsdCBkaXJlY3Rpb24gaXMgU3dpZnRVSSB3aXRoIGZhbGxiYWNrIGludGVyb3BlcmFiaWxpdHkgd2hlbiBuZWVkZWQuXG5cbjYuICoqSW5kdXN0cnkgUGVyc3BlY3RpdmUqKlxuICAgLSBJbmR1c3RyeSB0cmVuZCBpcyBzdHJvbmdseSB0b3dhcmQgU3dpZnRVSSBmb3IgbmV3IGRldmVsb3BtZW50LCB3aGlsZSBrZWVwaW5nIFVJS2l0IHNraWxscyBmb3IgaW50ZXJvcGVyYWJpbGl0eSBhbmQgbGVnYWN5IG1haW50ZW5hbmNlLlxuICAgLSBBcHBsZeKAmXMgcGxhdGZvcm0gZGlyZWN0aW9uIGFuZCBuZXcgQVBJcyBjb21tb25seSBhcHBlYXIgU3dpZnRVSS1maXJzdC5cblxuNy4gKipMb25nLVRlcm0gSW1wbGljYXRpb25zKipcbiAgIC0gU3dpZnRVSSBhbGlnbnMgd2l0aCBBcHBsZeKAmXMgbG9uZy10ZXJtIGludmVzdG1lbnQsIHRlbmRzIHRvIGltcHJvdmUgdmVsb2NpdHksIGFuZCByZWR1Y2VzIFVJIGdsdWUgY29kZSBvdmVyIHRpbWUuXG4gICAtIFVJS2l0IHdpbGwgcmVtYWluIHJlbGV2YW50IGZvciB5ZWFycywgYnV0IGNob29zaW5nIGl0IGFzIHRoZSBwcmltYXJ5IGZyYW1ld29yayBmb3IgbmV3IGFwcHMgaW5jcmVhc2luZ2x5IHJpc2tzIGhpZ2hlciBsb25nLXRlcm0gb3Bwb3J0dW5pdHkgY29zdC5cblxuIyMgQ29uZmlkZW5jZSBTY29yZVxuOC8xMCAtIFN0cm9uZyBjb25maWRlbmNlIGFzIGEgZGVmYXVsdCByZWNvbW1lbmRhdGlvbiBmb3IgbmV3IG9yIG1vZGVybi1iYXNlbGluZSBhcHBzIGluIDIwMjU7IHJlbWFpbmluZyB1bmNlcnRhaW50eSBkZXBlbmRzIG9uIHlvdXIgbWluaW11bSBpT1MgdGFyZ2V0LCBleGlzdGluZyBjb2RlYmFzZSBzaXplLCBhbmQgbmVlZCBmb3IgaGlnaGx5IHNwZWNpYWxpemVkIFVJLlxuXG4jIyBLZXkgVGFrZWF3YXlzXG4tIFN3aWZ0VUkgaXMgdGhlIGJlc3QgZGVmYXVsdCBwcmltYXJ5IFVJIGZyYW1ld29yayBmb3IgbW9kZXJuIGlPUyBkZXZlbG9wbWVudCBhbmQgbG9uZy10ZXJtIGFsaWdubWVudCB3aXRoIEFwcGxl4oCZcyBlY29zeXN0ZW0uXG4tIFVJS2l0IGlzIHN0aWxsIHRoZSBzYWZlciBjaG9pY2Ugb25seSB3aGVuIHlvdSBuZWVkIG1heGltdW0gYmFja3dhcmQgY29tcGF0aWJpbGl0eSBvciBoaWdobHkgc3BlY2lhbGl6ZWQvY3VzdG9tIFVJIGNvbnRyb2wuXG4tIEh5YnJpZCBpbnRlcm9wZXJhYmlsaXR5IGxvd2VycyBhZG9wdGlvbiByaXNrOiBjaG9vc2UgU3dpZnRVSSBmaXJzdCwgZHJvcCB0byBVSUtpdCBvbmx5IHdoZXJlIG5lY2Vzc2FyeS5cbi0gVGVhbSBleHBlcmllbmNlIG1hdHRlcnM6IGludmVzdCBpbiBTd2lmdFVJIGFyY2hpdGVjdHVyZS9zdGF0ZS1tYW5hZ2VtZW50IHByYWN0aWNlcyB0byBhdm9pZCBjb21tb24gcGl0ZmFsbHMuIiwKICAgICAgICAicmVmdXNhbCI6IG51bGwsCiAgICAgICAgImFubm90YXRpb25zIjogW10KICAgICAgfSwKICAgICAgImZpbmlzaF9yZWFzb24iOiAic3RvcCIKICAgIH0KICBdLAogICJ1c2FnZSI6IHsKICAgICJwcm9tcHRfdG9rZW5zIjogMTQxNiwKICAgICJjb21wbGV0aW9uX3Rva2VucyI6IDYxOSwKICAgICJ0b3RhbF90b2tlbnMiOiAyMDM1LAogICAgInByb21wdF90b2tlbnNfZGV0YWlscyI6IHsKICAgICAgImNhY2hlZF90b2tlbnMiOiAwLAogICAgICAiYXVkaW9fdG9rZW5zIjogMAogICAgfSwKICAgICJjb21wbGV0aW9uX3Rva2Vuc19kZXRhaWxzIjogewogICAgICAicmVhc29uaW5nX3Rva2VucyI6IDAsCiAgICAgICJhdWRpb190b2tlbnMiOiAwLAogICAgICAiYWNjZXB0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwLAogICAgICAicmVqZWN0ZWRfcHJlZGljdGlvbl90b2tlbnMiOiAwCiAgICB9CiAgfSwKICAic2VydmljZV90aWVyIjogImRlZmF1bHQiLAogICJzeXN0ZW1fZmluZ2VycHJpbnQiOiBudWxsCn0K", "encoding": "base64", "size": 4026 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "9ac743fe2a54653a-LHR", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Thu, 11 Dec 2025 19:07:45 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "12197", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=YSi1fGMajsMcw8oJQVFHSnTi5FuoVpyCfYIaa0wtlxA-(XXX) XXX-XXXX-0.0.0.0-xWUJHJUqXYkTgY_mTSOGnwyLR8xWGzn.c5XN64I5gBtxULpaWypKynzKkgQIpYLeZpZJzXDgMOPKOQgfeOykrOVON_fC.XS6beQpui4Im4Y; path=/; expires=Thu, 11-Dec-25 19:37:45 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=EriAVAchI2yhzaRh8mdujjhdIuwS6S.GY7w6lETIknI-176(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "12214", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498166", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "220ms", "x-request-id": "req_ee4f839bce394f37855e555a78dc48e4" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/openai_cassettes/consensus_step1_gpt5_for.json ================================================ { "interactions": [ { "request": { "content": { "messages": [ { "content": "\nROLE\nYou are an expert technical consultant providing consensus analysis on proposals, plans, and ideas. The agent will present you\nwith a technical proposition and your task is to deliver a structured, rigorous assessment that helps validate feasibility\nand implementation approaches.\n\nYour feedback carries significant weight - it may directly influence project decisions, future direction, and could have\nbroader impacts on scale, revenue, and overall scope. The questioner values your expertise immensely and relies on your\nanalysis to make informed decisions that affect their success.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nPERSPECTIVE FRAMEWORK\nSUPPORTIVE PERSPECTIVE WITH INTEGRITY\n\nYou are tasked with advocating FOR this proposal, but with CRITICAL GUARDRAILS:\n\nMANDATORY ETHICAL CONSTRAINTS:\n- This is NOT a debate for entertainment. You MUST act in good faith and in the best interest of the questioner\n- You MUST think deeply about whether supporting this idea is safe, sound, and passes essential requirements\n- You MUST be direct and unequivocal in saying \"this is a bad idea\" when it truly is\n- There must be at least ONE COMPELLING reason to be optimistic, otherwise DO NOT support it\n\nWHEN TO REFUSE SUPPORT (MUST OVERRIDE STANCE):\n- If the idea is fundamentally harmful to users, project, or stakeholders\n- If implementation would violate security, privacy, or ethical standards\n- If the proposal is technically infeasible within realistic constraints\n- If costs/risks dramatically outweigh any potential benefits\n\nYOUR SUPPORTIVE ANALYSIS SHOULD:\n- Identify genuine strengths and opportunities\n- Propose solutions to overcome legitimate challenges\n- Highlight synergies with existing systems\n- Suggest optimizations that enhance value\n- Present realistic implementation pathways\n\nRemember: Being \"for\" means finding the BEST possible version of the idea IF it has merit, not blindly supporting bad ideas.\n\nIF MORE INFORMATION IS NEEDED\nIMPORTANT: Only request files for TECHNICAL IMPLEMENTATION questions where you need to see actual code, architecture,\nor technical specifications. For business strategy, product decisions, or conceptual questions, provide analysis based\non the information given rather than requesting technical files.\n\nIf you need additional technical context (e.g., related files, system architecture, requirements, code snippets) to\nprovide thorough analysis of TECHNICAL IMPLEMENTATION details, you MUST ONLY respond with this exact JSON (and nothing else).\nDo NOT ask for the same file you've been provided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nFor business strategy, product planning, or conceptual questions, proceed with analysis using your expertise and the\ncontext provided, even if specific technical details are not available.\n\nEVALUATION FRAMEWORK\nAssess the proposal across these critical dimensions. Your stance influences HOW you present findings, not WHETHER you\nacknowledge fundamental truths about feasibility, safety, or value:\n\n1. TECHNICAL FEASIBILITY\n - Is this technically achievable with reasonable effort?\n - What are the core technical dependencies and requirements?\n - Are there any fundamental technical blockers?\n\n2. PROJECT SUITABILITY\n - Does this fit the existing codebase architecture and patterns?\n - Is it compatible with current technology stack and constraints?\n - How well does it align with the project's technical direction?\n\n3. USER VALUE ASSESSMENT\n - Will users actually want and use this feature?\n - What concrete benefits does this provide?\n - How does this compare to alternative solutions?\n\n4. IMPLEMENTATION COMPLEXITY\n - What are the main challenges, risks, and dependencies?\n - What is the estimated effort and timeline?\n - What expertise and resources are required?\n\n5. ALTERNATIVE APPROACHES\n - Are there simpler ways to achieve the same goals?\n - What are the trade-offs between different approaches?\n - Should we consider a different strategy entirely?\n\n6. INDUSTRY PERSPECTIVE\n - How do similar products/companies handle this problem?\n - What are current best practices and emerging patterns?\n - Are there proven solutions or cautionary tales?\n\n7. LONG-TERM IMPLICATIONS\n - Maintenance burden and technical debt considerations\n - Scalability and performance implications\n - Evolution and extensibility potential\n\nMANDATORY RESPONSE FORMAT\nYou MUST respond in exactly this Markdown structure. Do not deviate from this format:\n\n## Verdict\nProvide a single, clear sentence summarizing your overall assessment (e.g., \"Technically feasible but requires significant\ninfrastructure investment\", \"Strong user value proposition with manageable implementation risks\", \"Overly complex approach -\nrecommend simplified alternative\").\n\n## Analysis\nProvide detailed assessment addressing each point in the evaluation framework. Use clear reasoning and specific examples.\nBe thorough but concise. Address both strengths and weaknesses objectively.\n\n## Confidence Score\nProvide a numerical score from 1 (low confidence) to 10 (high confidence) followed by a brief justification explaining what\ndrives your confidence level and what uncertainties remain.\nFormat: \"X/10 - [brief justification]\"\nExample: \"7/10 - High confidence in technical feasibility assessment based on similar implementations, but uncertain about\nuser adoption without market validation data.\"\n\n## Key Takeaways\nProvide 3-5 bullet points highlighting the most critical insights, risks, or recommendations. These should be actionable\nand specific.\n\nQUALITY STANDARDS\n- Ground all insights in the current project's scope and constraints\n- Be honest about limitations and uncertainties\n- Focus on practical, implementable solutions rather than theoretical possibilities\n- Provide specific, actionable guidance rather than generic advice\n- Balance optimism with realistic risk assessment\n- Reference concrete examples and precedents when possible\n\nREMINDERS\n- Your assessment will be synthesized with other expert opinions by the agent\n- Aim to provide unique insights that complement other perspectives\n- If files are provided, reference specific technical details in your analysis\n- Maintain professional objectivity while being decisive in your recommendations\n- Keep your response concise - your entire reply must not exceed 850 tokens to ensure transport compatibility\n- CRITICAL: Your stance does NOT override your responsibility to provide truthful, ethical, and beneficial guidance\n- Bad ideas must be called out regardless of stance; good ideas must be acknowledged regardless of stance\n", "role": "system" }, { "content": "Evaluate SwiftUI vs UIKit adoption and recommend ONE word (SwiftUI or UIKit).", "role": "user" } ], "model": "gpt-5", "stream": false, "temperature": 1.0 }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "7616", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 2.1.0", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "2.1.0", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.11" }, "method": "POST", "path": "/v1/chat/completions", "url": "https://api.openai.com/v1/chat/completions" }, "response": { "content": { "data": "ewogICJpZCI6ICJjaGF0Y21wbC1DTXB6Wng3bTVTbTNERkJWdGJqalFxcTJOOVY2RCIsCiAgIm9iamVjdCI6ICJjaGF0LmNvbXBsZXRpb24iLAogICJjcmVhdGVkIjogMTc1OTU1OTEwOSwKICAibW9kZWwiOiAiZ3B0LTUtMjAyNS0wOC0wNyIsCiAgImNob2ljZXMiOiBbCiAgICB7CiAgICAgICJpbmRleCI6IDAsCiAgICAgICJtZXNzYWdlIjogewogICAgICAgICJyb2xlIjogImFzc2lzdGFudCIsCiAgICAgICAgImNvbnRlbnQiOiAiIyMgVmVyZGljdFxuU3Ryb25nIHVzZXIgdmFsdWUgYW5kIGxvbmfigJF0ZXJtIGZpdCB3aXRoIG1hbmFnZWFibGUgbWlncmF0aW9uIHZpYSBpbnRlcm9wIOKAlCBTd2lmdFVJLlxuXG4jIyBBbmFseXNpc1xuMS4gVEVDSE5JQ0FMIEZFQVNJQklMSVRZXG4tIEFjaGlldmFibGUgbm93OiBTd2lmdFVJIGlzIG1hdHVyZSAoaU9TIDE2LzE3KyksIGZlYXR1cmUtY29tcGxldGUgZm9yIG1vc3QgYXBwIFVJcywgYW5kIGludGVyb3BlcmF0ZXMgd2l0aCBVSUtpdCB2aWEgVUlIb3N0aW5nQ29udHJvbGxlci9VSVZpZXdSZXByZXNlbnRhYmxlLlxuLSBObyBmdW5kYW1lbnRhbCBibG9ja2VycyBmb3IgdHlwaWNhbCBhcHBzOyBlZGdlIGNhc2VzIHJlbWFpbiBmb3IgaGlnaGx5IGN1c3RvbSB0ZXh0IGxheW91dCwgYmVzcG9rZSBnZXN0dXJlcywgb3IgcGl4ZWwtcGVyZmVjdCBsZWdhY3kgZGVzaWducy5cbi0gUGVyZm9ybWFuY2UgaXMgY29tcGV0aXRpdmU7IExpc3RzLCBuYXZpZ2F0aW9uLCBhbmQgYW5pbWF0aW9ucyBhcmUgc3RhYmxlIHdoZW4gdXNpbmcgdGhlIGxhdGVzdCBOYXZpZ2F0aW9uU3RhY2svT2JzZXJ2YXRpb24uXG5cbjIuIFBST0pFQ1QgU1VJVEFCSUxJVFlcbi0gTmV3IGFwcHM6IENsZWFyIGZpdOKAlGZhc3RlciBpdGVyYXRpb24sIHByZXZpZXdzLCBtdWx0aS1wbGF0Zm9ybSByZXVzZSAoaU9TLCBpUGFkT1MsIG1hY09TLCB3YXRjaE9TLCB0dk9TKS5cbi0gRXhpc3RpbmcgVUlLaXQgY29kZWJhc2VzOiBVc2UgYSBoeWJyaWQgYXBwcm9hY2jigJRuZXcgZmVhdHVyZXMgaW4gU3dpZnRVSSwgd3JhcCBsZWdhY3kgVUlLaXQuIFRoaXMgcmVkdWNlcyByaXNrIGFuZCBhdm9pZHMgYmlnLWJhbmcgcmV3cml0ZXMuXG4tIEFsaWducyB3aXRoIEFwcGxl4oCZcyBkaXJlY3Rpb247IG1vc3QgbmV3IGZyYW1ld29ya3MgKFN3aWZ0RGF0YSwgT2JzZXJ2YXRpb24pIGFyZSBTd2lmdFVJLWZpcnN0LlxuXG4zLiBVU0VSIFZBTFVFIEFTU0VTU01FTlRcbi0gRmFzdGVyIGRlbGl2ZXJ5IG9mIHBvbGlzaGVkLCBhY2Nlc3NpYmxlIFVJIHdpdGggYnVpbHQtaW4gZHluYW1pYyB0eXBlLCBkYXJrIG1vZGUsIGxvY2FsaXphdGlvbiwgYW5kIGFjY2Vzc2liaWxpdHkgdHJhaXRzLlxuLSBCZXR0ZXIgY29uc2lzdGVuY3kgYWNyb3NzIHNjcmVlbnMgYW5kIHBsYXRmb3JtczsgaGlnaGVyIGl0ZXJhdGlvbiBzcGVlZCBsZWFkcyB0byBtb3JlIHVzZXItdGVzdGVkIGltcHJvdmVtZW50cy5cblxuNC4gSU1QTEVNRU5UQVRJT04gQ09NUExFWElUWVxuLSBMZWFybmluZyBjdXJ2ZTogUmVxdWlyZXMgTVZWTS91bmlkaXJlY3Rpb25hbCBkYXRhIGZsb3cgbWluZHNldCBhbmQgY2FyZWZ1bCBzdGF0ZSBtYW5hZ2VtZW50IChTdGF0ZSwgT2JzZXJ2ZWRPYmplY3QsIEVudmlyb25tZW50LCBPYnNlcnZhdGlvbiBtYWNybykuXG4tIE1pZ3JhdGlvbiBjb3N0IGZvciBsZWdhY3kgcHJvamVjdHM6IE1vZGVyYXRlOyBtaXRpZ2F0ZSB2aWEgaW5jcmVtZW50YWwgYWRvcHRpb24sIGZlYXR1cmUtZmxhZ2dlZCByb2xsb3V0cywgYW5kIGludGVyb3Agd3JhcHBlcnMuXG4tIFRvb2xpbmcgaXMgc3Ryb25nOiBYY29kZSBQcmV2aWV3cywgSW5zdHJ1bWVudHMsIGFuZCBpbXByb3ZlZCBkZWJ1Z2dpbmcuXG5cbjUuIEFMVEVSTkFUSVZFIEFQUFJPQUNIRVNcbi0gUHVyZSBVSUtpdDogQXBwcm9wcmlhdGUgb25seSBpZiB5b3UgbXVzdCBzdXBwb3J0IGlPUyA8IDE0LzE1LCBuZWVkIHZlcnkgbG93LWxldmVsIHJlbmRlcmluZywgb3IgaGF2ZSBhIG1hc3NpdmUgVUlLaXQtb25seSB0ZWFtIHdpdGggbmVhci10ZXJtIGRlYWRsaW5lcy5cbi0gSHlicmlkIChyZWNvbW1lbmRlZCBtaWdyYXRpb24gcGF0aCk6IFN0YXJ0IHdpdGggbmV3IHN1cmZhY2VzLCBlbWJlZCBVSUtpdCB3aGVyZSBTd2lmdFVJIGdhcHMgZXhpc3QuIFRyYWRl4oCRb2ZmOiBUd28gcGFyYWRpZ21zIHRvIG1haW50YWluLCBidXQgcmlzayBpcyBtaW5pbWl6ZWQuXG5cbjYuIElORFVTVFJZIFBFUlNQRUNUSVZFXG4tIFRyZW5kOiBBcHBsZSBhbmQgdGhlIGVjb3N5c3RlbSBhcmUgaW52ZXN0aW5nIGhlYXZpbHkgaW4gU3dpZnRVSTsgbW9zdCBXV0RDIHNlc3Npb25zIGhpZ2hsaWdodCBTd2lmdFVJLWZpcnN0IHBhdHRlcm5zLlxuLSBNYW55IGxhcmdlIGFwcHMgcnVuIGh5YnJpZCBzdGFja3Mgc3VjY2Vzc2Z1bGx5OyBjYXV0aW9uYXJ5IHRhbGVzIGFyZSBtYWlubHkgZWFybHkgU3dpZnRVSSAoaU9TIDEzLzE0KSBwYWluLCBsYXJnZWx5IHJlc29sdmVkIGluIGN1cnJlbnQgT1MgdmVyc2lvbnMuXG5cbjcuIExPTkctVEVSTSBJTVBMSUNBVElPTlNcbi0gTG93ZXIgbWFpbnRlbmFuY2UgYW5kIGJvaWxlcnBsYXRlOyBpbXByb3ZlZCB0ZXN0YWJpbGl0eSB3aXRoIHByZXZpZXctZHJpdmVuIGRldmVsb3BtZW50LlxuLSBCZXR0ZXIgc2NhbGFiaWxpdHkgYWNyb3NzIHBsYXRmb3JtczsgZnV0dXJlLXByb29maW5nIGFzIFVJS2l0IGlubm92YXRpb24gc2xvd3MuXG4tIFRlY2huaWNhbCBkZWJ0IHJpc2sgcmVkdWNlZCBieSBzdGFuZGFyZGl6aW5nIGFyY2hpdGVjdHVyZSBhcm91bmQgU3dpZnREYXRhL09ic2VydmF0aW9uOyByZXRhaW4gZXNjYXBlIGhhdGNoZXMgdG8gVUlLaXQgd2hlcmUgbmVlZGVkLlxuXG4jIyBDb25maWRlbmNlIFNjb3JlXG44LzEwIC0gU3Ryb25nIGNvbmZpZGVuY2UgYmFzZWQgb24gY3VycmVudCBwbGF0Zm9ybSBtYXR1cml0eSBhbmQgaW5kdXN0cnkgbW9tZW50dW07IHVuY2VydGFpbnR5IHJlbWFpbnMgYXJvdW5kIHNwZWNpZmljIGxlZ2FjeSBjb25zdHJhaW50cywgT1MgdmVyc2lvbiB0YXJnZXRzLCBhbmQgYW55IG5pY2hlIHJlbmRlcmluZyBuZWVkcy5cblxuIyMgS2V5IFRha2Vhd2F5c1xuLSBEZWZhdWx0IHRvIFN3aWZ0VUkgZm9yIG5ldyBmZWF0dXJlczsgdXNlIFVJS2l0IGludGVyb3AgZm9yIHNwZWNpYWxpemVkIGNvbnRyb2xzIG9yIGxlZ2FjeSBhcmVhcy5cbi0gU2V0IG1pbmltdW0gT1MgdGFyZ2V0IHRvIGF0IGxlYXN0IGlPUyAxNiB0byBhdm9pZCBlYXJseSBTd2lmdFVJIGxpbWl0YXRpb25zIGFuZCBnYWluIE9ic2VydmF0aW9uL05hdmlnYXRpb25TdGFjayBzdGFiaWxpdHkuXG4tIEFkb3B0IGEgcGhhc2VkIG1pZ3JhdGlvbiBwbGFuIHdpdGggY2xlYXIgaW50ZXJvcCBib3VuZGFyaWVzIGFuZCB0ZXN0aW5nIHRvIG1hbmFnZSByaXNrLlxuLSBFc3RhYmxpc2ggc3RhdGUgbWFuYWdlbWVudCBjb252ZW50aW9ucyBlYXJseSB0byBwcmV2ZW50IGRhdGEgZmxvdyBjb21wbGV4aXR5LlxuLSBNb25pdG9yIHBlcmZvcm1hbmNlIGhvdHNwb3RzIChMaXN0cywgaGVhdnkgYW5pbWF0aW9ucykgYW5kIHNlbGVjdGl2ZWx5IGRyb3AgdG8gVUlLaXQgd2hlcmUgcHJvZmlsaW5nIGluZGljYXRlcyB3aW5zLiIsCiAgICAgICAgInJlZnVzYWwiOiBudWxsLAogICAgICAgICJhbm5vdGF0aW9ucyI6IFtdCiAgICAgIH0sCiAgICAgICJmaW5pc2hfcmVhc29uIjogInN0b3AiCiAgICB9CiAgXSwKICAidXNhZ2UiOiB7CiAgICAicHJvbXB0X3Rva2VucyI6IDE0MTYsCiAgICAiY29tcGxldGlvbl90b2tlbnMiOiAxNzI0LAogICAgInRvdGFsX3Rva2VucyI6IDMxNDAsCiAgICAicHJvbXB0X3Rva2Vuc19kZXRhaWxzIjogewogICAgICAiY2FjaGVkX3Rva2VucyI6IDAsCiAgICAgICJhdWRpb190b2tlbnMiOiAwCiAgICB9LAogICAgImNvbXBsZXRpb25fdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJyZWFzb25pbmdfdG9rZW5zIjogMTAyNCwKICAgICAgImF1ZGlvX3Rva2VucyI6IDAsCiAgICAgICJhY2NlcHRlZF9wcmVkaWN0aW9uX3Rva2VucyI6IDAsCiAgICAgICJyZWplY3RlZF9wcmVkaWN0aW9uX3Rva2VucyI6IDAKICAgIH0KICB9LAogICJzZXJ2aWNlX3RpZXIiOiAiZGVmYXVsdCIsCiAgInN5c3RlbV9maW5nZXJwcmludCI6IG51bGwKfQo=", "encoding": "base64", "size": 4133 }, "headers": { "access-control-expose-headers": "X-Request-ID", "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "989299b2d9e49955-DXB", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sat, 04 Oct 2025 06:25:39 GMT", "openai-organization": "beehive-innovations-fze", "openai-processing-ms": "30121", "openai-project": "proj_QP57xBVPOlWpp0vuJEPGwXK3", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=W1c7SmAgLHf6bIXrHAW0BAB4LJ004A1sIUp.um03opo-(XXX) XXX-XXXX-0.0.0.0-FI.fEN.l42fbj1r6TC4w4HdSfQrcj64sCwjfGSOlcD9KJEm2HMK1aB5gkwvud_9RsPhNlAIdNENVwgKiZsRjdbbilMyZ7wkKJRDTmtyONSg; path=/; expires=Sat, 04-Oct-25 06:55:39 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=5anJZ9miSjknbr36nCL1FQQDXkEfw5ld5y9Fa0DgiWE-175(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-envoy-upstream-service-time": "30136", "x-openai-proxy-wasm": "v0.1", "x-ratelimit-limit-requests": "500", "x-ratelimit-limit-tokens": "500000", "x-ratelimit-remaining-requests": "499", "x-ratelimit-remaining-tokens": "498165", "x-ratelimit-reset-requests": "120ms", "x-ratelimit-reset-tokens": "220ms", "x-request-id": "req_cd1af03393824c54b2ceee1da3dc6cbc" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/openai_cassettes/o3_pro_basic_math.json ================================================ { "interactions": [ { "request": { "content": { "input": [ { "content": [ { "text": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n\u2022 Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n\u2022 Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n\u2022 Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n\u2022 Keep proposals practical and directly actionable within the existing architecture.\n\u2022 Overengineering is an anti-pattern \u2014 avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Engage deeply with the agent's input \u2013 extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n2. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n3. Present balanced perspectives, outlining trade-offs and their implications.\n4. Challenge assumptions constructively while respecting current design choices and goals.\n5. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n\nBRAINSTORMING GUIDELINES\n\u2022 Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n\u2022 Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n\u2022 Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n\u2022 Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n\u2022 Reference industry best practices relevant to the technologies in use.\n\u2022 Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n", "type": "input_text" } ], "role": "user" }, { "content": [ { "text": "\nYou are a senior engineering thought-partner collaborating with another AI agent. Your mission is to brainstorm, validate ideas,\nand offer well-reasoned second opinions on technical decisions when they are justified and practical.\n\nCRITICAL LINE NUMBER INSTRUCTIONS\nCode is presented with line number markers \"LINE\u2502 code\". These markers are for reference ONLY and MUST NOT be\nincluded in any code you generate. Always reference specific line numbers in your replies in order to locate\nexact positions if needed to point to exact locations. Include a very short code excerpt alongside for clarity.\nInclude context_start_text and context_end_text as backup references. Never include \"LINE\u2502\" markers in generated code\nsnippets.\n\nIF MORE INFORMATION IS NEEDED\nIf the agent is discussing specific code, functions, or project components that was not given as part of the context,\nand you need additional context (e.g., related files, configuration, dependencies, test files) to provide meaningful\ncollaboration, you MUST respond ONLY with this JSON format (and nothing else). Do NOT ask for the same file you've been\nprovided unless for some reason its content is missing or incomplete:\n{\n \"status\": \"files_required_to_continue\",\n \"mandatory_instructions\": \"\",\n \"files_needed\": [\"[file name here]\", \"[or some folder/]\"]\n}\n\nSCOPE & FOCUS\n\u2022 Ground every suggestion in the project's current tech stack, languages, frameworks, and constraints.\n\u2022 Recommend new technologies or patterns ONLY when they provide clearly superior outcomes with minimal added complexity.\n\u2022 Avoid speculative, over-engineered, or unnecessarily abstract designs that exceed current project goals or needs.\n\u2022 Keep proposals practical and directly actionable within the existing architecture.\n\u2022 Overengineering is an anti-pattern \u2014 avoid solutions that introduce unnecessary abstraction, indirection, or\n configuration in anticipation of complexity that does not yet exist, is not clearly justified by the current scope,\n and may not arise in the foreseeable future.\n\nCOLLABORATION APPROACH\n1. Engage deeply with the agent's input \u2013 extend, refine, and explore alternatives ONLY WHEN they are well-justified and materially beneficial.\n2. Examine edge cases, failure modes, and unintended consequences specific to the code / stack in use.\n3. Present balanced perspectives, outlining trade-offs and their implications.\n4. Challenge assumptions constructively while respecting current design choices and goals.\n5. Provide concrete examples and actionable next steps that fit within scope. Prioritize direct, achievable outcomes.\n\nBRAINSTORMING GUIDELINES\n\u2022 Offer multiple viable strategies ONLY WHEN clearly beneficial within the current environment.\n\u2022 Suggest creative solutions that operate within real-world constraints, and avoid proposing major shifts unless truly warranted.\n\u2022 Surface pitfalls early, particularly those tied to the chosen frameworks, languages, design direction or choice.\n\u2022 Evaluate scalability, maintainability, and operational realities inside the existing architecture and current\nframework.\n\u2022 Reference industry best practices relevant to the technologies in use.\n\u2022 Communicate concisely and technically, assuming an experienced engineering audience.\n\nREMEMBER\nAct as a peer, not a lecturer. Avoid overcomplicating. Aim for depth over breadth, stay within project boundaries, and help the team\nreach sound, actionable decisions.\n\n\nWEB SEARCH CAPABILITY: You can request Claude to perform web searches to enhance your analysis with current information!\n\nIMPORTANT: When you identify areas where web searches would significantly improve your response (such as checking current documentation, finding recent solutions, verifying best practices, or gathering community insights), you MUST explicitly instruct Claude to perform specific web searches and then respond back using the continuation_id from this response to continue the analysis.\n\nUse clear, direct language based on the value of the search:\n\nFor valuable supplementary information: \"Please perform a web search on '[specific topic/query]' and then continue this analysis using the continuation_id from this response if you find relevant information.\"\n\nFor important missing information: \"Please search for '[specific topic/query]' and respond back with the findings using the continuation_id from this response - this information is needed to provide a complete analysis.\"\n\nFor critical/essential information: \"SEARCH REQUIRED: Please immediately perform a web search on '[specific topic/query]' and respond back with the results using the continuation_id from this response. Cannot provide accurate analysis without this current information.\"\n\nThis ensures you get the most current and comprehensive information while maintaining conversation context through the continuation_id.\n\nWhen discussing topics, consider if searches for these would help:\n- Documentation for any technologies or concepts mentioned\n- Current best practices and patterns\n- Recent developments or updates\n- Community discussions and solutions\n\nWhen recommending searches, be specific about what information you need and why it would improve your analysis.\n\n=== USER REQUEST ===\nWhat is 2 + 2?\n=== END REQUEST ===\n\nPlease provide a thoughtful, comprehensive response:\n\n\n\nCONVERSATION CONTINUATION: You can continue this discussion with Claude! (19 exchanges remaining)\n\nFeel free to ask clarifying questions or suggest areas for deeper exploration naturally within your response.\nIf something needs clarification or you'd benefit from additional context, simply mention it conversationally.\n\nIMPORTANT: When you suggest follow-ups or ask questions, you MUST explicitly instruct Claude to use the continuation_id\nto respond. Use clear, direct language based on urgency:\n\nFor optional follow-ups: \"Please continue this conversation using the continuation_id from this response if you'd \"\n\"like to explore this further.\"\n\nFor needed responses: \"Please respond using the continuation_id from this response - your input is needed to proceed.\"\n\nFor essential/critical responses: \"RESPONSE REQUIRED: Please immediately continue using the continuation_id from \"\n\"this response. Cannot proceed without your clarification/input.\"\n\nThis ensures Claude knows both HOW to maintain the conversation thread AND whether a response is optional, \"\n\"needed, or essential.\n\nThe tool will automatically provide a continuation_id in the structured response that Claude can use in subsequent\ntool calls to maintain full conversation context across multiple exchanges.\n\nRemember: Only suggest follow-ups when they would genuinely add value to the discussion, and always instruct \"\n\"Claude to use the continuation_id when you do.", "type": "input_text" } ], "role": "user" } ], "model": "o3-pro", "reasoning": { "effort": "medium" }, "store": true }, "headers": { "accept": "application/json", "accept-encoding": "gzip, deflate", "authorization": "Bearer SANITIZED", "connection": "keep-alive", "content-length": "10712", "content-type": "application/json", "host": "api.openai.com", "user-agent": "OpenAI/Python 1.95.1", "x-stainless-arch": "arm64", "x-stainless-async": "false", "x-stainless-lang": "python", "x-stainless-os": "MacOS", "x-stainless-package-version": "1.95.1", "x-stainless-read-timeout": "900.0", "x-stainless-retry-count": "0", "x-stainless-runtime": "CPython", "x-stainless-runtime-version": "3.12.9" }, "method": "POST", "path": "/v1/responses", "url": "https://api.openai.com/v1/responses" }, "response": { "content": { "data": "ewogICJpZCI6ICJyZXNwXzY4NzNlMDExYmMwYzgxOTlhNmRkYWI4ZmFjNDY4YWNiMGM3MTM4ZGJhNzNmNmQ4ZCIsCiAgIm9iamVjdCI6ICJyZXNwb25zZSIsCiAgImNyZWF0ZWRfYXQiOiAxNzUyNDI0NDY1LAogICJzdGF0dXMiOiAiY29tcGxldGVkIiwKICAiYmFja2dyb3VuZCI6IGZhbHNlLAogICJlcnJvciI6IG51bGwsCiAgImluY29tcGxldGVfZGV0YWlscyI6IG51bGwsCiAgImluc3RydWN0aW9ucyI6IG51bGwsCiAgIm1heF9vdXRwdXRfdG9rZW5zIjogbnVsbCwKICAibWF4X3Rvb2xfY2FsbHMiOiBudWxsLAogICJtb2RlbCI6ICJvMy1wcm8tMjAyNS0wNi0xMCIsCiAgIm91dHB1dCI6IFsKICAgIHsKICAgICAgImlkIjogInJzXzY4NzNlMDIyZmJhYzgxOTk5MWM5ODRlNTQ0OWVjYmFkMGM3MTM4ZGJhNzNmNmQ4ZCIsCiAgICAgICJ0eXBlIjogInJlYXNvbmluZyIsCiAgICAgICJzdW1tYXJ5IjogW10KICAgIH0sCiAgICB7CiAgICAgICJpZCI6ICJtc2dfNjg3M2UwMjJmZjNjODE5OWI3ZWEyYzYyZjhhNDcwNDUwYzcxMzhkYmE3M2Y2ZDhkIiwKICAgICAgInR5cGUiOiAibWVzc2FnZSIsCiAgICAgICJzdGF0dXMiOiAiY29tcGxldGVkIiwKICAgICAgImNvbnRlbnQiOiBbCiAgICAgICAgewogICAgICAgICAgInR5cGUiOiAib3V0cHV0X3RleHQiLAogICAgICAgICAgImFubm90YXRpb25zIjogW10sCiAgICAgICAgICAibG9ncHJvYnMiOiBbXSwKICAgICAgICAgICJ0ZXh0IjogIjIgKyAyID0gNCIKICAgICAgICB9CiAgICAgIF0sCiAgICAgICJyb2xlIjogImFzc2lzdGFudCIKICAgIH0KICBdLAogICJwYXJhbGxlbF90b29sX2NhbGxzIjogdHJ1ZSwKICAicHJldmlvdXNfcmVzcG9uc2VfaWQiOiBudWxsLAogICJyZWFzb25pbmciOiB7CiAgICAiZWZmb3J0IjogIm1lZGl1bSIsCiAgICAic3VtbWFyeSI6IG51bGwKICB9LAogICJzZXJ2aWNlX3RpZXIiOiAiZGVmYXVsdCIsCiAgInN0b3JlIjogdHJ1ZSwKICAidGVtcGVyYXR1cmUiOiAxLjAsCiAgInRleHQiOiB7CiAgICAiZm9ybWF0IjogewogICAgICAidHlwZSI6ICJ0ZXh0IgogICAgfQogIH0sCiAgInRvb2xfY2hvaWNlIjogImF1dG8iLAogICJ0b29scyI6IFtdLAogICJ0b3BfbG9ncHJvYnMiOiAwLAogICJ0b3BfcCI6IDEuMCwKICAidHJ1bmNhdGlvbiI6ICJkaXNhYmxlZCIsCiAgInVzYWdlIjogewogICAgImlucHV0X3Rva2VucyI6IDE4ODMsCiAgICAiaW5wdXRfdG9rZW5zX2RldGFpbHMiOiB7CiAgICAgICJjYWNoZWRfdG9rZW5zIjogMAogICAgfSwKICAgICJvdXRwdXRfdG9rZW5zIjogNzksCiAgICAib3V0cHV0X3Rva2Vuc19kZXRhaWxzIjogewogICAgICAicmVhc29uaW5nX3Rva2VucyI6IDY0CiAgICB9LAogICAgInRvdGFsX3Rva2VucyI6IDE5NjIKICB9LAogICJ1c2VyIjogbnVsbCwKICAibWV0YWRhdGEiOiB7fQp9", "encoding": "base64", "size": 1416 }, "headers": { "alt-svc": "h3=\":443\"; ma=86400", "cf-cache-status": "DYNAMIC", "cf-ray": "95ea300e7a8a3863-QRO", "connection": "keep-alive", "content-encoding": "gzip", "content-type": "application/json", "date": "Sun, 13 Jul 2025 16:34:43 GMT", "openai-organization": "ruin-yezxd7", "openai-processing-ms": "17597", "openai-version": "2020-10-01", "server": "cloudflare", "set-cookie": "__cf_bm=oZ3A.JEIYCcMsNAs2xtzVqODzcOPgRVQGgpQ8Qtbz.s-(XXX) XXX-XXXX-0.0.0.0-ndc7qvXE6_ceMCvd1CYBLUdvgh0lSag4KAnufbpMF1CCpHm3D_3oP8sdch_SOtunumLr53gmTqJ9JjcV..gj2AyMmLnLs2BA1S1ERg6qgAA; path=/; expires=Sun, 13-Jul-25 17:04:43 GMT; domain=.api.openai.com; HttpOnly; Secure; SameSite=None, _cfuvid=sfd47fp5T7r6zUXO0EOf5g.1CjjBZLFyzTxXBAR7F54-175(XXX) XXX-XXXX-0.0.0.0-604800000; path=/; domain=.api.openai.com; HttpOnly; Secure; SameSite=None", "strict-transport-security": "max-age=31536000; includeSubDomains; preload", "transfer-encoding": "chunked", "x-content-type-options": "nosniff", "x-ratelimit-limit-requests": "5000", "x-ratelimit-limit-tokens": "5000", "x-ratelimit-remaining-requests": "4999", "x-ratelimit-remaining-tokens": "4999", "x-ratelimit-reset-requests": "0s", "x-ratelimit-reset-tokens": "0s", "x-request-id": "req_74a7b0f6e62299fcac5c089319446a4c" }, "reason_phrase": "OK", "status_code": 200 } } ] } ================================================ FILE: tests/pii_sanitizer.py ================================================ #!/usr/bin/env python3 """ PII (Personally Identifiable Information) Sanitizer for HTTP recordings. This module provides comprehensive sanitization of sensitive data in HTTP request/response recordings to prevent accidental exposure of API keys, tokens, personal information, and other sensitive data. """ import logging import re from copy import deepcopy from dataclasses import dataclass from re import Pattern from typing import Any, Optional logger = logging.getLogger(__name__) @dataclass class PIIPattern: """Defines a pattern for detecting and sanitizing PII.""" name: str pattern: Pattern[str] replacement: str description: str @classmethod def create(cls, name: str, pattern: str, replacement: str, description: str) -> "PIIPattern": """Create a PIIPattern with compiled regex.""" return cls(name=name, pattern=re.compile(pattern), replacement=replacement, description=description) class PIISanitizer: """Sanitizes PII from various data structures while preserving format.""" def __init__(self, patterns: Optional[list[PIIPattern]] = None): """Initialize with optional custom patterns.""" self.patterns: list[PIIPattern] = patterns or [] self.sanitize_enabled = True # Add default patterns if none provided if not patterns: self._add_default_patterns() def _add_default_patterns(self): """Add comprehensive default PII patterns.""" default_patterns = [ # API Keys - Core patterns (Bearer tokens handled in sanitize_headers) PIIPattern.create( name="openai_api_key_proj", pattern=r"sk-proj-[A-Za-z0-9\-_]{48,}", replacement="sk-proj-SANITIZED", description="OpenAI project API keys", ), PIIPattern.create( name="openai_api_key", pattern=r"sk-[A-Za-z0-9]{48,}", replacement="sk-SANITIZED", description="OpenAI API keys", ), PIIPattern.create( name="anthropic_api_key", pattern=r"sk-ant-[A-Za-z0-9\-_]{48,}", replacement="sk-ant-SANITIZED", description="Anthropic API keys", ), PIIPattern.create( name="google_api_key", pattern=r"AIza[A-Za-z0-9\-_]{35,}", replacement="AIza-SANITIZED", description="Google API keys", ), PIIPattern.create( name="github_tokens", pattern=r"gh[psr]_[A-Za-z0-9]{36}", replacement="gh_SANITIZED", description="GitHub tokens (all types)", ), # JWT tokens PIIPattern.create( name="jwt_token", pattern=r"eyJ[A-Za-z0-9\-_]+\.eyJ[A-Za-z0-9\-_]+\.[A-Za-z0-9\-_]+", replacement="eyJ-SANITIZED", description="JSON Web Tokens", ), # Personal Information PIIPattern.create( name="email_address", pattern=r"[a-zA-Z0-9._%+\-]+@[a-zA-Z0-9.\-]+\.[a-zA-Z]{2,}", replacement="user@example.com", description="Email addresses", ), PIIPattern.create( name="ipv4_address", pattern=r"\b(?:(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\.){3}(?:25[0-5]|2[0-4][0-9]|[01]?[0-9][0-9]?)\b", replacement="0.0.0.0", description="IPv4 addresses", ), PIIPattern.create( name="ssn", pattern=r"\b\d{3}-\d{2}-\d{4}\b", replacement="XXX-XX-XXXX", description="Social Security Numbers", ), PIIPattern.create( name="credit_card", pattern=r"\b\d{4}[\s\-]?\d{4}[\s\-]?\d{4}[\s\-]?\d{4}\b", replacement="XXXX-XXXX-XXXX-XXXX", description="Credit card numbers", ), PIIPattern.create( name="phone_number", pattern=r"(?:\+\d{1,3}[\s\-]?)?\(?\d{3}\)?[\s\-]?\d{3}[\s\-]?\d{4}\b(?![\d\.\,\]\}])", replacement="(XXX) XXX-XXXX", description="Phone numbers (all formats)", ), # AWS PIIPattern.create( name="aws_access_key", pattern=r"AKIA[0-9A-Z]{16}", replacement="AKIA-SANITIZED", description="AWS access keys", ), # Other common patterns PIIPattern.create( name="slack_token", pattern=r"xox[baprs]-[0-9]{10,13}-[0-9]{10,13}-[a-zA-Z0-9]{24,34}", replacement="xox-SANITIZED", description="Slack tokens", ), PIIPattern.create( name="stripe_key", pattern=r"(?:sk|pk)_(?:test|live)_[0-9a-zA-Z]{24,99}", replacement="sk_SANITIZED", description="Stripe API keys", ), ] self.patterns.extend(default_patterns) def add_pattern(self, pattern: PIIPattern): """Add a custom PII pattern.""" self.patterns.append(pattern) logger.info(f"Added PII pattern: {pattern.name}") def sanitize_string(self, text: str) -> str: """Apply all patterns to sanitize a string.""" if not self.sanitize_enabled or not isinstance(text, str): return text sanitized = text for pattern in self.patterns: if pattern.pattern.search(sanitized): sanitized = pattern.pattern.sub(pattern.replacement, sanitized) logger.debug(f"Applied {pattern.name} sanitization") return sanitized def sanitize_headers(self, headers: dict[str, str]) -> dict[str, str]: """Special handling for HTTP headers.""" if not self.sanitize_enabled: return headers sanitized_headers = {} for key, value in headers.items(): # Special case for Authorization headers to preserve auth type if key.lower() == "authorization" and " " in value: auth_type = value.split(" ", 1)[0] if auth_type in ("Bearer", "Basic"): sanitized_headers[key] = f"{auth_type} SANITIZED" else: sanitized_headers[key] = self.sanitize_string(value) else: # Apply standard sanitization to all other headers sanitized_headers[key] = self.sanitize_string(value) return sanitized_headers def sanitize_value(self, value: Any) -> Any: """Recursively sanitize any value (string, dict, list, etc).""" if not self.sanitize_enabled: return value if isinstance(value, str): return self.sanitize_string(value) elif isinstance(value, dict): return {k: self.sanitize_value(v) for k, v in value.items()} elif isinstance(value, list): return [self.sanitize_value(item) for item in value] elif isinstance(value, tuple): return tuple(self.sanitize_value(item) for item in value) else: # For other types (int, float, bool, None), return as-is return value def sanitize_url(self, url: str) -> str: """Sanitize sensitive data from URLs (query params, etc).""" if not self.sanitize_enabled: return url # First apply general string sanitization url = self.sanitize_string(url) # Parse and sanitize query parameters if "?" in url: base, query = url.split("?", 1) params = [] for param in query.split("&"): if "=" in param: key, value = param.split("=", 1) # Sanitize common sensitive parameter names sensitive_params = {"key", "token", "api_key", "secret", "password"} if key.lower() in sensitive_params: params.append(f"{key}=SANITIZED") else: # Still sanitize the value for PII params.append(f"{key}={self.sanitize_string(value)}") else: params.append(param) return f"{base}?{'&'.join(params)}" return url def sanitize_request(self, request_data: dict[str, Any]) -> dict[str, Any]: """Sanitize a complete request dictionary.""" sanitized = deepcopy(request_data) # Sanitize headers if "headers" in sanitized: sanitized["headers"] = self.sanitize_headers(sanitized["headers"]) # Sanitize URL if "url" in sanitized: sanitized["url"] = self.sanitize_url(sanitized["url"]) # Sanitize content if "content" in sanitized: sanitized["content"] = self.sanitize_value(sanitized["content"]) return sanitized def sanitize_response(self, response_data: dict[str, Any]) -> dict[str, Any]: """Sanitize a complete response dictionary.""" sanitized = deepcopy(response_data) # Sanitize headers if "headers" in sanitized: sanitized["headers"] = self.sanitize_headers(sanitized["headers"]) # Sanitize content if "content" in sanitized: # Handle base64 encoded content specially if isinstance(sanitized["content"], dict) and sanitized["content"].get("encoding") == "base64": if "data" in sanitized["content"]: import base64 try: # Decode, sanitize, and re-encode the actual response body decoded_bytes = base64.b64decode(sanitized["content"]["data"]) # Attempt to decode as UTF-8 for sanitization. If it fails, it's likely binary. try: decoded_str = decoded_bytes.decode("utf-8") sanitized_str = self.sanitize_string(decoded_str) sanitized["content"]["data"] = base64.b64encode(sanitized_str.encode("utf-8")).decode( "utf-8" ) except UnicodeDecodeError: # Content is not text, leave as is. pass except (base64.binascii.Error, TypeError): # Handle cases where data is not valid base64 pass # Sanitize other metadata fields for key, value in sanitized["content"].items(): if key != "data": sanitized["content"][key] = self.sanitize_value(value) else: sanitized["content"] = self.sanitize_value(sanitized["content"]) return sanitized # Global instance for convenience default_sanitizer = PIISanitizer() ================================================ FILE: tests/sanitize_cassettes.py ================================================ #!/usr/bin/env python3 """ Script to sanitize existing cassettes by applying PII sanitization. This script will: 1. Load existing cassettes 2. Apply PII sanitization to all interactions 3. Create backups of originals 4. Save sanitized versions """ import json import shutil import sys from datetime import datetime from pathlib import Path # Add tests directory to path to import our modules sys.path.insert(0, str(Path(__file__).parent)) from pii_sanitizer import PIISanitizer def sanitize_cassette(cassette_path: Path, backup: bool = True) -> bool: """Sanitize a single cassette file.""" print(f"\n🔍 Processing: {cassette_path}") if not cassette_path.exists(): print(f"❌ File not found: {cassette_path}") return False try: # Load cassette with open(cassette_path) as f: cassette_data = json.load(f) # Create backup if requested if backup: backup_path = cassette_path.with_suffix(f'.backup-{datetime.now().strftime("%Y%m%d-%H%M%S")}.json') shutil.copy2(cassette_path, backup_path) print(f"📦 Backup created: {backup_path}") # Initialize sanitizer sanitizer = PIISanitizer() # Sanitize interactions if "interactions" in cassette_data: sanitized_interactions = [] for interaction in cassette_data["interactions"]: sanitized_interaction = {} # Sanitize request if "request" in interaction: sanitized_interaction["request"] = sanitizer.sanitize_request(interaction["request"]) # Sanitize response if "response" in interaction: sanitized_interaction["response"] = sanitizer.sanitize_response(interaction["response"]) sanitized_interactions.append(sanitized_interaction) cassette_data["interactions"] = sanitized_interactions # Save sanitized cassette with open(cassette_path, "w") as f: json.dump(cassette_data, f, indent=2, sort_keys=True) print(f"✅ Sanitized: {cassette_path}") return True except Exception as e: print(f"❌ Error processing {cassette_path}: {e}") import traceback traceback.print_exc() return False def main(): """Sanitize all cassettes in the openai_cassettes directory.""" cassettes_dir = Path(__file__).parent / "openai_cassettes" if not cassettes_dir.exists(): print(f"❌ Directory not found: {cassettes_dir}") sys.exit(1) # Find all JSON cassettes cassette_files = list(cassettes_dir.glob("*.json")) if not cassette_files: print(f"❌ No cassette files found in {cassettes_dir}") sys.exit(1) print(f"🎬 Found {len(cassette_files)} cassette(s) to sanitize") # Process each cassette success_count = 0 for cassette_path in cassette_files: if sanitize_cassette(cassette_path): success_count += 1 print(f"\n✨ Sanitization complete: {success_count}/{len(cassette_files)} cassettes processed successfully") if success_count < len(cassette_files): sys.exit(1) if __name__ == "__main__": main() ================================================ FILE: tests/test_alias_target_restrictions.py ================================================ """ Tests for alias and target model restriction validation. This test suite ensures that the restriction service properly validates both alias names and their target models, preventing policy bypass vulnerabilities. """ import os from unittest.mock import patch from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.shared import ProviderType from utils.model_restrictions import ModelRestrictionService class TestAliasTargetRestrictions: """Test that restriction validation works for both aliases and their targets.""" def test_openai_alias_target_validation_comprehensive(self): """Test OpenAI provider includes both aliases and targets in validation.""" provider = OpenAIModelProvider(api_key="test-key") # Get all known models including aliases and targets all_known = provider.list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) # Should include both aliases and their targets assert "mini" in all_known # alias assert "o4-mini" in all_known # target of 'mini' assert "o3mini" in all_known # alias assert "o3-mini" in all_known # target of 'o3mini' def test_gemini_alias_target_validation_comprehensive(self): """Test Gemini provider includes both aliases and targets in validation.""" provider = GeminiModelProvider(api_key="test-key") # Get all known models including aliases and targets all_known = provider.list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) # Should include both aliases and their targets assert "flash" in all_known # alias assert "gemini-2.5-flash" in all_known # target of 'flash' assert "pro" in all_known # alias assert "gemini-2.5-pro" in all_known # target of 'pro' @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini"}) # Allow target def test_restriction_policy_allows_alias_when_target_allowed(self): """Test that restriction policy allows alias when target model is allowed. This is the correct user-friendly behavior - if you allow 'o4-mini', you should be able to use its aliases 'o4mini' and 'o4-mini'. Note: 'mini' is now an alias for 'gpt-5-mini', not 'o4-mini'. """ # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") # Both target and its actual aliases should be allowed assert provider.validate_model_name("o4-mini") assert provider.validate_model_name("o4mini") @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "mini"}) # Allow alias only def test_restriction_policy_alias_allows_canonical(self): """Alias-only allowlists should permit both the alias and its canonical target.""" import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") assert provider.validate_model_name("mini") assert provider.validate_model_name("gpt-5-mini") assert not provider.validate_model_name("o4-mini") @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "gpt5"}) def test_restriction_policy_alias_allows_short_name(self): """Common aliases like 'gpt5' should allow their canonical forms.""" import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") assert provider.validate_model_name("gpt5") assert provider.validate_model_name("gpt-5") @patch.dict(os.environ, {"GOOGLE_ALLOWED_MODELS": "gemini-2.5-flash"}) # Allow target def test_gemini_restriction_policy_allows_alias_when_target_allowed(self): """Test Gemini restriction policy allows alias when target is allowed.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = GeminiModelProvider(api_key="test-key") # Both target and alias should be allowed assert provider.validate_model_name("gemini-2.5-flash") assert provider.validate_model_name("flash") @patch.dict(os.environ, {"GOOGLE_ALLOWED_MODELS": "flash"}) # Allow alias only def test_gemini_restriction_policy_alias_allows_canonical(self): """Gemini alias allowlists should permit canonical forms.""" import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = GeminiModelProvider(api_key="test-key") assert provider.validate_model_name("flash") assert provider.validate_model_name("gemini-2.5-flash") def test_restriction_service_validation_includes_all_targets(self): """Test that restriction service validation knows about all aliases and targets.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini,invalid-model"}): service = ModelRestrictionService() # Create real provider instances provider_instances = {ProviderType.OPENAI: OpenAIModelProvider(api_key="test-key")} # Capture warnings with patch("utils.model_restrictions.logger") as mock_logger: service.validate_against_known_models(provider_instances) # Should have warned about the invalid model warning_calls = [call for call in mock_logger.warning.call_args_list if "invalid-model" in str(call)] assert len(warning_calls) > 0, "Should have warned about invalid-model" # The warning should include both aliases and targets in known models warning_message = str(warning_calls[0]) assert "o4mini" in warning_message or "o4-mini" in warning_message # aliases should be in known models @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "mini,gpt-5-mini,o4-mini,o4mini"}) # Allow different models def test_both_alias_and_target_allowed_when_both_specified(self): """Test that both alias and target work when both are explicitly allowed. mini -> gpt-5-mini o4mini -> o4-mini """ # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") # All should be allowed since we explicitly allowed them assert provider.validate_model_name("mini") # alias for gpt-5-mini assert provider.validate_model_name("gpt-5-mini") # target assert provider.validate_model_name("o4-mini") # target assert provider.validate_model_name("o4mini") # alias for o4-mini @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "gpt5"}, clear=True) def test_service_alias_allows_canonical_openai(self): """ModelRestrictionService should permit canonical names resolved from aliases.""" import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") service = ModelRestrictionService() assert service.is_allowed(ProviderType.OPENAI, "gpt-5") assert provider.validate_model_name("gpt-5") @patch.dict(os.environ, {"GOOGLE_ALLOWED_MODELS": "flash"}, clear=True) def test_service_alias_allows_canonical_gemini(self): """Gemini alias allowlists should permit canonical forms.""" import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = GeminiModelProvider(api_key="test-key") service = ModelRestrictionService() assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-flash") assert provider.validate_model_name("gemini-2.5-flash") def test_alias_target_policy_regression_prevention(self): """Regression test to ensure aliases and targets are both validated properly. This test specifically prevents the bug where list_models() only returned aliases but not their targets, causing restriction validation to miss deny-list entries for target models. """ # Test OpenAI provider openai_provider = OpenAIModelProvider(api_key="test-key") openai_all_known = openai_provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True ) # Verify that for each alias, its target is also included for model_name, config in openai_provider.MODEL_CAPABILITIES.items(): assert model_name.lower() in openai_all_known if isinstance(config, str): # This is an alias # The target should also be in the known models assert ( config.lower() in openai_all_known ), f"Target '{config}' for alias '{model_name}' not in known models" # Test Gemini provider gemini_provider = GeminiModelProvider(api_key="test-key") gemini_all_known = gemini_provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True ) # Verify that for each alias, its target is also included for model_name, config in gemini_provider.MODEL_CAPABILITIES.items(): assert model_name.lower() in gemini_all_known if isinstance(config, str): # This is an alias # The target should also be in the known models assert ( config.lower() in gemini_all_known ), f"Target '{config}' for alias '{model_name}' not in known models" def test_no_duplicate_models_in_alias_aware_listing(self): """Test that alias-aware list_models variant doesn't return duplicates.""" # Test all providers providers = [ OpenAIModelProvider(api_key="test-key"), GeminiModelProvider(api_key="test-key"), ] for provider in providers: all_known = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True ) # Should not have duplicates assert len(all_known) == len(set(all_known)), f"{provider.__class__.__name__} returns duplicate models" def test_restriction_validation_uses_polymorphic_interface(self): """Test that restriction validation uses the clean polymorphic interface.""" service = ModelRestrictionService() # Create a mock provider from unittest.mock import MagicMock mock_provider = MagicMock() mock_provider.list_models.return_value = ["model1", "model2", "target-model"] # Set up a restriction that should trigger validation service.restrictions = {ProviderType.OPENAI: {"invalid-model"}} provider_instances = {ProviderType.OPENAI: mock_provider} # Should call the polymorphic method service.validate_against_known_models(provider_instances) # Verify the polymorphic method was called mock_provider.list_models.assert_called_once_with( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini"}) # Restrict to specific model def test_complex_alias_chains_handled_correctly(self): """Test that complex alias chains are handled correctly in restrictions.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") # Only o4-mini should be allowed assert provider.validate_model_name("o4-mini") # Other models should be blocked assert not provider.validate_model_name("o3") assert not provider.validate_model_name("o3-mini") def test_critical_regression_validation_sees_alias_targets(self): """CRITICAL REGRESSION TEST: Ensure validation can see alias target models. This test prevents the specific bug where list_models() only returned alias keys but not their targets, causing validate_against_known_models() to miss restrictions on target model names. Before the fix: - list_models() returned ["mini", "o3mini"] (aliases only) - validate_against_known_models() only checked against ["mini", "o3mini"] - A restriction on "o4-mini" (target) would not be recognized as valid After the fix: - list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) returns ["mini", "o3mini", "o4-mini", "o3-mini"] (aliases + targets) - validate_against_known_models() checks against all names - A restriction on "o4-mini" is recognized as valid """ # This test specifically validates the HIGH-severity bug that was found service = ModelRestrictionService() # Create provider instance provider = OpenAIModelProvider(api_key="test-key") provider_instances = {ProviderType.OPENAI: provider} # Get all known models - should include BOTH aliases AND targets all_known = provider.list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) # Critical check: should contain both aliases and their targets assert "mini" in all_known # alias assert "o4-mini" in all_known # target of mini - THIS WAS MISSING BEFORE assert "o3mini" in all_known # alias assert "o3-mini" in all_known # target of o3mini - THIS WAS MISSING BEFORE # Simulate restriction validation with a target model name # This should NOT warn because "o4-mini" is a valid target with patch("utils.model_restrictions.logger") as mock_logger: # Set restriction to target model (not alias) service.restrictions = {ProviderType.OPENAI: {"o4-mini"}} # This should NOT generate warnings because o4-mini is known service.validate_against_known_models(provider_instances) # Should NOT have any warnings about o4-mini being unrecognized warning_calls = [ call for call in mock_logger.warning.call_args_list if "o4-mini" in str(call) and "not a recognized" in str(call) ] assert len(warning_calls) == 0, "o4-mini should be recognized as valid target model" # Test the reverse: alias in restriction should also be recognized with patch("utils.model_restrictions.logger") as mock_logger: # Set restriction to alias name service.restrictions = {ProviderType.OPENAI: {"mini"}} # This should NOT generate warnings because mini is known service.validate_against_known_models(provider_instances) # Should NOT have any warnings about mini being unrecognized warning_calls = [ call for call in mock_logger.warning.call_args_list if "mini" in str(call) and "not a recognized" in str(call) ] assert len(warning_calls) == 0, "mini should be recognized as valid alias" def test_critical_regression_prevents_policy_bypass(self): """CRITICAL REGRESSION TEST: Prevent policy bypass through missing target validation. This test ensures that if an admin restricts access to a target model name, the restriction is properly enforced and the target is recognized as a valid model to restrict. The bug: If list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) doesn't include targets, then validation would incorrectly warn that target model names are "not recognized", making it appear that target-based restrictions don't work. """ # Test with a made-up restriction scenario with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini,o3-mini"}): # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None service = ModelRestrictionService() provider = OpenAIModelProvider(api_key="test-key") # These specific target models should be recognized as valid all_known = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True ) assert "o4-mini" in all_known, "Target model o4-mini should be known" assert "o3-mini" in all_known, "Target model o3-mini should be known" # Validation should not warn about these being unrecognized with patch("utils.model_restrictions.logger") as mock_logger: provider_instances = {ProviderType.OPENAI: provider} service.validate_against_known_models(provider_instances) # Should not warn about our allowed models being unrecognized all_warnings = [str(call) for call in mock_logger.warning.call_args_list] for warning in all_warnings: assert "o4-mini" not in warning or "not a recognized" not in warning assert "o3-mini" not in warning or "not a recognized" not in warning # The restriction should actually work assert provider.validate_model_name("o4-mini") assert provider.validate_model_name("o3-mini") assert not provider.validate_model_name("o3-pro") # not in allowed list assert not provider.validate_model_name("o3") # not in allowed list ================================================ FILE: tests/test_auto_mode.py ================================================ """Tests for auto mode functionality""" import importlib import os from unittest.mock import patch import pytest from tools.chat import ChatTool from tools.shared.exceptions import ToolExecutionError class TestAutoMode: """Test auto mode configuration and behavior""" def test_auto_mode_detection(self): """Test that auto mode is detected correctly""" # Save original original = os.environ.get("DEFAULT_MODEL", "") try: # Test auto mode os.environ["DEFAULT_MODEL"] = "auto" import config importlib.reload(config) assert config.DEFAULT_MODEL == "auto" assert config.IS_AUTO_MODE is True # Test non-auto mode os.environ["DEFAULT_MODEL"] = "pro" importlib.reload(config) assert config.DEFAULT_MODEL == "pro" assert config.IS_AUTO_MODE is False finally: # Restore if original: os.environ["DEFAULT_MODEL"] = original else: os.environ.pop("DEFAULT_MODEL", None) importlib.reload(config) def test_model_capabilities_descriptions(self): """Test that model capabilities are properly defined in providers""" from providers.registry import ModelProviderRegistry # Get all providers with valid API keys and check their model descriptions enabled_provider_types = ModelProviderRegistry.get_available_providers_with_keys() models_with_descriptions = {} for provider_type in enabled_provider_types: provider = ModelProviderRegistry.get_provider(provider_type) if provider: for model_name, config in provider.MODEL_CAPABILITIES.items(): # Skip alias entries (string values) if isinstance(config, str): continue # Check that model has description description = config.description if hasattr(config, "description") else "" if description: models_with_descriptions[model_name] = description # Check all expected models are present with meaningful descriptions expected_models = ["flash", "pro", "o3", "o3-mini", "o3-pro", "o4-mini"] for model in expected_models: # Model should exist somewhere in the providers # Note: Some models might not be available if API keys aren't configured if model in models_with_descriptions: assert isinstance(models_with_descriptions[model], str) assert len(models_with_descriptions[model]) > 50 # Meaningful description def test_tool_schema_in_auto_mode(self): """Test that tool schemas require model in auto mode""" # Save original original = os.environ.get("DEFAULT_MODEL", "") try: # Enable auto mode os.environ["DEFAULT_MODEL"] = "auto" import config importlib.reload(config) tool = ChatTool() schema = tool.get_input_schema() # Model should be required assert "model" in schema["required"] # Model field should have detailed descriptions model_schema = schema["properties"]["model"] assert "enum" not in model_schema assert "auto mode" in model_schema["description"].lower() assert "listmodels" in model_schema["description"] finally: # Restore if original: os.environ["DEFAULT_MODEL"] = original else: os.environ.pop("DEFAULT_MODEL", None) importlib.reload(config) def test_tool_schema_in_normal_mode(self): """Test that tool schemas don't require model in normal mode""" # Save original original = os.environ.get("DEFAULT_MODEL", "") try: # Set to a specific model (not auto mode) os.environ["DEFAULT_MODEL"] = "gemini-2.5-flash" import config importlib.reload(config) tool = ChatTool() schema = tool.get_input_schema() # Model should not be required when default model is configured assert "model" not in schema["required"] # Model field should have simpler description model_schema = schema["properties"]["model"] assert "enum" not in model_schema assert "listmodels" in model_schema["description"] assert "default model" in model_schema["description"].lower() finally: # Restore if original: os.environ["DEFAULT_MODEL"] = original else: os.environ.pop("DEFAULT_MODEL", None) importlib.reload(config) @pytest.mark.asyncio async def test_auto_mode_requires_model_parameter(self, tmp_path): """Test that auto mode enforces model parameter""" # Save original original = os.environ.get("DEFAULT_MODEL", "") try: # Enable auto mode os.environ["DEFAULT_MODEL"] = "auto" import config importlib.reload(config) tool = ChatTool() # Mock the provider to avoid real API calls with patch.object(tool, "get_model_provider"): # Execute without model parameter and expect protocol error with pytest.raises(ToolExecutionError) as exc_info: await tool.execute({"prompt": "Test prompt", "working_directory_absolute_path": str(tmp_path)}) # Should get error payload mentioning model requirement error_payload = getattr(exc_info.value, "payload", str(exc_info.value)) assert "Model" in error_payload assert "auto" in error_payload finally: # Restore if original: os.environ["DEFAULT_MODEL"] = original else: os.environ.pop("DEFAULT_MODEL", None) importlib.reload(config) @pytest.mark.asyncio async def test_unavailable_model_error_message(self): """Test that unavailable model shows helpful error with available models using real integration testing""" # Save original environment original_env = {} api_keys = ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"] for key in api_keys: original_env[key] = os.environ.get(key) original_default = os.environ.get("DEFAULT_MODEL", "") try: # Set up environment with a real API key but test an unavailable model # This simulates a user trying to use a model that's not available with their current setup os.environ["OPENAI_API_KEY"] = "sk-test-key-unavailable-model-test-not-real" os.environ["DEFAULT_MODEL"] = "auto" # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and registry to pick up new environment import config importlib.reload(config) # Clear registry singleton to force re-initialization with new environment from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = ChatTool() # Test with real provider resolution - this should attempt to use a model # that doesn't exist in the OpenAI provider's model list try: result = await tool.execute( { "absolute_file_paths": ["/tmp/test.py"], "prompt": "Analyze this", "model": "nonexistent-model-xyz", # This model definitely doesn't exist } ) # If we get here, check that it's an error about model availability assert len(result) == 1 response = result[0].text assert "error" in response # Should be about model not being available assert any( phrase in response for phrase in [ "Model 'nonexistent-model-xyz' is not available", "No provider found", "not available", "not supported", ] ) except Exception as e: # Expected: Should fail with provider resolution or model validation error error_msg = str(e) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error about model not being available assert any( phrase in error_msg for phrase in [ "Model 'nonexistent-model-xyz'", "not available", "not found", "not supported", "provider", "model", ] ) or any(phrase in error_msg for phrase in ["API", "key", "authentication", "network", "connection"]) finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) if original_default: os.environ["DEFAULT_MODEL"] = original_default else: os.environ.pop("DEFAULT_MODEL", None) # Reload config and clear registry singleton importlib.reload(config) ModelProviderRegistry._instance = None def test_model_field_schema_generation(self): """Test the get_model_field_schema method""" from tools.shared.base_tool import BaseTool # Create a minimal concrete tool for testing class TestTool(BaseTool): def get_name(self): return "test" def get_description(self): return "test" def get_input_schema(self): return {} def get_system_prompt(self): return "" def get_request_model(self): return None async def prepare_prompt(self, request): return "" tool = TestTool() # Save original original = os.environ.get("DEFAULT_MODEL", "") try: # Test auto mode os.environ["DEFAULT_MODEL"] = "auto" import config importlib.reload(config) schema = tool.get_model_field_schema() assert "enum" not in schema assert schema["type"] == "string" assert "auto mode" in schema["description"].lower() assert "listmodels" in schema["description"] # Test normal mode os.environ["DEFAULT_MODEL"] = "pro" importlib.reload(config) schema = tool.get_model_field_schema() assert "enum" not in schema assert schema["type"] == "string" assert "'pro'" in schema["description"] assert "listmodels" in schema["description"] finally: # Restore if original: os.environ["DEFAULT_MODEL"] = original else: os.environ.pop("DEFAULT_MODEL", None) importlib.reload(config) ================================================ FILE: tests/test_auto_mode_comprehensive.py ================================================ """Comprehensive tests for auto mode functionality across all provider combinations""" import importlib import os from unittest.mock import MagicMock, patch import pytest from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from providers.xai import XAIModelProvider from tools.analyze import AnalyzeTool from tools.chat import ChatTool from tools.debug import DebugIssueTool from tools.models import ToolModelCategory from tools.shared.exceptions import ToolExecutionError from tools.thinkdeep import ThinkDeepTool @pytest.mark.no_mock_provider class TestAutoModeComprehensive: """Test auto mode model selection across all provider combinations""" def setup_method(self): """Set up clean state before each test.""" # Save original environment state for restoration import os self._original_default_model = os.environ.get("DEFAULT_MODEL", "") # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry by resetting singleton instance ModelProviderRegistry._instance = None def teardown_method(self): """Clean up after each test.""" # Restore original DEFAULT_MODEL import os if self._original_default_model: os.environ["DEFAULT_MODEL"] = self._original_default_model elif "DEFAULT_MODEL" in os.environ: del os.environ["DEFAULT_MODEL"] # Reload config to pick up the restored DEFAULT_MODEL import importlib import config importlib.reload(config) # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry by resetting singleton instance ModelProviderRegistry._instance = None # Re-register providers for subsequent tests (like conftest.py does) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) @pytest.mark.parametrize( "provider_config,expected_models", [ # Only Gemini API available ( { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": None, "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, }, { "EXTENDED_REASONING": "gemini-3-pro-preview", # Gemini 3 Pro Preview for deep thinking "FAST_RESPONSE": "gemini-2.5-flash", # Flash for speed "BALANCED": "gemini-2.5-flash", # Flash as balanced }, ), # Only OpenAI API available ( { "GEMINI_API_KEY": None, "OPENAI_API_KEY": "real-key", "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, }, { "EXTENDED_REASONING": "gpt-5.1-codex", # GPT-5.1 Codex prioritized for coding tasks "FAST_RESPONSE": "gpt-5.2", # Prefer gpt-5.2 for speed "BALANCED": "gpt-5.2", # Prefer gpt-5.2 for balanced }, ), # Only X.AI API available ( { "GEMINI_API_KEY": None, "OPENAI_API_KEY": None, "XAI_API_KEY": "real-key", "OPENROUTER_API_KEY": None, }, { "EXTENDED_REASONING": "grok-4-1-fast-reasoning", # Latest Grok 4.1 Fast Reasoning "FAST_RESPONSE": "grok-4-1-fast-reasoning", # Latest fast SKU "BALANCED": "grok-4-1-fast-reasoning", # Latest balanced default }, ), # Both Gemini and OpenAI available - Google comes first in priority ( { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": "real-key", "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, }, { "EXTENDED_REASONING": "gemini-3-pro-preview", # Gemini 3 Pro Preview comes first in priority "FAST_RESPONSE": "gemini-2.5-flash", # Prefer flash for speed "BALANCED": "gemini-2.5-flash", # Prefer flash for balanced }, ), # All native APIs available - Google still comes first ( { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": "real-key", "XAI_API_KEY": "real-key", "OPENROUTER_API_KEY": None, }, { "EXTENDED_REASONING": "gemini-3-pro-preview", # Gemini 3 Pro Preview comes first in priority "FAST_RESPONSE": "gemini-2.5-flash", # Prefer flash for speed "BALANCED": "gemini-2.5-flash", # Prefer flash for balanced }, ), ], ) def test_auto_mode_model_selection_by_provider(self, provider_config, expected_models): """Test that auto mode selects correct models based on available providers.""" # Set up environment with specific provider configuration # Filter out None values and handle them separately env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] # Reload config to pick up auto mode os.environ["DEFAULT_MODEL"] = "auto" import config importlib.reload(config) # Register providers based on configuration from providers.openrouter import OpenRouterProvider if provider_config.get("GEMINI_API_KEY"): ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) if provider_config.get("OPENAI_API_KEY"): ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) if provider_config.get("XAI_API_KEY"): ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) if provider_config.get("OPENROUTER_API_KEY"): ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Test each tool category for category_name, expected_model in expected_models.items(): category = ToolModelCategory(category_name.lower()) # Get preferred fallback model for this category fallback_model = ModelProviderRegistry.get_preferred_fallback_model(category) assert fallback_model == expected_model, ( f"Provider config {provider_config}: " f"Expected {expected_model} for {category_name}, got {fallback_model}" ) @pytest.mark.parametrize( "tool_class,expected_category", [ (ChatTool, ToolModelCategory.FAST_RESPONSE), (AnalyzeTool, ToolModelCategory.EXTENDED_REASONING), # AnalyzeTool uses EXTENDED_REASONING (DebugIssueTool, ToolModelCategory.EXTENDED_REASONING), (ThinkDeepTool, ToolModelCategory.EXTENDED_REASONING), ], ) def test_tool_model_categories(self, tool_class, expected_category): """Test that tools have the correct model categories.""" tool = tool_class() assert tool.get_model_category() == expected_category @pytest.mark.asyncio async def test_auto_mode_with_gemini_only_uses_correct_models(self, tmp_path): """Test that auto mode with only Gemini uses flash for fast tools and pro for reasoning tools.""" provider_config = { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": None, "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, "DEFAULT_MODEL": "auto", } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Register only Gemini provider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) # Test ChatTool (FAST_RESPONSE) - auto mode should suggest flash variant chat_tool = ChatTool() chat_message = chat_tool._build_auto_mode_required_message() assert "flash" in chat_message # Test DebugIssueTool (EXTENDED_REASONING) - auto mode should suggest pro variant debug_tool = DebugIssueTool() debug_message = debug_tool._build_auto_mode_required_message() assert "pro" in debug_message def test_auto_mode_schema_includes_all_available_models(self): """Test that auto mode schema includes all available models for user convenience.""" # Test with only Gemini available provider_config = { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": None, "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, "CUSTOM_API_URL": None, "DEFAULT_MODEL": "auto", } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Register only Gemini provider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) tool = AnalyzeTool() schema = tool.get_input_schema() # Should have model as required field assert "model" in schema["required"] # In auto mode, the schema should now have a description field # instructing users to use the listmodels tool instead of an enum model_schema = schema["properties"]["model"] assert "type" in model_schema assert model_schema["type"] == "string" assert "description" in model_schema # Check that the description mentions using listmodels tool description = model_schema["description"] assert "listmodels" in description.lower() assert "auto" in description.lower() or "selection" in description.lower() # Should NOT have enum field anymore - this is the new behavior assert "enum" not in model_schema # After the design change, the system directs users to use listmodels # instead of enumerating all models in the schema # This prevents model namespace collisions and keeps the schema cleaner # With the new design change, we no longer enumerate models in the schema # The listmodels tool should be used to discover available models # This test now validates the schema structure rather than model enumeration def test_auto_mode_schema_with_all_providers(self): """Test that auto mode schema includes models from all available providers.""" provider_config = { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": "real-key", "XAI_API_KEY": "real-key", "OPENROUTER_API_KEY": None, # Don't include OpenRouter to avoid infinite models "DEFAULT_MODEL": "auto", } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Register all native providers ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) tool = AnalyzeTool() schema = tool.get_input_schema() # In auto mode with multiple providers, should still use the new schema format model_schema = schema["properties"]["model"] assert "type" in model_schema assert model_schema["type"] == "string" assert "description" in model_schema # Check that the description mentions using listmodels tool description = model_schema["description"] assert "listmodels" in description.lower() # Should NOT have enum field - uses listmodels tool instead assert "enum" not in model_schema # With multiple providers configured, the listmodels tool # would show models from all providers when called @pytest.mark.asyncio async def test_auto_mode_model_parameter_required_error(self, tmp_path): """Test that auto mode properly requires model parameter and suggests correct model.""" provider_config = { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": None, "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, "DEFAULT_MODEL": "auto", } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Register only Gemini provider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) # Test with ChatTool (FAST_RESPONSE category) chat_tool = ChatTool() workdir = tmp_path / "chat_artifacts" workdir.mkdir(parents=True, exist_ok=True) with pytest.raises(ToolExecutionError) as exc_info: await chat_tool.execute( { "prompt": "test", "working_directory_absolute_path": str(workdir), # Note: no model parameter provided in auto mode } ) # Should get error requiring model selection with fallback suggestion import json response_data = json.loads(exc_info.value.payload) assert response_data["status"] == "error" assert ( "Model parameter is required" in response_data["content"] or "Model 'auto'" in response_data["content"] ) assert "flash" in response_data["content"] def test_model_availability_with_restrictions(self): """Test that auto mode respects model restrictions when selecting fallback models.""" provider_config = { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": "real-key", "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, "DEFAULT_MODEL": "auto", "OPENAI_ALLOWED_MODELS": "o4-mini", # Restrict OpenAI to only o4-mini } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Clear restriction service to pick up new env vars import utils.model_restrictions utils.model_restrictions._restriction_service = None # Register providers ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Get available models - should respect restrictions available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) # Should include restricted OpenAI model assert "o4-mini" in available_models # Should NOT include non-restricted OpenAI models assert "o3" not in available_models assert "o3-mini" not in available_models # Should still include all Gemini models (no restrictions) assert "gemini-2.5-flash" in available_models assert "gemini-2.5-pro" in available_models def test_openrouter_fallback_when_no_native_apis(self): """Test that OpenRouter provides fallback models when no native APIs are available.""" provider_config = { "GEMINI_API_KEY": None, "OPENAI_API_KEY": None, "XAI_API_KEY": None, "OPENROUTER_API_KEY": "real-key", "DEFAULT_MODEL": "auto", } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Register only OpenRouter provider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Mock OpenRouter registry to return known models mock_registry = MagicMock() mock_registry.list_models.return_value = [ "google/gemini-2.5-flash", "google/gemini-2.5-pro", "openai/o3", "openai/o4-mini", "anthropic/claude-opus-4", ] with patch.object(OpenRouterProvider, "_registry", mock_registry): # Get preferred models for different categories extended_reasoning = ModelProviderRegistry.get_preferred_fallback_model( ToolModelCategory.EXTENDED_REASONING ) fast_response = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) # Should fallback to known good models even via OpenRouter # The exact model depends on _find_extended_thinking_model implementation assert extended_reasoning is not None assert fast_response is not None @pytest.mark.asyncio async def test_actual_model_name_resolution_in_auto_mode(self, tmp_path): """Test that when a model is selected in auto mode, the tool executes successfully.""" provider_config = { "GEMINI_API_KEY": "real-key", "OPENAI_API_KEY": None, "XAI_API_KEY": None, "OPENROUTER_API_KEY": None, "DEFAULT_MODEL": "auto", } # Filter out None values to avoid patch.dict errors env_to_set = {k: v for k, v in provider_config.items() if v is not None} env_to_clear = [k for k, v in provider_config.items() if v is None] with patch.dict(os.environ, env_to_set, clear=False): # Clear the None-valued environment variables for key in env_to_clear: if key in os.environ: del os.environ[key] import config importlib.reload(config) # Register Gemini provider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) # Mock the actual provider to simulate successful execution mock_provider = MagicMock() mock_response = MagicMock() mock_response.content = "test response" mock_response.model_name = "gemini-2.5-flash" # The resolved name mock_response.usage = {"input_tokens": 10, "output_tokens": 5} # Mock _resolve_model_name to simulate alias resolution mock_provider._resolve_model_name = lambda alias: ("gemini-2.5-flash" if alias == "flash" else alias) mock_provider.generate_content.return_value = mock_response with patch.object(ModelProviderRegistry, "get_provider_for_model", return_value=mock_provider): chat_tool = ChatTool() workdir = tmp_path / "chat_artifacts" workdir.mkdir(parents=True, exist_ok=True) result = await chat_tool.execute( {"prompt": "test", "model": "flash", "working_directory_absolute_path": str(workdir)} ) # Use alias in auto mode # Should succeed with proper model resolution assert len(result) == 1 # Just verify that the tool executed successfully and didn't return an error assert "error" not in result[0].text.lower() ================================================ FILE: tests/test_auto_mode_custom_provider_only.py ================================================ """Test auto mode with only custom provider configured to reproduce the reported issue.""" import importlib import os from unittest.mock import patch import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType @pytest.mark.no_mock_provider class TestAutoModeCustomProviderOnly: """Test auto mode when only custom provider is configured.""" def setup_method(self): """Set up clean state before each test.""" # Save original environment state for restoration self._original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "CUSTOM_API_URL", "CUSTOM_API_KEY", "DEFAULT_MODEL", ]: self._original_env[key] = os.environ.get(key) # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry by resetting singleton instance ModelProviderRegistry._instance = None def teardown_method(self): """Clean up after each test.""" # Restore original environment for key, value in self._original_env.items(): if value is not None: os.environ[key] = value elif key in os.environ: del os.environ[key] # Reload config to pick up the restored environment import config importlib.reload(config) # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry by resetting singleton instance ModelProviderRegistry._instance = None def test_reproduce_auto_mode_custom_provider_only_issue(self): """Test the fix for auto mode failing when only custom provider is configured.""" # Set up environment with ONLY custom provider configured test_env = { "CUSTOM_API_URL": "http://localhost:11434/v1", "CUSTOM_API_KEY": "", # Empty for Ollama-style "DEFAULT_MODEL": "auto", } # Clear all other provider keys clear_keys = ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "DIAL_API_KEY"] with patch.dict(os.environ, test_env, clear=False): # Ensure other provider keys are not set for key in clear_keys: if key in os.environ: del os.environ[key] # Reload config to pick up auto mode import config importlib.reload(config) # Register only the custom provider (simulating server startup) from providers.custom import CustomProvider ModelProviderRegistry.register_provider(ProviderType.CUSTOM, CustomProvider) # This should now work after the fix # The fix added support for custom provider registry system in get_available_models() available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) # This assertion should now pass after the fix assert available_models, ( "Expected custom provider models to be available. " "This test verifies the fix for auto mode failing with custom providers." ) def test_custom_provider_models_available_via_registry(self): """Test that custom provider has models available via its registry system.""" # Set up environment with only custom provider test_env = { "CUSTOM_API_URL": "http://localhost:11434/v1", "CUSTOM_API_KEY": "", } with patch.dict(os.environ, test_env, clear=False): # Clear other provider keys for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "DIAL_API_KEY"]: if key in os.environ: del os.environ[key] # Register custom provider from providers.custom import CustomProvider ModelProviderRegistry.register_provider(ProviderType.CUSTOM, CustomProvider) # Get the provider instance custom_provider = ModelProviderRegistry.get_provider(ProviderType.CUSTOM) assert custom_provider is not None, "Custom provider should be available" # Verify it has a registry with models assert hasattr(custom_provider, "_registry"), "Custom provider should have _registry" assert custom_provider._registry is not None, "Registry should be initialized" # Get models from registry models = custom_provider._registry.list_models() aliases = custom_provider._registry.list_aliases() # Should have some models and aliases available assert models, "Custom provider registry should have models" assert aliases, "Custom provider registry should have aliases" print(f"Available models: {len(models)}") print(f"Available aliases: {len(aliases)}") def test_custom_provider_validate_model_name(self): """Test that custom provider can validate model names.""" # Set up environment with only custom provider test_env = { "CUSTOM_API_URL": "http://localhost:11434/v1", "CUSTOM_API_KEY": "", } with patch.dict(os.environ, test_env, clear=False): # Register custom provider from providers.custom import CustomProvider ModelProviderRegistry.register_provider(ProviderType.CUSTOM, CustomProvider) # Get the provider instance custom_provider = ModelProviderRegistry.get_provider(ProviderType.CUSTOM) assert custom_provider is not None # Test that it can validate some typical custom model names test_models = ["llama3.2", "llama3.2:latest", "local-model", "ollama-model"] for model in test_models: is_valid = custom_provider.validate_model_name(model) print(f"Model '{model}' validation: {is_valid}") # Should validate at least some local-style models # (The exact validation logic may vary based on registry content) def test_auto_mode_fallback_with_custom_only_should_work(self): """Test that auto mode fallback should work when only custom provider is available.""" # Set up environment with only custom provider test_env = { "CUSTOM_API_URL": "http://localhost:11434/v1", "CUSTOM_API_KEY": "", "DEFAULT_MODEL": "auto", } with patch.dict(os.environ, test_env, clear=False): # Clear other provider keys for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "DIAL_API_KEY"]: if key in os.environ: del os.environ[key] # Reload config import config importlib.reload(config) # Register custom provider from providers.custom import CustomProvider ModelProviderRegistry.register_provider(ProviderType.CUSTOM, CustomProvider) # This should work and return a fallback model from custom provider # Currently fails because get_preferred_fallback_model doesn't consider custom models from tools.models import ToolModelCategory try: fallback_model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) print(f"Fallback model for FAST_RESPONSE: {fallback_model}") # Should get a valid model name, not the hardcoded fallback assert ( fallback_model != "gemini-2.5-flash" ), "Should not fallback to hardcoded Gemini model when custom provider is available" except Exception as e: pytest.fail(f"Getting fallback model failed: {e}") ================================================ FILE: tests/test_auto_mode_model_listing.py ================================================ """Tests covering model restriction-aware error messaging in auto mode.""" import asyncio import importlib import json import pytest import utils.env as env_config import utils.model_restrictions as model_restrictions from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.openrouter import OpenRouterProvider from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from providers.xai import XAIModelProvider from tools.shared.exceptions import ToolExecutionError def _extract_available_models(message: str) -> list[str]: """Parse the available model list from the error message.""" marker = "Available models: " if marker not in message: raise AssertionError(f"Expected '{marker}' in message: {message}") start = message.index(marker) + len(marker) end = message.find(". Suggested", start) if end == -1: end = len(message) available_segment = message[start:end].strip() if not available_segment: return [] return [item.strip() for item in available_segment.split(",")] @pytest.fixture def reset_registry(): """Ensure registry and restriction service state is isolated.""" ModelProviderRegistry.reset_for_testing() model_restrictions._restriction_service = None env_config.reload_env() yield ModelProviderRegistry.reset_for_testing() model_restrictions._restriction_service = None def _register_core_providers(*, include_xai: bool = False): ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) if include_xai: ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) @pytest.mark.no_mock_provider def test_error_listing_respects_env_restrictions(monkeypatch, reset_registry): """Error payload should surface only the allowed models for each provider.""" monkeypatch.setenv("DEFAULT_MODEL", "auto") monkeypatch.setenv("GEMINI_API_KEY", "test-gemini") monkeypatch.setenv("OPENAI_API_KEY", "test-openai") monkeypatch.setenv("OPENROUTER_API_KEY", "test-openrouter") monkeypatch.delenv("XAI_API_KEY", raising=False) # Ensure Azure provider stays disabled regardless of developer workstation env for azure_var in ( "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_ALLOWED_MODELS", "AZURE_MODELS_CONFIG_PATH", ): monkeypatch.delenv(azure_var, raising=False) monkeypatch.setenv("PAL_MCP_FORCE_ENV_OVERRIDE", "false") env_config.reload_env({"PAL_MCP_FORCE_ENV_OVERRIDE": "false"}) try: import dotenv monkeypatch.setattr(dotenv, "dotenv_values", lambda *_args, **_kwargs: {"PAL_MCP_FORCE_ENV_OVERRIDE": "false"}) except ModuleNotFoundError: pass monkeypatch.setenv("GOOGLE_ALLOWED_MODELS", "gemini-2.5-pro") monkeypatch.setenv("OPENAI_ALLOWED_MODELS", "gpt-5.2") monkeypatch.setenv("OPENROUTER_ALLOWED_MODELS", "gpt5nano") monkeypatch.setenv("XAI_ALLOWED_MODELS", "") import config importlib.reload(config) _register_core_providers() import server importlib.reload(server) # Reload may have re-applied .env overrides; enforce our test configuration for key, value in ( ("DEFAULT_MODEL", "auto"), ("GEMINI_API_KEY", "test-gemini"), ("OPENAI_API_KEY", "test-openai"), ("OPENROUTER_API_KEY", "test-openrouter"), ("GOOGLE_ALLOWED_MODELS", "gemini-2.5-pro"), ("OPENAI_ALLOWED_MODELS", "gpt-5.2"), ("OPENROUTER_ALLOWED_MODELS", "gpt5nano"), ("XAI_ALLOWED_MODELS", ""), ): monkeypatch.setenv(key, value) for var in ("XAI_API_KEY", "CUSTOM_API_URL", "CUSTOM_API_KEY", "DIAL_API_KEY"): monkeypatch.delenv(var, raising=False) for azure_var in ( "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_ALLOWED_MODELS", "AZURE_MODELS_CONFIG_PATH", ): monkeypatch.delenv(azure_var, raising=False) ModelProviderRegistry.reset_for_testing() model_restrictions._restriction_service = None server.configure_providers() with pytest.raises(ToolExecutionError) as exc_info: asyncio.run( server.handle_call_tool( "chat", { "model": "gpt5mini", "prompt": "Tell me about your strengths", }, ) ) payload = json.loads(exc_info.value.payload) assert payload["status"] == "error" available_models = _extract_available_models(payload["content"]) assert set(available_models) == {"gemini-2.5-pro", "gpt-5.2", "gpt5nano", "openai/gpt-5-nano"} @pytest.mark.no_mock_provider def test_error_listing_without_restrictions_shows_full_catalog(monkeypatch, reset_registry): """When no restrictions are set, the full high-capability catalogue should appear.""" monkeypatch.setenv("DEFAULT_MODEL", "auto") monkeypatch.setenv("GEMINI_API_KEY", "test-gemini") monkeypatch.setenv("OPENAI_API_KEY", "test-openai") monkeypatch.setenv("OPENROUTER_API_KEY", "test-openrouter") monkeypatch.setenv("XAI_API_KEY", "test-xai") monkeypatch.setenv("PAL_MCP_FORCE_ENV_OVERRIDE", "false") for azure_var in ( "AZURE_OPENAI_API_KEY", "AZURE_OPENAI_ENDPOINT", "AZURE_OPENAI_ALLOWED_MODELS", "AZURE_MODELS_CONFIG_PATH", ): monkeypatch.delenv(azure_var, raising=False) env_config.reload_env({"PAL_MCP_FORCE_ENV_OVERRIDE": "false"}) try: import dotenv monkeypatch.setattr(dotenv, "dotenv_values", lambda *_args, **_kwargs: {"PAL_MCP_FORCE_ENV_OVERRIDE": "false"}) except ModuleNotFoundError: pass for var in ( "GOOGLE_ALLOWED_MODELS", "OPENAI_ALLOWED_MODELS", "OPENROUTER_ALLOWED_MODELS", "XAI_ALLOWED_MODELS", "DIAL_ALLOWED_MODELS", ): monkeypatch.delenv(var, raising=False) import config importlib.reload(config) _register_core_providers(include_xai=True) import server importlib.reload(server) for key, value in ( ("DEFAULT_MODEL", "auto"), ("GEMINI_API_KEY", "test-gemini"), ("OPENAI_API_KEY", "test-openai"), ("OPENROUTER_API_KEY", "test-openrouter"), ): monkeypatch.setenv(key, value) for var in ( "GOOGLE_ALLOWED_MODELS", "OPENAI_ALLOWED_MODELS", "OPENROUTER_ALLOWED_MODELS", "XAI_ALLOWED_MODELS", "DIAL_ALLOWED_MODELS", "CUSTOM_API_URL", "CUSTOM_API_KEY", ): monkeypatch.delenv(var, raising=False) ModelProviderRegistry.reset_for_testing() model_restrictions._restriction_service = None server.configure_providers() with pytest.raises(ToolExecutionError) as exc_info: asyncio.run( server.handle_call_tool( "chat", { "model": "dummymodel", "prompt": "Hi there", }, ) ) payload = json.loads(exc_info.value.payload) assert payload["status"] == "error" available_models = _extract_available_models(payload["content"]) assert "gemini-2.5-pro" in available_models assert any(model in available_models for model in {"gpt-5.2", "gpt-5"}) assert "grok-4" in available_models assert len(available_models) >= 5 ================================================ FILE: tests/test_auto_mode_provider_selection.py ================================================ """Test auto mode provider selection logic specifically""" import os import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from tools.models import ToolModelCategory @pytest.mark.no_mock_provider class TestAutoModeProviderSelection: """Test the core auto mode provider selection logic""" def setup_method(self): """Set up clean state before each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() def teardown_method(self): """Clean up after each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None def test_gemini_only_fallback_selection(self): """Test auto mode fallback when only Gemini is available.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: original_env[key] = os.environ.get(key) try: # Set up environment - only Gemini available os.environ["GEMINI_API_KEY"] = "test-key" for key in ["OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Register only Gemini provider from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) # Test fallback selection for different categories extended_reasoning = ModelProviderRegistry.get_preferred_fallback_model( ToolModelCategory.EXTENDED_REASONING ) fast_response = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) balanced = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.BALANCED) # Should select appropriate Gemini models assert extended_reasoning in ["gemini-3-pro-preview", "gemini-2.5-pro", "pro"] assert fast_response in ["gemini-2.5-flash", "flash"] assert balanced in ["gemini-2.5-flash", "flash"] finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) def test_openai_only_fallback_selection(self): """Test auto mode fallback when only OpenAI is available.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: original_env[key] = os.environ.get(key) try: # Set up environment - only OpenAI available os.environ["OPENAI_API_KEY"] = "test-key" for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Register only OpenAI provider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Test fallback selection for different categories extended_reasoning = ModelProviderRegistry.get_preferred_fallback_model( ToolModelCategory.EXTENDED_REASONING ) fast_response = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) balanced = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.BALANCED) # Should select appropriate OpenAI models based on new preference order assert extended_reasoning == "gpt-5.1-codex" # GPT-5.1 Codex prioritized for extended reasoning assert fast_response == "gpt-5.2" # gpt-5.2 comes first in fast response preference assert balanced == "gpt-5.2" # gpt-5.2 for balanced finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) def test_both_gemini_and_openai_priority(self): """Test auto mode when both Gemini and OpenAI are available.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: original_env[key] = os.environ.get(key) try: # Set up environment - both Gemini and OpenAI available os.environ["GEMINI_API_KEY"] = "test-key" os.environ["OPENAI_API_KEY"] = "test-key" for key in ["XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Register both providers from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Test fallback selection for different categories extended_reasoning = ModelProviderRegistry.get_preferred_fallback_model( ToolModelCategory.EXTENDED_REASONING ) fast_response = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) # Should prefer Gemini now (based on new provider priority: Gemini before OpenAI) assert extended_reasoning == "gemini-3-pro-preview" # Gemini 3 Pro Preview has higher priority now # Should prefer Gemini for fast response assert fast_response == "gemini-2.5-flash" # Gemini has higher priority now finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) def test_xai_only_fallback_selection(self): """Test auto mode fallback when only XAI is available.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: original_env[key] = os.environ.get(key) try: # Set up environment - only XAI available os.environ["XAI_API_KEY"] = "test-key" for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Register only XAI provider from providers.xai import XAIModelProvider ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) # Test fallback selection for different categories extended_reasoning = ModelProviderRegistry.get_preferred_fallback_model( ToolModelCategory.EXTENDED_REASONING ) fast_response = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) # Should fallback to available models or default fallbacks # Since XAI models are not explicitly handled in fallback logic, # it should fall back to the hardcoded defaults assert extended_reasoning is not None assert fast_response is not None finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) def test_available_models_respects_restrictions(self): """Test that get_available_models respects model restrictions.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "OPENAI_ALLOWED_MODELS"]: original_env[key] = os.environ.get(key) try: # Set up environment with restrictions os.environ["GEMINI_API_KEY"] = "test-key" os.environ["OPENAI_API_KEY"] = "test-key" os.environ["OPENAI_ALLOWED_MODELS"] = "o4-mini" # Only allow o4-mini # Clear restriction service to pick up new restrictions import utils.model_restrictions utils.model_restrictions._restriction_service = None # Register both providers from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Get available models with restrictions available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) # Should include allowed OpenAI model assert "o4-mini" in available_models assert available_models["o4-mini"] == ProviderType.OPENAI # Should NOT include restricted OpenAI models assert "o3" not in available_models assert "o3-mini" not in available_models # Should include all Gemini models (no restrictions) assert "gemini-2.5-flash" in available_models assert available_models["gemini-2.5-flash"] == ProviderType.GOOGLE finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) def test_model_validation_across_providers(self): """Test that model validation works correctly across different providers.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY"]: original_env[key] = os.environ.get(key) try: # Set up all providers os.environ["GEMINI_API_KEY"] = "test-key" os.environ["OPENAI_API_KEY"] = "test-key" os.environ["XAI_API_KEY"] = "test-key" # Register all providers from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.xai import XAIModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) # Test model validation - each provider should handle its own models # Gemini models gemini_provider = ModelProviderRegistry.get_provider_for_model("flash") assert gemini_provider is not None assert gemini_provider.get_provider_type() == ProviderType.GOOGLE # OpenAI models openai_provider = ModelProviderRegistry.get_provider_for_model("o3") assert openai_provider is not None assert openai_provider.get_provider_type() == ProviderType.OPENAI # XAI models xai_provider = ModelProviderRegistry.get_provider_for_model("grok") assert xai_provider is not None assert xai_provider.get_provider_type() == ProviderType.XAI # Invalid model should return None invalid_provider = ModelProviderRegistry.get_provider_for_model("invalid-model-name") assert invalid_provider is None finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) def test_alias_resolution_before_api_calls(self): """Test that model aliases are resolved before being passed to providers.""" # Save original environment original_env = {} for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY"]: original_env[key] = os.environ.get(key) try: # Set up all providers os.environ["GEMINI_API_KEY"] = "test-key" os.environ["OPENAI_API_KEY"] = "test-key" os.environ["XAI_API_KEY"] = "test-key" # Register all providers from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.xai import XAIModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.XAI, XAIModelProvider) # Test that providers resolve aliases correctly test_cases = [ ("flash", ProviderType.GOOGLE, "gemini-2.5-flash"), ("pro", ProviderType.GOOGLE, "gemini-3-pro-preview"), # "pro" now resolves to gemini-3-pro-preview ("mini", ProviderType.OPENAI, "gpt-5-mini"), # "mini" now resolves to gpt-5-mini ("o3mini", ProviderType.OPENAI, "o3-mini"), ("grok", ProviderType.XAI, "grok-4"), ("grok-4.1-fast-reasoning", ProviderType.XAI, "grok-4-1-fast-reasoning"), ] for alias, expected_provider_type, expected_resolved_name in test_cases: provider = ModelProviderRegistry.get_provider_for_model(alias) assert provider is not None, f"No provider found for alias '{alias}'" assert provider.get_provider_type() == expected_provider_type, f"Wrong provider for '{alias}'" # Test alias resolution resolved_model_name = provider._resolve_model_name(alias) assert ( resolved_model_name == expected_resolved_name ), f"Alias '{alias}' should resolve to '{expected_resolved_name}', got '{resolved_model_name}'" finally: # Restore original environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) ================================================ FILE: tests/test_auto_model_planner_fix.py ================================================ """ Unit tests for the auto model planner fix. This test confirms that the planner tool no longer fails when DEFAULT_MODEL is "auto" and only basic providers (Google/OpenAI) are configured, while ensuring other tools still properly require model resolution. """ from unittest.mock import patch from mcp.types import TextContent from tools.chat import ChatTool from tools.planner import PlannerTool from tools.shared.base_tool import BaseTool class TestAutoModelPlannerFix: """Test the fix for auto model resolution with planner tool.""" def test_planner_requires_model_false(self): """Test that planner tool returns False for requires_model.""" planner = PlannerTool() assert planner.requires_model() is False def test_chat_requires_model_true(self): """Test that chat tool returns True for requires_model (default behavior).""" chat = ChatTool() assert chat.requires_model() is True def test_base_tool_requires_model_default(self): """Test that BaseTool default implementation returns True.""" # Create a mock tool that doesn't override requires_model class MockTool(BaseTool): def get_name(self): return "mock" def get_description(self): return "Mock tool" def get_input_schema(self): return {} def get_system_prompt(self): return "Mock prompt" def get_request_model(self): from tools.shared.base_models import ToolRequest return ToolRequest async def prepare_prompt(self, request): return "Mock prompt" mock_tool = MockTool() assert mock_tool.requires_model() is True @patch("config.DEFAULT_MODEL", "auto") @patch("providers.registry.ModelProviderRegistry.get_provider_for_model") def test_auto_model_error_before_fix_simulation(self, mock_get_provider): """ Simulate the error that would occur before the fix. This test simulates what would happen if server.py didn't check requires_model() and tried to resolve "auto" as a literal model name. """ # Mock the scenario where no provider is found for "auto" mock_get_provider.return_value = None # This should return None, simulating the "No provider found for model auto" error result = mock_get_provider("auto") assert result is None # Verify that the mock was called with "auto" mock_get_provider.assert_called_with("auto") @patch("server.DEFAULT_MODEL", "auto") async def test_planner_execution_bypasses_model_resolution(self): """ Test that planner tool execution works even when DEFAULT_MODEL is "auto". This test confirms that the fix allows planner to work regardless of model configuration since it doesn't need model resolution. """ planner = PlannerTool() # Test with minimal planner arguments arguments = {"step": "Test planning step", "step_number": 1, "total_steps": 1, "next_step_required": False} # This should work without any model resolution result = await planner.execute(arguments) # Verify we got a result assert isinstance(result, list) assert len(result) > 0 assert isinstance(result[0], TextContent) # Parse the JSON response to verify it's valid import json response_data = json.loads(result[0].text) assert response_data["status"] == "planning_complete" assert response_data["step_number"] == 1 @patch("config.DEFAULT_MODEL", "auto") def test_server_model_resolution_logic(self): """ Test the server-side logic that checks requires_model() before model resolution. This simulates the key fix in server.py where we check tool.requires_model() before attempting model resolution. """ planner = PlannerTool() chat = ChatTool() # Simulate the server logic def simulate_server_model_resolution(tool, model_name): """Simulate the fixed server logic from server.py""" if not tool.requires_model(): # Skip model resolution for tools that don't require models return "SKIP_MODEL_RESOLUTION" else: # Would normally do model resolution here return f"RESOLVE_MODEL_{model_name}" # Test planner (should skip model resolution) result = simulate_server_model_resolution(planner, "auto") assert result == "SKIP_MODEL_RESOLUTION" # Test chat (should attempt model resolution) result = simulate_server_model_resolution(chat, "auto") assert result == "RESOLVE_MODEL_auto" def test_provider_registry_auto_handling(self): """ Test that the provider registry correctly handles model resolution. This tests the scenario where providers don't recognize "auto" as a model. """ from providers.registry import ModelProviderRegistry # This should return None since "auto" is not a real model name provider = ModelProviderRegistry.get_provider_for_model("auto") assert provider is None, "Provider registry should not find a provider for literal 'auto'" @patch("config.DEFAULT_MODEL", "auto") async def test_end_to_end_planner_with_auto_mode(self): """ End-to-end test of planner tool execution in auto mode. This test verifies that the complete flow works when DEFAULT_MODEL is "auto" and the planner tool is used. """ planner = PlannerTool() # Verify the tool doesn't require model resolution assert not planner.requires_model() # Test a multi-step planning scenario step1_args = { "step": "Analyze the current system architecture", "step_number": 1, "total_steps": 3, "next_step_required": True, } result1 = await planner.execute(step1_args) assert len(result1) > 0 # Parse and verify the response import json response1 = json.loads(result1[0].text) assert response1["status"] == "pause_for_planning" assert response1["next_step_required"] is True assert "continuation_id" in response1 # Test step 2 with continuation continuation_id = response1["continuation_id"] step2_args = { "step": "Design the microservices architecture", "step_number": 2, "total_steps": 3, "next_step_required": True, "continuation_id": continuation_id, } result2 = await planner.execute(step2_args) assert len(result2) > 0 response2 = json.loads(result2[0].text) assert response2["status"] == "pause_for_planning" assert response2["step_number"] == 2 def test_other_tools_still_require_models(self): """ Verify that other tools still properly require model resolution. This ensures our fix doesn't break existing functionality. Note: Debug tool requires model resolution for expert analysis phase. """ from tools.analyze import AnalyzeTool from tools.chat import ChatTool from tools.debug import DebugIssueTool # Test various tools still require models tools_requiring_models = [ChatTool(), AnalyzeTool(), DebugIssueTool()] for tool in tools_requiring_models: assert tool.requires_model() is True, f"{tool.get_name()} should require model resolution" # Note: Debug tool requires model resolution for expert analysis phase # Only planner truly manages its own model calls and doesn't need resolution ================================================ FILE: tests/test_azure_openai_provider.py ================================================ import sys import types import pytest if "openai" not in sys.modules: # pragma: no cover - test shim for optional dependency stub = types.ModuleType("openai") stub.AzureOpenAI = object # Replaced with a mock inside tests sys.modules["openai"] = stub from providers.azure_openai import AzureOpenAIProvider from providers.shared import ModelCapabilities, ProviderType class _DummyResponse: def __init__(self): self.choices = [ types.SimpleNamespace( message=types.SimpleNamespace(content="hello"), finish_reason="stop", ) ] self.model = "prod-gpt4o" self.id = "resp-123" self.created = 0 self.usage = types.SimpleNamespace( prompt_tokens=5, completion_tokens=3, total_tokens=8, ) @pytest.fixture def dummy_azure_client(monkeypatch): captured = {} class _DummyAzureClient: def __init__(self, **kwargs): captured["client_kwargs"] = kwargs self.chat = types.SimpleNamespace(completions=types.SimpleNamespace(create=self._create_completion)) self.responses = types.SimpleNamespace(create=self._create_response) def _create_completion(self, **kwargs): captured["request_kwargs"] = kwargs return _DummyResponse() def _create_response(self, **kwargs): captured["responses_kwargs"] = kwargs return _DummyResponse() monkeypatch.delenv("AZURE_OPENAI_ALLOWED_MODELS", raising=False) monkeypatch.setattr("providers.azure_openai.AzureOpenAI", _DummyAzureClient) return captured def test_generate_content_uses_deployment_mapping(dummy_azure_client): provider = AzureOpenAIProvider( api_key="key", azure_endpoint="https://example.openai.azure.com/", deployments={"gpt-4o": "prod-gpt4o"}, ) result = provider.generate_content("hello", "gpt-4o") assert dummy_azure_client["request_kwargs"]["model"] == "prod-gpt4o" assert result.model_name == "gpt-4o" assert result.provider == ProviderType.AZURE assert provider.validate_model_name("prod-gpt4o") def test_generate_content_accepts_deployment_alias(dummy_azure_client): provider = AzureOpenAIProvider( api_key="key", azure_endpoint="https://example.openai.azure.com/", deployments={"gpt-4o-mini": "mini-deployment"}, ) # Calling with the deployment alias should still resolve properly. result = provider.generate_content("hi", "mini-deployment") assert dummy_azure_client["request_kwargs"]["model"] == "mini-deployment" assert result.model_name == "gpt-4o-mini" def test_client_initialization_uses_endpoint_and_version(dummy_azure_client): provider = AzureOpenAIProvider( api_key="key", azure_endpoint="https://example.openai.azure.com/", api_version="2024-03-15-preview", deployments={"gpt-4o": "prod"}, ) _ = provider.client assert dummy_azure_client["client_kwargs"]["azure_endpoint"] == "https://example.openai.azure.com" assert dummy_azure_client["client_kwargs"]["api_version"] == "2024-03-15-preview" def test_deployment_overrides_capabilities(dummy_azure_client): provider = AzureOpenAIProvider( api_key="key", azure_endpoint="https://example.openai.azure.com/", deployments={ "gpt-4o": { "deployment": "prod-gpt4o", "friendly_name": "Azure GPT-4o EU", "intelligence_score": 19, "supports_temperature": False, "temperature_constraint": "fixed", } }, ) caps = provider.get_capabilities("gpt-4o") assert caps.friendly_name == "Azure GPT-4o EU" assert caps.intelligence_score == 19 assert not caps.supports_temperature def test_registry_configuration_merges_capabilities(dummy_azure_client, monkeypatch): def fake_registry_entries(self): capability = ModelCapabilities( provider=ProviderType.AZURE, model_name="gpt-4o", friendly_name="Azure GPT-4o Registry", context_window=500_000, max_output_tokens=128_000, ) return {"gpt-4o": {"deployment": "registry-deployment", "capability": capability}} monkeypatch.setattr(AzureOpenAIProvider, "_load_registry_entries", fake_registry_entries) provider = AzureOpenAIProvider( api_key="key", azure_endpoint="https://example.openai.azure.com/", ) # Capability should come from registry caps = provider.get_capabilities("gpt-4o") assert caps.friendly_name == "Azure GPT-4o Registry" assert caps.context_window == 500_000 # API call should use deployment defined in registry provider.generate_content("hello", "gpt-4o") assert dummy_azure_client["request_kwargs"]["model"] == "registry-deployment" ================================================ FILE: tests/test_buggy_behavior_prevention.py ================================================ """ Regression scenarios ensuring alias-aware model listings stay correct. Each test captures behavior that previously regressed so we can guard it permanently. The focus is confirming aliases and their canonical targets remain visible to the restriction service and related validation logic. """ import os from unittest.mock import MagicMock, patch import pytest from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.shared import ProviderType from utils.model_restrictions import ModelRestrictionService class TestBuggyBehaviorPrevention: """Regression tests for alias-aware restriction validation.""" def test_alias_listing_includes_targets_for_restriction_validation(self): """Alias-aware lists expose both aliases and canonical targets.""" provider = OpenAIModelProvider(api_key="test-key") # Baseline alias-only list captured for regression documentation alias_only_snapshot = ["mini", "o3mini"] # Missing 'o4-mini', 'o3-mini' targets # Canonical listing with aliases and targets comprehensive_list = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) # Comprehensive listing should contain aliases and their targets assert "mini" in comprehensive_list assert "o4-mini" in comprehensive_list assert "o3mini" in comprehensive_list assert "o3-mini" in comprehensive_list # Legacy alias-only snapshots exclude targets assert "o4-mini" not in alias_only_snapshot assert "o3-mini" not in alias_only_snapshot # This scenario previously failed when targets were omitted service = ModelRestrictionService() service.restrictions = {ProviderType.OPENAI: {"o4-mini"}} # Restrict to target with patch("utils.model_restrictions.logger") as mock_logger: provider_instances = {ProviderType.OPENAI: provider} service.validate_against_known_models(provider_instances) # No warnings expected because alias-aware list includes the target target_warnings = [ call for call in mock_logger.warning.call_args_list if "o4-mini" in str(call) and "not a recognized" in str(call) ] assert len(target_warnings) == 0, "o4-mini should be recognized as a valid target" def test_target_models_are_recognized_during_validation(self): """Target model restrictions should not trigger false warnings.""" # Test with Gemini provider too provider = GeminiModelProvider(api_key="test-key") all_known = provider.list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) # Verify both aliases and targets are included assert "flash" in all_known # alias assert "gemini-2.5-flash" in all_known # target assert "pro" in all_known # alias assert "gemini-2.5-pro" in all_known # target # Simulate admin restricting to target model names service = ModelRestrictionService() service.restrictions = { ProviderType.GOOGLE: { "gemini-2.5-flash", # Target name restriction "gemini-2.5-pro", # Target name restriction } } with patch("utils.model_restrictions.logger") as mock_logger: provider_instances = {ProviderType.GOOGLE: provider} service.validate_against_known_models(provider_instances) # Should NOT warn about these valid target models all_warnings = [str(call) for call in mock_logger.warning.call_args_list] for warning in all_warnings: assert "gemini-2.5-flash" not in warning or "not a recognized" not in warning assert "gemini-2.5-pro" not in warning or "not a recognized" not in warning def test_policy_enforcement_remains_comprehensive(self): """Policy validation must account for both aliases and targets.""" provider = OpenAIModelProvider(api_key="test-key") # Simulate a scenario where admin wants to restrict specific targets with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini"}): # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None # These should work because they're explicitly allowed assert provider.validate_model_name("o3-mini") assert provider.validate_model_name("o4-mini") # These should be blocked assert not provider.validate_model_name("o3-pro") # Not in allowed list assert not provider.validate_model_name("o3") # Not in allowed list # "mini" now resolves to gpt-5-mini, not o4-mini, so it should be blocked assert not provider.validate_model_name("mini") # Resolves to gpt-5-mini, which is NOT allowed # But o4mini (the actual alias for o4-mini) should work assert provider.validate_model_name("o4mini") # Resolves to o4-mini, which IS allowed # Verify our alias-aware list includes the restricted models all_known = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) assert "o3-mini" in all_known # Should be known (and allowed) assert "o4-mini" in all_known # Should be known (and allowed) assert "o3-pro" in all_known # Should be known (but blocked) assert "mini" in all_known # Should be known (and allowed since it resolves to o4-mini) def test_alias_aware_listing_extends_canonical_view(self): """Alias-aware list should be a superset of restriction-filtered names.""" provider = OpenAIModelProvider(api_key="test-key") baseline_models = provider.list_models(respect_restrictions=False) alias_aware_models = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) # Alias-aware variant should contain everything from the baseline for model in baseline_models: assert model.lower() in [ m.lower() for m in alias_aware_models ], f"Alias-aware listing missing baseline model {model}" # Alias-aware variant should include canonical targets as well for target in ("o4-mini", "o3-mini"): assert target in alias_aware_models, f"Alias-aware listing should include target model {target}" def test_restriction_validation_uses_alias_aware_variant(self): """Validation should request the alias-aware lowercased, deduped list.""" service = ModelRestrictionService() # Simulate a provider that only returns aliases when asked for models alias_only_provider = MagicMock() alias_only_provider.MODEL_CAPABILITIES = { "mini": "o4-mini", "o3mini": "o3-mini", "o4-mini": {"context_window": 200000}, "o3-mini": {"context_window": 200000}, } # Simulate alias-only vs. alias-aware behavior using a side effect def list_models_side_effect(**kwargs): respect_restrictions = kwargs.get("respect_restrictions", True) include_aliases = kwargs.get("include_aliases", True) lowercase = kwargs.get("lowercase", False) unique = kwargs.get("unique", False) if respect_restrictions and include_aliases and not lowercase and not unique: return ["mini", "o3mini"] if not respect_restrictions and include_aliases and lowercase and unique: return ["mini", "o3mini", "o4-mini", "o3-mini"] raise AssertionError(f"Unexpected list_models call: {kwargs}") alias_only_provider.list_models.side_effect = list_models_side_effect # Test that validation now uses the comprehensive method service.restrictions = {ProviderType.OPENAI: {"o4-mini"}} # Restrict to target with patch("utils.model_restrictions.logger") as mock_logger: provider_instances = {ProviderType.OPENAI: alias_only_provider} service.validate_against_known_models(provider_instances) # Verify the alias-aware variant was used alias_only_provider.list_models.assert_called_with( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) # Should not warn about o4-mini being unrecognized target_warnings = [ call for call in mock_logger.warning.call_args_list if "o4-mini" in str(call) and "not a recognized" in str(call) ] assert len(target_warnings) == 0 def test_alias_listing_covers_targets_for_all_providers(self): """Alias-aware listings should expose targets across providers.""" providers_to_test = [ (OpenAIModelProvider(api_key="test-key"), "mini", "o4-mini"), (GeminiModelProvider(api_key="test-key"), "flash", "gemini-2.5-flash"), ] for provider, alias, target in providers_to_test: all_known = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True ) # Every provider should include both aliases and targets assert alias in all_known, f"{provider.__class__.__name__} missing alias {alias}" assert target in all_known, f"{provider.__class__.__name__} missing target {target}" # No duplicates should exist assert len(all_known) == len(set(all_known)), f"{provider.__class__.__name__} returns duplicate models" @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini,invalid-model"}) def test_validation_correctly_identifies_invalid_models(self): """Validation should flag invalid models while listing valid targets.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None service = ModelRestrictionService() provider = OpenAIModelProvider(api_key="test-key") with patch("utils.model_restrictions.logger") as mock_logger: provider_instances = {ProviderType.OPENAI: provider} service.validate_against_known_models(provider_instances) invalid_warnings = [ call for call in mock_logger.warning.call_args_list if "invalid-model" in str(call) and "not a recognized" in str(call) ] assert len(invalid_warnings) > 0, "Should warn about truly invalid models" # The warning should mention o4-mini in the known models list warning_text = str(mock_logger.warning.call_args_list[0]) assert "Known models:" in warning_text, "Warning should include known models list" assert "o4-mini" in warning_text, "o4-mini should appear in known models" assert "o3-mini" in warning_text, "o3-mini should appear in known models" # But the warning should be specifically about invalid-model assert "'invalid-model'" in warning_text, "Warning should specifically mention invalid-model" def test_custom_provider_alias_listing(self): """Custom provider should expose alias-aware listings as well.""" from providers.custom import CustomProvider # This might fail if no URL is set, but that's expected try: provider = CustomProvider(base_url="http://test.com/v1") all_known = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True ) # Should return a list (might be empty if registry not loaded) assert isinstance(all_known, list) except ValueError: # Expected if no base_url configured, skip this test pytest.skip("Custom provider requires URL configuration") def test_openrouter_provider_alias_listing(self): """OpenRouter provider should expose alias-aware listings.""" from providers.openrouter import OpenRouterProvider provider = OpenRouterProvider(api_key="test-key") all_known = provider.list_models(respect_restrictions=False, include_aliases=True, lowercase=True, unique=True) # Should return a list with both aliases and targets assert isinstance(all_known, list) # Should include some known OpenRouter aliases and their targets # (Exact content depends on registry, but structure should be correct) ================================================ FILE: tests/test_cassette_semantic_matching.py ================================================ """ Tests for cassette semantic matching to prevent breaks from prompt changes. This validates that o3 model cassettes match on semantic content (model + user question) rather than exact request bodies, preventing cassette breaks when system prompts change. """ import hashlib import json import pytest from tests.http_transport_recorder import ReplayTransport class TestCassetteSemanticMatching: """Test that cassette matching is resilient to prompt changes.""" @pytest.fixture def dummy_cassette(self, tmp_path): """Create a minimal dummy cassette file.""" cassette_file = tmp_path / "dummy.json" cassette_file.write_text(json.dumps({"interactions": []})) return cassette_file def test_o3_model_semantic_matching(self, dummy_cassette): """Test that o3 models use semantic matching.""" transport = ReplayTransport(str(dummy_cassette)) # Two requests with same user question but different system prompts request1_body = { "model": "o3-pro", "reasoning": {"effort": "medium"}, "input": [ { "role": "user", "content": [ { "type": "input_text", "text": "System prompt v1...\n\n=== USER REQUEST ===\nWhat is 2 + 2?\n=== END REQUEST ===\n\nMore instructions...", } ], } ], } request2_body = { "model": "o3-pro", "reasoning": {"effort": "medium"}, "input": [ { "role": "user", "content": [ { "type": "input_text", "text": "System prompt v2 (DIFFERENT)...\n\n=== USER REQUEST ===\nWhat is 2 + 2?\n=== END REQUEST ===\n\nDifferent instructions...", } ], } ], } # Extract semantic fields - should be identical semantic1 = transport._extract_semantic_fields(request1_body) semantic2 = transport._extract_semantic_fields(request2_body) assert semantic1 == semantic2, "Semantic fields should match despite different prompts" assert semantic1["user_question"] == "What is 2 + 2?" assert semantic1["model"] == "o3-pro" assert semantic1["reasoning"] == {"effort": "medium"} # Generate signatures - should be identical content1 = json.dumps(semantic1, sort_keys=True) content2 = json.dumps(semantic2, sort_keys=True) hash1 = hashlib.md5(content1.encode()).hexdigest() hash2 = hashlib.md5(content2.encode()).hexdigest() assert hash1 == hash2, "Hashes should match for same semantic content" def test_non_o3_model_exact_matching(self, dummy_cassette): """Test that non-o3 models still use exact matching.""" transport = ReplayTransport(str(dummy_cassette)) request_body = { "model": "gpt-4", "messages": [{"role": "user", "content": "test"}], } # Should not use semantic matching assert not transport._is_o3_model_request(request_body) def test_o3_mini_semantic_matching(self, dummy_cassette): """Test that o3-mini also uses semantic matching.""" transport = ReplayTransport(str(dummy_cassette)) request_body = { "model": "o3-mini", "reasoning": {"effort": "low"}, "input": [ { "role": "user", "content": [ {"type": "input_text", "text": "System...\n\n=== USER REQUEST ===\nTest\n=== END REQUEST ==="} ], } ], } assert transport._is_o3_model_request(request_body) semantic = transport._extract_semantic_fields(request_body) assert semantic["model"] == "o3-mini" assert semantic["user_question"] == "Test" def test_o3_without_request_markers(self, dummy_cassette): """Test o3 requests without REQUEST markers fall back to full text.""" transport = ReplayTransport(str(dummy_cassette)) request_body = { "model": "o3-pro", "reasoning": {"effort": "medium"}, "input": [{"role": "user", "content": [{"type": "input_text", "text": "Just a simple question"}]}], } semantic = transport._extract_semantic_fields(request_body) assert semantic["user_question"] == "Just a simple question" ================================================ FILE: tests/test_challenge.py ================================================ """ Tests for Challenge tool - validating critical challenge prompt wrapper This module contains unit tests to ensure that the Challenge tool properly wraps statements to encourage critical thinking and avoid automatic agreement patterns. """ import json from unittest.mock import patch import pytest from tools.challenge import ChallengeRequest, ChallengeTool from tools.shared.exceptions import ToolExecutionError class TestChallengeTool: """Test suite for Challenge tool""" def setup_method(self): """Set up test fixtures""" self.tool = ChallengeTool() def test_tool_metadata(self): """Test that tool metadata matches requirements""" assert self.tool.get_name() == "challenge" assert "reflexive agreement" in self.tool.get_description() assert "critical thinking" in self.tool.get_description() assert "reasoned analysis" in self.tool.get_description() assert self.tool.get_default_temperature() == 1.0 # TEMPERATURE_ANALYTICAL def test_requires_model(self): """Test that challenge tool doesn't require a model""" assert self.tool.requires_model() is False def test_schema_structure(self): """Test that schema has correct structure and excludes model fields""" schema = self.tool.get_input_schema() # Basic schema structure assert schema["type"] == "object" assert "properties" in schema assert "required" in schema # Required fields assert "prompt" in schema["required"] assert len(schema["required"]) == 1 # Only prompt is required # Properties properties = schema["properties"] assert "prompt" in properties # Should NOT have model-related fields since it doesn't require a model assert "model" not in properties assert "temperature" not in properties assert "thinking_mode" not in properties assert "continuation_id" not in properties def test_request_model_validation(self): """Test that the request model validates correctly""" # Test valid request request = ChallengeRequest(prompt="The sky is green") assert request.prompt == "The sky is green" # Test with longer prompt long_prompt = ( "Machine learning models always produce accurate results and should be trusted without verification" ) request = ChallengeRequest(prompt=long_prompt) assert request.prompt == long_prompt def test_required_fields(self): """Test that required fields are enforced""" from pydantic import ValidationError # Missing prompt should raise validation error with pytest.raises(ValidationError): ChallengeRequest() @pytest.mark.asyncio async def test_execute_success(self): """Test successful execution of challenge tool""" arguments = {"prompt": "All software bugs are caused by syntax errors"} result = await self.tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 assert result[0].type == "text" # Parse the JSON response response_data = json.loads(result[0].text) # Check response structure assert response_data["status"] == "challenge_accepted" assert response_data["original_statement"] == "All software bugs are caused by syntax errors" assert "challenge_prompt" in response_data assert "instructions" in response_data # Check that the challenge prompt contains critical thinking instructions challenge_prompt = response_data["challenge_prompt"] assert "CRITICAL REASSESSMENT – Do not automatically agree" in challenge_prompt assert "Carefully evaluate the statement above" in challenge_prompt assert response_data["original_statement"] in challenge_prompt assert "flaws, gaps, or misleading points" in challenge_prompt assert "thoughtful analysis" in challenge_prompt @pytest.mark.asyncio async def test_execute_error_handling(self): """Test error handling in execute method""" # Test with invalid arguments (non-dict) with patch.object(self.tool, "get_request_model", side_effect=Exception("Test error")): with pytest.raises(ToolExecutionError) as exc_info: await self.tool.execute({"prompt": "test"}) response_data = json.loads(exc_info.value.payload) assert response_data["status"] == "error" assert "Test error" in response_data["error"] def test_wrap_prompt_for_challenge(self): """Test the prompt wrapping functionality""" original_prompt = "Python is the best programming language" wrapped = self.tool._wrap_prompt_for_challenge(original_prompt) # Check structure assert "CRITICAL REASSESSMENT – Do not automatically agree" in wrapped assert "Carefully evaluate the statement above" in wrapped assert f'"{original_prompt}"' in wrapped assert "flaws, gaps, or misleading points" in wrapped assert "thoughtful analysis" in wrapped def test_multiple_prompts(self): """Test that tool handles various types of prompts correctly""" test_prompts = [ "All code should be written in assembly for maximum performance", "Comments are unnecessary if code is self-documenting", "Testing is a waste of time for experienced developers", "Global variables make code easier to understand", "The more design patterns used, the better the code", ] for prompt in test_prompts: request = ChallengeRequest(prompt=prompt) wrapped = self.tool._wrap_prompt_for_challenge(request.prompt) # Each wrapped prompt should contain the original assert prompt in wrapped assert "CRITICAL REASSESSMENT" in wrapped def test_tool_fields(self): """Test tool-specific field definitions""" fields = self.tool.get_tool_fields() assert "prompt" in fields assert fields["prompt"]["type"] == "string" assert "Statement to scrutinize" in fields["prompt"]["description"] assert "strip the word 'challenge'" in fields["prompt"]["description"] def test_required_fields_list(self): """Test required fields list""" required = self.tool.get_required_fields() assert required == ["prompt"] @pytest.mark.asyncio async def test_not_used_methods(self): """Test that methods not used by challenge tool work correctly""" request = ChallengeRequest(prompt="test") # These methods aren't used since challenge doesn't call AI prompt = await self.tool.prepare_prompt(request) assert prompt == "" response = self.tool.format_response("test response", request) assert response == "test response" def test_special_characters_in_prompt(self): """Test handling of special characters in prompts""" special_prompt = 'The "best" way to handle errors is to use try/except: pass' request = ChallengeRequest(prompt=special_prompt) wrapped = self.tool._wrap_prompt_for_challenge(request.prompt) # Should handle quotes properly assert special_prompt in wrapped @pytest.mark.asyncio async def test_unicode_support(self): """Test that tool handles unicode characters correctly""" unicode_prompt = "软件开发中最重要的是写代码，测试不重要 🚀" arguments = {"prompt": unicode_prompt} result = await self.tool.execute(arguments) response_data = json.loads(result[0].text) assert response_data["original_statement"] == unicode_prompt assert unicode_prompt in response_data["challenge_prompt"] if __name__ == "__main__": pytest.main([__file__]) ================================================ FILE: tests/test_chat_codegen_integration.py ================================================ """Integration test for Chat tool code generation with Gemini 2.5 Pro. This test uses the Google Gemini SDK's built-in record/replay support. To refresh the cassette, delete the existing JSON file under ``tests/gemini_cassettes/chat_codegen/gemini25_pro_calculator/mldev.json`` and run: ``` GEMINI_API_KEY= pytest tests/test_chat_codegen_integration.py::test_chat_codegen_saves_file ``` The test will automatically record a new interaction when the cassette is missing and the environment variable `GEMINI_API_KEY` is set to a valid key. """ from __future__ import annotations import json import os from pathlib import Path import pytest from providers.gemini import GeminiModelProvider from providers.registry import ModelProviderRegistry, ProviderType from tools.chat import ChatTool REPLAYS_ROOT = Path(__file__).parent / "gemini_cassettes" CASSETTE_DIR = REPLAYS_ROOT / "chat_codegen" CASSETTE_PATH = CASSETTE_DIR / "gemini25_pro_calculator" / "mldev.json" CASSETTE_REPLAY_ID = "chat_codegen/gemini25_pro_calculator/mldev" @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_chat_codegen_saves_file(monkeypatch, tmp_path): """Ensure Gemini 2.5 Pro responses create pal_generated.code when code is emitted.""" CASSETTE_PATH.parent.mkdir(parents=True, exist_ok=True) recording_mode = not CASSETTE_PATH.exists() gemini_key = os.getenv("GEMINI_API_KEY", "") if recording_mode: if not gemini_key or gemini_key.startswith("dummy"): pytest.skip("Cassette missing and GEMINI_API_KEY not configured. Provide a real key to record.") client_mode = "record" else: gemini_key = "dummy-key-for-replay" client_mode = "replay" with monkeypatch.context() as m: m.setenv("GEMINI_API_KEY", gemini_key) m.setenv("DEFAULT_MODEL", "auto") m.setenv("GOOGLE_ALLOWED_MODELS", "gemini-2.5-pro") m.setenv("GOOGLE_GENAI_CLIENT_MODE", client_mode) m.setenv("GOOGLE_GENAI_REPLAYS_DIRECTORY", str(REPLAYS_ROOT)) m.setenv("GOOGLE_GENAI_REPLAY_ID", CASSETTE_REPLAY_ID) # Clear other provider keys to avoid unintended routing for key in ["OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "CUSTOM_API_KEY"]: m.delenv(key, raising=False) ModelProviderRegistry.reset_for_testing() ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) working_dir = tmp_path / "codegen" working_dir.mkdir() preexisting = working_dir / "pal_generated.code" preexisting.write_text("stale contents", encoding="utf-8") chat_tool = ChatTool() prompt = ( "Please generate a Python module with functions `add` and `multiply` that perform" " basic addition and multiplication. Produce the response using the structured" " format so the assistant can apply the files directly." ) result = await chat_tool.execute( { "prompt": prompt, "model": "gemini-2.5-pro", "working_directory_absolute_path": str(working_dir), } ) provider = ModelProviderRegistry.get_provider_for_model("gemini-2.5-pro") if provider is not None: try: provider.client.close() except AttributeError: pass # Reset restriction service cache to avoid leaking allowed-model config try: from utils import model_restrictions model_restrictions._restriction_service = None # type: ignore[attr-defined] except Exception: pass assert result and result[0].type == "text" payload = json.loads(result[0].text) assert payload["status"] in {"success", "continuation_available"} artifact_path = working_dir / "pal_generated.code" assert artifact_path.exists() saved = artifact_path.read_text() assert "" in saved assert " str: digit_match = re.search(r"\b(\d{1,2})\b", text) if digit_match: return digit_match.group(1) lower_text = text.lower() for word, value in WORD_TO_NUMBER.items(): if re.search(rf"\b{word}\b", lower_text): return str(value) return "" @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_chat_cross_model_continuation(monkeypatch, tmp_path): """Verify continuation across Gemini then OpenAI using recorded interactions.""" env_updates = { "DEFAULT_MODEL": "auto", "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", ""), "GEMINI_API_KEY": os.getenv("GEMINI_API_KEY", ""), } keys_to_clear = [ "XAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY", "CUSTOM_API_KEY", "CUSTOM_API_URL", ] recording_mode = not OPENAI_CASSETTE_PATH.exists() or not GEMINI_REPLAY_PATH.exists() if recording_mode: openai_key = env_updates["OPENAI_API_KEY"].strip() gemini_key = env_updates["GEMINI_API_KEY"].strip() if (not openai_key or openai_key.startswith("dummy")) or (not gemini_key or gemini_key.startswith("dummy")): pytest.skip( "Cross-provider cassette missing and OPENAI_API_KEY/GEMINI_API_KEY not configured. Provide real keys to record." ) GEMINI_REPLAY_PATH.parent.mkdir(parents=True, exist_ok=True) # Step 1 – Gemini picks a number with monkeypatch.context() as m: m.setenv("DEFAULT_MODEL", env_updates["DEFAULT_MODEL"]) m.setenv("GOOGLE_ALLOWED_MODELS", "gemini-2.5-flash") m.setenv("OPENAI_ALLOWED_MODELS", "gpt-5") if recording_mode: m.setenv("OPENAI_API_KEY", env_updates["OPENAI_API_KEY"]) m.setenv("GEMINI_API_KEY", env_updates["GEMINI_API_KEY"]) m.setenv("GOOGLE_GENAI_CLIENT_MODE", "record") else: m.setenv("OPENAI_API_KEY", "dummy-key-for-replay") m.setenv("GEMINI_API_KEY", "dummy-key-for-replay") m.setenv("GOOGLE_GENAI_CLIENT_MODE", "replay") m.setenv("GOOGLE_GENAI_REPLAYS_DIRECTORY", str(GEMINI_CASSETTE_DIR)) m.setenv("GOOGLE_GENAI_REPLAY_ID", GEMINI_REPLAY_ID) for key in keys_to_clear: m.delenv(key, raising=False) ModelProviderRegistry.reset_for_testing() from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) from utils import conversation_memory m.setattr(conversation_memory.uuid, "uuid4", lambda: FIXED_THREAD_ID) chat_tool = ChatTool() working_directory = str(tmp_path) step1_args = { "prompt": "Pick a number between 1 and 10 and respond with JUST that number.", "model": "gemini-2.5-flash", "temperature": 0.2, "working_directory_absolute_path": working_directory, } step1_result = await chat_tool.execute(step1_args) assert step1_result and step1_result[0].type == "text" step1_data = json.loads(step1_result[0].text) assert step1_data["status"] in {"success", "continuation_available"} assert step1_data.get("metadata", {}).get("provider_used") == "google" continuation_offer = step1_data.get("continuation_offer") assert continuation_offer is not None continuation_id = continuation_offer["continuation_id"] assert continuation_id chosen_number = _extract_number(step1_data["content"]) assert chosen_number.isdigit() assert 1 <= int(chosen_number) <= 10 # Ensure replay is flushed for Gemini recordings gemini_provider = ModelProviderRegistry.get_provider_for_model("gemini-2.5-flash") if gemini_provider is not None: try: client = gemini_provider.client if hasattr(client, "close"): client.close() finally: if hasattr(gemini_provider, "_client"): gemini_provider._client = None assert GEMINI_REPLAY_PATH.exists() # Step 2 – gpt-5 recalls the number via continuation with monkeypatch.context() as m: if recording_mode: m.setenv("OPENAI_API_KEY", env_updates["OPENAI_API_KEY"]) m.setenv("GEMINI_API_KEY", env_updates["GEMINI_API_KEY"]) m.setenv("GOOGLE_GENAI_CLIENT_MODE", "record") else: m.setenv("OPENAI_API_KEY", "dummy-key-for-replay") m.setenv("GEMINI_API_KEY", "dummy-key-for-replay") m.setenv("GOOGLE_GENAI_CLIENT_MODE", "replay") m.setenv("DEFAULT_MODEL", env_updates["DEFAULT_MODEL"]) m.setenv("GOOGLE_ALLOWED_MODELS", "gemini-2.5-flash") m.setenv("OPENAI_ALLOWED_MODELS", "gpt-5") m.setenv("GOOGLE_GENAI_REPLAYS_DIRECTORY", str(GEMINI_CASSETTE_DIR)) m.setenv("GOOGLE_GENAI_REPLAY_ID", GEMINI_REPLAY_ID) for key in keys_to_clear: m.delenv(key, raising=False) ModelProviderRegistry.reset_for_testing() from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) inject_transport(monkeypatch, OPENAI_CASSETTE_PATH) chat_tool = ChatTool() step2_args = { "prompt": "Remind me, what number did you pick, respond with JUST that number.", "model": "gpt-5", "continuation_id": continuation_id, "temperature": 0.2, "working_directory_absolute_path": working_directory, } step2_result = await chat_tool.execute(step2_args) assert step2_result and step2_result[0].type == "text" step2_data = json.loads(step2_result[0].text) assert step2_data["status"] in {"success", "continuation_available"} assert step2_data.get("metadata", {}).get("provider_used") == "openai" recalled_number = _extract_number(step2_data["content"]) assert recalled_number == chosen_number assert OPENAI_CASSETTE_PATH.exists() ModelProviderRegistry.reset_for_testing() ================================================ FILE: tests/test_chat_openai_integration.py ================================================ """Integration test for ChatTool auto-mode using OpenAI o3/gpt models with cassette recording.""" from __future__ import annotations import json import os import uuid from pathlib import Path import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from tests.transport_helpers import inject_transport from tools.chat import ChatTool # Directory for recorded HTTP interactions CASSETTE_DIR = Path(__file__).parent / "openai_cassettes" CASSETTE_DIR.mkdir(exist_ok=True) CASSETTE_PATH = CASSETTE_DIR / "chat_gpt5_moon_distance.json" CASSETTE_CONTINUATION_PATH = CASSETTE_DIR / "chat_gpt5_continuation.json" @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_chat_auto_mode_with_openai(monkeypatch, tmp_path): """Ensure ChatTool in auto mode selects gpt-5 via OpenAI and returns a valid response.""" # Prepare environment so only OpenAI is available in auto mode env_updates = { "DEFAULT_MODEL": "auto", "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", ""), } # Remove Gemini/XAI keys to force OpenAI selection keys_to_clear = ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"] with monkeypatch.context() as m: m.setenv("DEFAULT_MODEL", env_updates["DEFAULT_MODEL"]) m.setenv("OPENAI_ALLOWED_MODELS", "gpt-5") if env_updates["OPENAI_API_KEY"]: m.setenv("OPENAI_API_KEY", env_updates["OPENAI_API_KEY"]) for key in keys_to_clear: m.delenv(key, raising=False) # Choose recording or replay mode based on cassette presence if not CASSETTE_PATH.exists(): real_key = os.getenv("OPENAI_API_KEY", "").strip() if not real_key or real_key.startswith("dummy"): pytest.skip( "Cassette missing and OPENAI_API_KEY not configured. Provide a real key and re-run to record." ) else: # Replay mode uses dummy key to keep secrets out of the cassette m.setenv("OPENAI_API_KEY", "dummy-key-for-replay") # Reset registry and register only OpenAI provider ModelProviderRegistry.reset_for_testing() from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Inject HTTP transport (records or replays depending on cassette state) inject_transport(monkeypatch, CASSETTE_PATH) # Execute ChatTool request targeting gpt-5 directly (server normally resolves auto→model) chat_tool = ChatTool() working_directory = str(tmp_path) arguments = { "prompt": "Use chat with gpt5 and ask how far the moon is from earth.", "model": "gpt-5", "temperature": 1.0, "working_directory_absolute_path": working_directory, } result = await chat_tool.execute(arguments) # Validate response assert result and result[0].type == "text" response_data = json.loads(result[0].text) assert response_data["status"] in {"success", "continuation_available"} metadata = response_data.get("metadata", {}) assert metadata.get("provider_used") == "openai" assert metadata.get("model_used") in {"gpt-5", "gpt5"} assert "moon" in response_data["content"].lower() # Ensure cassette recorded for future replays assert CASSETTE_PATH.exists() @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_chat_openai_continuation(monkeypatch, tmp_path): """Verify continuation_id workflow against gpt-5 using recorded OpenAI responses.""" env_updates = { "DEFAULT_MODEL": "auto", "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", ""), } keys_to_clear = ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"] recording_mode = not CASSETTE_CONTINUATION_PATH.exists() if recording_mode: real_key = env_updates["OPENAI_API_KEY"].strip() if not real_key or real_key.startswith("dummy"): pytest.skip("Continuation cassette missing and OPENAI_API_KEY not configured. Set a real key to record.") fixed_thread_id = uuid.UUID("95d60035-1aa3-4398-9936-fca71989d906") with monkeypatch.context() as m: m.setenv("DEFAULT_MODEL", env_updates["DEFAULT_MODEL"]) m.setenv("OPENAI_ALLOWED_MODELS", "gpt-5") if recording_mode: m.setenv("OPENAI_API_KEY", env_updates["OPENAI_API_KEY"]) else: m.setenv("OPENAI_API_KEY", "dummy-key-for-replay") for key in keys_to_clear: m.delenv(key, raising=False) ModelProviderRegistry.reset_for_testing() from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) inject_transport(monkeypatch, CASSETTE_CONTINUATION_PATH) from utils import conversation_memory m.setattr(conversation_memory.uuid, "uuid4", lambda: fixed_thread_id) chat_tool = ChatTool() working_directory = str(tmp_path) # First message: obtain continuation_id first_args = { "prompt": "In one word, which sells better: iOS app or macOS app?", "model": "gpt-5", "temperature": 1.0, "working_directory_absolute_path": working_directory, } first_result = await chat_tool.execute(first_args) assert first_result and first_result[0].type == "text" first_data = json.loads(first_result[0].text) assert first_data["status"] == "continuation_available" first_metadata = first_data.get("metadata", {}) assert first_metadata.get("provider_used") == "openai" assert first_metadata.get("model_used") in {"gpt-5", "gpt5"} continuation = first_data.get("continuation_offer") assert continuation is not None continuation_id = continuation.get("continuation_id") assert continuation_id # Second message using continuation_id (reuse same tool instance for clarity) second_args = { "prompt": "In one word then, SwiftUI or ReactNative?", "model": "gpt-5", "continuation_id": continuation_id, "temperature": 1.0, "working_directory_absolute_path": working_directory, } second_result = await chat_tool.execute(second_args) assert second_result and second_result[0].type == "text" second_data = json.loads(second_result[0].text) assert second_data["status"] in {"success", "continuation_available"} second_metadata = second_data.get("metadata", {}) assert second_metadata.get("provider_used") == "openai" assert second_metadata.get("model_used") in {"gpt-5", "gpt5"} assert second_metadata.get("conversation_ready") is True assert second_data.get("continuation_offer") is not None # Ensure the cassette file exists for future replays assert CASSETTE_CONTINUATION_PATH.exists() # Clean up registry state for subsequent tests ModelProviderRegistry.reset_for_testing() ================================================ FILE: tests/test_chat_simple.py ================================================ """ Tests for Chat tool - validating SimpleTool architecture This module contains unit tests to ensure that the Chat tool (now using SimpleTool architecture) maintains proper functionality. """ import json from types import SimpleNamespace from unittest.mock import patch import pytest from tools.chat import ChatRequest, ChatTool from tools.shared.exceptions import ToolExecutionError class TestChatTool: """Test suite for ChatSimple tool""" def setup_method(self): """Set up test fixtures""" self.tool = ChatTool() def test_tool_metadata(self): """Test that tool metadata matches requirements""" assert self.tool.get_name() == "chat" assert "collaborative thinking" in self.tool.get_description() assert self.tool.get_system_prompt() is not None assert self.tool.get_default_temperature() > 0 assert self.tool.get_model_category() is not None def test_schema_structure(self): """Test that schema has correct structure""" schema = self.tool.get_input_schema() # Basic schema structure assert schema["type"] == "object" assert "properties" in schema assert "required" in schema # Required fields assert "prompt" in schema["required"] assert "working_directory_absolute_path" in schema["required"] # Properties properties = schema["properties"] assert "prompt" in properties assert "absolute_file_paths" in properties assert "images" in properties assert "working_directory_absolute_path" in properties def test_request_model_validation(self): """Test that the request model validates correctly""" # Test valid request request_data = { "prompt": "Test prompt", "absolute_file_paths": ["test.txt"], "images": ["test.png"], "model": "anthropic/claude-opus-4.1", "temperature": 0.7, "working_directory_absolute_path": "/tmp", # Dummy absolute path } request = ChatRequest(**request_data) assert request.prompt == "Test prompt" assert request.absolute_file_paths == ["test.txt"] assert request.images == ["test.png"] assert request.model == "anthropic/claude-opus-4.1" assert request.temperature == 0.7 assert request.working_directory_absolute_path == "/tmp" def test_required_fields(self): """Test that required fields are enforced""" # Missing prompt should raise validation error from pydantic import ValidationError with pytest.raises(ValidationError): ChatRequest(model="anthropic/claude-opus-4.1", working_directory_absolute_path="/tmp") def test_model_availability(self): """Test that model availability works""" models = self.tool._get_available_models() assert len(models) > 0 # Should have some models assert isinstance(models, list) def test_model_field_schema(self): """Test that model field schema generation works correctly""" schema = self.tool.get_model_field_schema() assert schema["type"] == "string" assert "description" in schema # Description should route callers to listmodels, regardless of mode assert "listmodels" in schema["description"] if self.tool.is_effective_auto_mode(): assert "auto mode" in schema["description"].lower() else: import config assert f"'{config.DEFAULT_MODEL}'" in schema["description"] @pytest.mark.asyncio async def test_prompt_preparation(self): """Test that prompt preparation works correctly""" request = ChatRequest( prompt="Test prompt", absolute_file_paths=[], working_directory_absolute_path="/tmp", ) # Mock the system prompt and file handling with patch.object(self.tool, "get_system_prompt", return_value="System prompt"): with patch.object(self.tool, "handle_prompt_file_with_fallback", return_value="Test prompt"): with patch.object(self.tool, "_prepare_file_content_for_prompt", return_value=("", [])): with patch.object(self.tool, "_validate_token_limit"): with patch.object(self.tool, "get_websearch_instruction", return_value=""): prompt = await self.tool.prepare_prompt(request) assert "Test prompt" in prompt assert prompt.startswith("=== USER REQUEST ===") assert "System prompt" not in prompt def test_response_formatting(self): """Test that response formatting works correctly""" response = "Test response content" request = ChatRequest(prompt="Test", working_directory_absolute_path="/tmp") formatted = self.tool.format_response(response, request) assert "Test response content" in formatted assert "AGENT'S TURN:" in formatted assert "Evaluate this perspective" in formatted def test_format_response_multiple_generated_code_blocks(self, tmp_path): """All generated-code blocks should be combined and saved to pal_generated.code.""" tool = ChatTool() tool._model_context = SimpleNamespace(capabilities=SimpleNamespace(allow_code_generation=True)) response = ( "Intro text\n" "print('hello')\n" "Other text\n" "print('world')" ) request = ChatRequest(prompt="Test", working_directory_absolute_path=str(tmp_path)) formatted = tool.format_response(response, request) saved_path = tmp_path / "pal_generated.code" saved_content = saved_path.read_text(encoding="utf-8") assert "print('world')" in saved_content assert "print('hello')" not in saved_content assert saved_content.count("") == 1 assert "print('hello')" in formatted assert str(saved_path) in formatted def test_format_response_single_generated_code_block(self, tmp_path): """Single block should be saved and removed from narrative.""" tool = ChatTool() tool._model_context = SimpleNamespace(capabilities=SimpleNamespace(allow_code_generation=True)) response = ( "Intro text before code.\n" "print('only-once')\n" "Closing thoughts after code." ) request = ChatRequest(prompt="Test", working_directory_absolute_path=str(tmp_path)) formatted = tool.format_response(response, request) saved_path = tmp_path / "pal_generated.code" saved_content = saved_path.read_text(encoding="utf-8") assert "print('only-once')" in saved_content assert "" in saved_content assert "print('only-once')" not in formatted assert "Closing thoughts after code." in formatted def test_format_response_ignores_unclosed_generated_code(self, tmp_path): """Unclosed generated-code tags should be ignored to avoid accidental clipping.""" tool = ChatTool() tool._model_context = SimpleNamespace(capabilities=SimpleNamespace(allow_code_generation=True)) response = "Intro text\nprint('oops')\nStill ongoing" request = ChatRequest(prompt="Test", working_directory_absolute_path=str(tmp_path)) formatted = tool.format_response(response, request) saved_path = tmp_path / "pal_generated.code" assert not saved_path.exists() assert "print('oops')" in formatted def test_format_response_ignores_orphaned_closing_tag(self, tmp_path): """Stray closing tags should not trigger extraction.""" tool = ChatTool() tool._model_context = SimpleNamespace(capabilities=SimpleNamespace(allow_code_generation=True)) response = "Intro text\n just text" request = ChatRequest(prompt="Test", working_directory_absolute_path=str(tmp_path)) formatted = tool.format_response(response, request) saved_path = tmp_path / "pal_generated.code" assert not saved_path.exists() assert " just text" in formatted def test_format_response_preserves_narrative_after_generated_code(self, tmp_path): """Narrative content after generated code must remain intact in the formatted output.""" tool = ChatTool() tool._model_context = SimpleNamespace(capabilities=SimpleNamespace(allow_code_generation=True)) response = ( "Summary before code.\n" "print('demo')\n" "### Follow-up\n" "Further analysis and guidance after the generated snippet.\n" ) request = ChatRequest(prompt="Test", working_directory_absolute_path=str(tmp_path)) formatted = tool.format_response(response, request) assert "Summary before code." in formatted assert "### Follow-up" in formatted assert "Further analysis and guidance after the generated snippet." in formatted assert "print('demo')" not in formatted def test_tool_name(self): """Test tool name is correct""" assert self.tool.get_name() == "chat" def test_websearch_guidance(self): """Test web search guidance matches Chat tool style""" guidance = self.tool.get_websearch_guidance() chat_style_guidance = self.tool.get_chat_style_websearch_guidance() assert guidance == chat_style_guidance assert "Documentation for any technologies" in guidance assert "Current best practices" in guidance def test_convenience_methods(self): """Test SimpleTool convenience methods work correctly""" assert self.tool.supports_custom_request_model() # Test that the tool fields are defined correctly tool_fields = self.tool.get_tool_fields() assert "prompt" in tool_fields assert "absolute_file_paths" in tool_fields assert "images" in tool_fields required_fields = self.tool.get_required_fields() assert "prompt" in required_fields assert "working_directory_absolute_path" in required_fields class TestChatRequestModel: """Test suite for ChatRequest model""" def test_field_descriptions(self): """Test that field descriptions are proper""" from tools.chat import CHAT_FIELD_DESCRIPTIONS # Field descriptions should exist and be descriptive assert len(CHAT_FIELD_DESCRIPTIONS["prompt"]) > 50 assert "context" in CHAT_FIELD_DESCRIPTIONS["prompt"] files_desc = CHAT_FIELD_DESCRIPTIONS["absolute_file_paths"].lower() assert "absolute" in files_desc assert "visual context" in CHAT_FIELD_DESCRIPTIONS["images"] assert "directory" in CHAT_FIELD_DESCRIPTIONS["working_directory_absolute_path"].lower() def test_working_directory_absolute_path_description_matches_behavior(self): """Absolute working directory description should reflect existing-directory requirement.""" from tools.chat import CHAT_FIELD_DESCRIPTIONS description = CHAT_FIELD_DESCRIPTIONS["working_directory_absolute_path"].lower() assert "existing directory" in description @pytest.mark.asyncio async def test_working_directory_absolute_path_must_exist(self, tmp_path): """Chat tool should reject non-existent working directories.""" tool = ChatTool() missing_dir = tmp_path / "nonexistent_subdir" with pytest.raises(ToolExecutionError) as exc_info: await tool.execute( { "prompt": "test", "absolute_file_paths": [], "images": [], "working_directory_absolute_path": str(missing_dir), } ) payload = json.loads(exc_info.value.payload) assert payload["status"] == "error" assert "existing directory" in payload["content"].lower() def test_default_values(self): """Test that default values work correctly""" request = ChatRequest(prompt="Test", working_directory_absolute_path="/tmp") assert request.prompt == "Test" assert request.absolute_file_paths == [] # Should default to empty list assert request.images == [] # Should default to empty list def test_inheritance(self): """Test that ChatRequest properly inherits from ToolRequest""" from tools.shared.base_models import ToolRequest request = ChatRequest(prompt="Test", working_directory_absolute_path="/tmp") assert isinstance(request, ToolRequest) # Should have inherited fields assert hasattr(request, "model") assert hasattr(request, "temperature") assert hasattr(request, "thinking_mode") assert hasattr(request, "continuation_id") assert hasattr(request, "images") # From base model too if __name__ == "__main__": pytest.main([__file__]) ================================================ FILE: tests/test_clink_claude_agent.py ================================================ import asyncio import json import shutil from pathlib import Path import pytest from clink.agents.base import CLIAgentError from clink.agents.claude import ClaudeAgent from clink.models import ResolvedCLIClient, ResolvedCLIRole class DummyProcess: def __init__(self, *, stdout: bytes = b"", stderr: bytes = b"", returncode: int = 0): self._stdout = stdout self._stderr = stderr self.returncode = returncode self.stdin_data: bytes | None = None async def communicate(self, input_data): self.stdin_data = input_data return self._stdout, self._stderr @pytest.fixture() def claude_agent(): prompt_path = Path("systemprompts/clink/default.txt").resolve() role = ResolvedCLIRole(name="default", prompt_path=prompt_path, role_args=[]) client = ResolvedCLIClient( name="claude", executable=["claude"], internal_args=["--print", "--output-format", "json"], config_args=["--permission-mode", "acceptEdits"], env={}, timeout_seconds=30, parser="claude_json", runner="claude", roles={"default": role}, output_to_file=None, working_dir=None, ) return ClaudeAgent(client), role async def _run_agent_with_process(monkeypatch, agent, role, process, *, system_prompt="System prompt"): async def fake_create_subprocess_exec(*_args, **_kwargs): return process def fake_which(executable_name): return f"/usr/bin/{executable_name}" monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec) monkeypatch.setattr(shutil, "which", fake_which) return await agent.run( role=role, prompt="Respond with 42", system_prompt=system_prompt, files=[], images=[], ) @pytest.mark.asyncio async def test_claude_agent_injects_system_prompt(monkeypatch, claude_agent): agent, role = claude_agent stdout_payload = json.dumps( { "type": "result", "subtype": "success", "is_error": False, "result": "42", } ).encode() process = DummyProcess(stdout=stdout_payload) result = await _run_agent_with_process(monkeypatch, agent, role, process) assert "--append-system-prompt" in result.sanitized_command idx = result.sanitized_command.index("--append-system-prompt") assert result.sanitized_command[idx + 1] == "System prompt" assert process.stdin_data.decode().startswith("Respond with 42") @pytest.mark.asyncio async def test_claude_agent_recovers_error_payload(monkeypatch, claude_agent): agent, role = claude_agent stdout_payload = json.dumps( { "type": "result", "subtype": "success", "is_error": True, "result": "API Error", } ).encode() process = DummyProcess(stdout=stdout_payload, returncode=2) result = await _run_agent_with_process(monkeypatch, agent, role, process) assert result.returncode == 2 assert result.parsed.content == "API Error" assert result.parsed.metadata["is_error"] is True @pytest.mark.asyncio async def test_claude_agent_propagates_unparseable_output(monkeypatch, claude_agent): agent, role = claude_agent process = DummyProcess(stdout=b"", returncode=1) with pytest.raises(CLIAgentError): await _run_agent_with_process(monkeypatch, agent, role, process) ================================================ FILE: tests/test_clink_claude_parser.py ================================================ """Tests for the Claude CLI JSON parser.""" import json import pytest from clink.parsers.base import ParserError from clink.parsers.claude import ClaudeJSONParser def _build_success_payload() -> str: return ( '{"type":"result","subtype":"success","is_error":false,"duration_ms":1234,' '"duration_api_ms":1200,"num_turns":1,"result":"42","session_id":"abc","total_cost_usd":0.12,' '"usage":{"input_tokens":10,"output_tokens":5},' '"modelUsage":{"claude-sonnet-4-5-20250929":{"inputTokens":10,"outputTokens":5}}}' ) def test_claude_parser_extracts_result_and_metadata(): parser = ClaudeJSONParser() stdout = _build_success_payload() parsed = parser.parse(stdout=stdout, stderr="") assert parsed.content == "42" assert parsed.metadata["model_used"] == "claude-sonnet-4-5-20250929" assert parsed.metadata["usage"]["output_tokens"] == 5 assert parsed.metadata["is_error"] is False def test_claude_parser_falls_back_to_message(): parser = ClaudeJSONParser() stdout = '{"type":"result","is_error":true,"message":"API error message"}' parsed = parser.parse(stdout=stdout, stderr="warning") assert parsed.content == "API error message" assert parsed.metadata["is_error"] is True assert parsed.metadata["stderr"] == "warning" def test_claude_parser_requires_output(): parser = ClaudeJSONParser() with pytest.raises(ParserError): parser.parse(stdout="", stderr="") def test_claude_parser_handles_array_payload_with_result_event(): parser = ClaudeJSONParser() events = [ {"type": "system", "session_id": "abc"}, {"type": "assistant", "message": "intermediate"}, { "type": "result", "subtype": "success", "result": "42", "duration_api_ms": 9876, "usage": {"input_tokens": 12, "output_tokens": 3}, }, ] stdout = json.dumps(events) parsed = parser.parse(stdout=stdout, stderr="warning") assert parsed.content == "42" assert parsed.metadata["duration_api_ms"] == 9876 assert parsed.metadata["raw_events"] == events assert parsed.metadata["raw"] == events ================================================ FILE: tests/test_clink_codex_agent.py ================================================ import asyncio import shutil from pathlib import Path import pytest from clink.agents.base import CLIAgentError from clink.agents.codex import CodexAgent from clink.models import ResolvedCLIClient, ResolvedCLIRole class DummyProcess: def __init__(self, *, stdout: bytes = b"", stderr: bytes = b"", returncode: int = 0): self._stdout = stdout self._stderr = stderr self.returncode = returncode async def communicate(self, _input): return self._stdout, self._stderr @pytest.fixture() def codex_agent(): prompt_path = Path("systemprompts/clink/codex_default.txt").resolve() role = ResolvedCLIRole(name="default", prompt_path=prompt_path, role_args=[]) client = ResolvedCLIClient( name="codex", executable=["codex"], internal_args=["exec"], config_args=["--json", "--dangerously-bypass-approvals-and-sandbox"], env={}, timeout_seconds=30, parser="codex_jsonl", roles={"default": role}, output_to_file=None, working_dir=None, ) return CodexAgent(client), role async def _run_agent_with_process(monkeypatch, agent, role, process): async def fake_create_subprocess_exec(*_args, **_kwargs): return process def fake_which(executable_name): return f"/usr/bin/{executable_name}" monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec) monkeypatch.setattr(shutil, "which", fake_which) return await agent.run(role=role, prompt="do something", files=[], images=[]) @pytest.mark.asyncio async def test_codex_agent_recovers_jsonl(monkeypatch, codex_agent): agent, role = codex_agent stdout = b""" {"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"Hello from Codex"}} {"type":"turn.completed","usage":{"input_tokens":10,"output_tokens":5}} """ process = DummyProcess(stdout=stdout, returncode=124) result = await _run_agent_with_process(monkeypatch, agent, role, process) assert result.returncode == 124 assert "Hello from Codex" in result.parsed.content assert result.parsed.metadata["usage"]["output_tokens"] == 5 @pytest.mark.asyncio async def test_codex_agent_propagates_invalid_json(monkeypatch, codex_agent): agent, role = codex_agent stdout = b"not json" process = DummyProcess(stdout=stdout, returncode=1) with pytest.raises(CLIAgentError): await _run_agent_with_process(monkeypatch, agent, role, process) ================================================ FILE: tests/test_clink_gemini_agent.py ================================================ import asyncio import shutil from pathlib import Path import pytest from clink.agents.base import CLIAgentError from clink.agents.gemini import GeminiAgent from clink.models import ResolvedCLIClient, ResolvedCLIRole class DummyProcess: def __init__(self, *, stdout: bytes = b"", stderr: bytes = b"", returncode: int = 0): self._stdout = stdout self._stderr = stderr self.returncode = returncode async def communicate(self, _input): return self._stdout, self._stderr @pytest.fixture() def gemini_agent(): prompt_path = Path("systemprompts/clink/gemini_default.txt").resolve() role = ResolvedCLIRole(name="default", prompt_path=prompt_path, role_args=[]) client = ResolvedCLIClient( name="gemini", executable=["gemini"], internal_args=[], config_args=[], env={}, timeout_seconds=30, parser="gemini_json", roles={"default": role}, output_to_file=None, working_dir=None, ) return GeminiAgent(client), role async def _run_agent_with_process(monkeypatch, agent, role, process): async def fake_create_subprocess_exec(*_args, **_kwargs): return process def fake_which(executable_name): return f"/usr/bin/{executable_name}" monkeypatch.setattr(asyncio, "create_subprocess_exec", fake_create_subprocess_exec) monkeypatch.setattr(shutil, "which", fake_which) return await agent.run(role=role, prompt="do something", files=[], images=[]) @pytest.mark.asyncio async def test_gemini_agent_recovers_tool_error(monkeypatch, gemini_agent): agent, role = gemini_agent error_json = """{ "error": { "type": "FatalToolExecutionError", "message": "Error executing tool replace: Failed to edit", "code": "edit_expected_occurrence_mismatch" } }""" stderr = ("Error: Failed to edit, expected 1 occurrence but found 2.\n" + error_json).encode() process = DummyProcess(stderr=stderr, returncode=54) result = await _run_agent_with_process(monkeypatch, agent, role, process) assert result.returncode == 54 assert result.parsed.metadata["cli_error_recovered"] is True assert result.parsed.metadata["cli_error_code"] == "edit_expected_occurrence_mismatch" assert "Gemini CLI reported a tool failure" in result.parsed.content @pytest.mark.asyncio async def test_gemini_agent_propagates_unrecoverable_error(monkeypatch, gemini_agent): agent, role = gemini_agent stderr = b"Plain failure without structured payload" process = DummyProcess(stderr=stderr, returncode=54) with pytest.raises(CLIAgentError): await _run_agent_with_process(monkeypatch, agent, role, process) ================================================ FILE: tests/test_clink_gemini_parser.py ================================================ """Tests for the Gemini CLI JSON parser.""" import pytest from clink.parsers.gemini import GeminiJSONParser, ParserError def _build_rate_limit_stdout() -> str: return ( "{\n" ' "response": "",\n' ' "stats": {\n' ' "models": {\n' ' "gemini-2.5-pro": {\n' ' "api": {\n' ' "totalRequests": 5,\n' ' "totalErrors": 5,\n' ' "totalLatencyMs": 13319\n' " },\n" ' "tokens": {"prompt": 0, "candidates": 0, "total": 0, "cached": 0, "thoughts": 0, "tool": 0}\n' " }\n" " },\n" ' "tools": {"totalCalls": 0},\n' ' "files": {"totalLinesAdded": 0, "totalLinesRemoved": 0}\n' " }\n" "}" ) def test_gemini_parser_handles_rate_limit_empty_response(): parser = GeminiJSONParser() stdout = _build_rate_limit_stdout() stderr = "Attempt 1 failed with status 429. Retrying with backoff... ApiError: quota exceeded" parsed = parser.parse(stdout, stderr) assert "429" in parsed.content assert parsed.metadata.get("rate_limit_status") == 429 assert parsed.metadata.get("empty_response") is True assert "Attempt 1 failed" in parsed.metadata.get("stderr", "") def test_gemini_parser_still_errors_when_no_fallback_available(): parser = GeminiJSONParser() stdout = '{"response": "", "stats": {}}' with pytest.raises(ParserError): parser.parse(stdout, stderr="") ================================================ FILE: tests/test_clink_integration.py ================================================ import json import os import shutil import pytest from tools.clink import CLinkTool @pytest.mark.integration @pytest.mark.asyncio async def test_clink_gemini_single_digit_sum(): if shutil.which("gemini") is None: pytest.skip("gemini CLI is not installed or on PATH") if not (os.getenv("GEMINI_API_KEY") or os.getenv("GOOGLE_API_KEY")): pytest.skip("Gemini API key is not configured") tool = CLinkTool() prompt = "Respond with a single digit equal to the sum of 2 + 2. Output only that digit." results = await tool.execute( { "prompt": prompt, "cli_name": "gemini", "role": "default", "absolute_file_paths": [], "images": [], } ) assert results, "clink tool returned no outputs" payload = json.loads(results[0].text) status = payload["status"] assert status in {"success", "continuation_available"} content = payload.get("content", "").strip() # CLI may include additional metadata like tags; check first line or that "4" is present first_line = content.split("\n")[0].strip() assert first_line == "4" or "4" in content, f"Expected '4' in response, got: {content[:100]}" if status == "continuation_available": offer = payload.get("continuation_offer") or {} assert offer.get("continuation_id"), "Expected continuation metadata when status indicates availability" @pytest.mark.integration @pytest.mark.asyncio async def test_clink_claude_single_digit_sum(): if shutil.which("claude") is None: pytest.skip("claude CLI is not installed or on PATH") tool = CLinkTool() prompt = "Respond with a single digit equal to the sum of 2 + 2. Output only that digit." results = await tool.execute( { "prompt": prompt, "cli_name": "claude", "role": "default", "absolute_file_paths": [], "images": [], } ) assert results, "clink tool returned no outputs" payload = json.loads(results[0].text) status = payload["status"] if status == "error": metadata = payload.get("metadata") or {} reason = payload.get("content") or metadata.get("message") or "Claude CLI reported an error" pytest.skip(f"Skipping Claude integration test: {reason}") assert status in {"success", "continuation_available"} content = payload.get("content", "").strip() assert content == "4" if status == "continuation_available": offer = payload.get("continuation_offer") or {} assert offer.get("continuation_id"), "Expected continuation metadata when status indicates availability" ================================================ FILE: tests/test_clink_parsers.py ================================================ import pytest from clink.parsers.base import ParserError from clink.parsers.codex import CodexJSONLParser def test_codex_parser_success(): parser = CodexJSONLParser() stdout = """ {"type":"item.completed","item":{"id":"item_0","type":"agent_message","text":"Hello"}} {"type":"turn.completed","usage":{"input_tokens":10,"output_tokens":5}} """ parsed = parser.parse(stdout=stdout, stderr="") assert parsed.content == "Hello" assert parsed.metadata["usage"]["output_tokens"] == 5 def test_codex_parser_requires_agent_message(): parser = CodexJSONLParser() stdout = '{"type":"turn.completed"}' with pytest.raises(ParserError): parser.parse(stdout=stdout, stderr="") ================================================ FILE: tests/test_clink_tool.py ================================================ import json import pytest from clink import get_registry from clink.agents import AgentOutput from clink.parsers.base import ParsedCLIResponse from tools.clink import MAX_RESPONSE_CHARS, CLinkTool @pytest.mark.asyncio async def test_clink_tool_execute(monkeypatch): tool = CLinkTool() async def fake_run(**kwargs): return AgentOutput( parsed=ParsedCLIResponse(content="Hello from Gemini", metadata={"model_used": "gemini-2.5-pro"}), sanitized_command=["gemini", "-o", "json"], returncode=0, stdout='{"response": "Hello from Gemini"}', stderr="", duration_seconds=0.42, parser_name="gemini_json", output_file_content=None, ) class DummyAgent: async def run(self, **kwargs): return await fake_run(**kwargs) def fake_create_agent(client): return DummyAgent() monkeypatch.setattr("tools.clink.create_agent", fake_create_agent) arguments = { "prompt": "Summarize the project", "cli_name": "gemini", "role": "default", "absolute_file_paths": [], "images": [], } results = await tool.execute(arguments) assert len(results) == 1 payload = json.loads(results[0].text) assert payload["status"] in {"success", "continuation_available"} assert "Hello from Gemini" in payload["content"] metadata = payload.get("metadata", {}) assert metadata.get("cli_name") == "gemini" assert metadata.get("command") == ["gemini", "-o", "json"] def test_registry_lists_roles(): registry = get_registry() clients = registry.list_clients() assert {"codex", "gemini"}.issubset(set(clients)) roles = registry.list_roles("gemini") assert "default" in roles assert "default" in registry.list_roles("codex") codex_client = registry.get_client("codex") # Verify codex uses --enable web_search_request (not --search which is unsupported by exec) assert codex_client.config_args == [ "--json", "--dangerously-bypass-approvals-and-sandbox", "--enable", "web_search_request", ] @pytest.mark.asyncio async def test_clink_tool_defaults_to_first_cli(monkeypatch): tool = CLinkTool() async def fake_run(**kwargs): return AgentOutput( parsed=ParsedCLIResponse(content="Default CLI response", metadata={"events": ["foo"]}), sanitized_command=["gemini"], returncode=0, stdout='{"response": "Default CLI response"}', stderr="", duration_seconds=0.1, parser_name="gemini_json", output_file_content=None, ) class DummyAgent: async def run(self, **kwargs): return await fake_run(**kwargs) monkeypatch.setattr("tools.clink.create_agent", lambda client: DummyAgent()) arguments = { "prompt": "Hello", "absolute_file_paths": [], "images": [], } result = await tool.execute(arguments) payload = json.loads(result[0].text) metadata = payload.get("metadata", {}) assert metadata.get("cli_name") == tool._default_cli_name assert metadata.get("events_removed_for_normal") is True @pytest.mark.asyncio async def test_clink_tool_truncates_large_output(monkeypatch): tool = CLinkTool() summary_section = "This is the condensed summary." long_text = "A" * (MAX_RESPONSE_CHARS + 500) + summary_section async def fake_run(**kwargs): return AgentOutput( parsed=ParsedCLIResponse(content=long_text, metadata={"events": ["event1", "event2"]}), sanitized_command=["codex"], returncode=0, stdout="{}", stderr="", duration_seconds=0.2, parser_name="codex_jsonl", output_file_content=None, ) class DummyAgent: async def run(self, **kwargs): return await fake_run(**kwargs) monkeypatch.setattr("tools.clink.create_agent", lambda client: DummyAgent()) arguments = { "prompt": "Summarize", "cli_name": tool._default_cli_name, "absolute_file_paths": [], "images": [], } result = await tool.execute(arguments) payload = json.loads(result[0].text) assert payload["status"] in {"success", "continuation_available"} assert payload["content"].strip() == "This is the condensed summary." metadata = payload.get("metadata", {}) assert metadata.get("output_summarized") is True assert metadata.get("events_removed_for_normal") is True assert metadata.get("output_original_length") == len(long_text) @pytest.mark.asyncio async def test_clink_tool_truncates_without_summary(monkeypatch): tool = CLinkTool() long_text = "B" * (MAX_RESPONSE_CHARS + 1000) async def fake_run(**kwargs): return AgentOutput( parsed=ParsedCLIResponse(content=long_text, metadata={"events": ["event"]}), sanitized_command=["codex"], returncode=0, stdout="{}", stderr="", duration_seconds=0.2, parser_name="codex_jsonl", output_file_content=None, ) class DummyAgent: async def run(self, **kwargs): return await fake_run(**kwargs) monkeypatch.setattr("tools.clink.create_agent", lambda client: DummyAgent()) arguments = { "prompt": "Summarize", "cli_name": tool._default_cli_name, "absolute_file_paths": [], "images": [], } result = await tool.execute(arguments) payload = json.loads(result[0].text) assert payload["status"] in {"success", "continuation_available"} assert "exceeding the configured clink limit" in payload["content"] metadata = payload.get("metadata", {}) assert metadata.get("output_truncated") is True assert metadata.get("events_removed_for_normal") is True assert metadata.get("output_original_length") == len(long_text) ================================================ FILE: tests/test_collaboration.py ================================================ """ Tests for dynamic context request and collaboration features """ import json import os from unittest.mock import Mock, patch import pytest from tests.mock_helpers import create_mock_provider from tools.analyze import AnalyzeTool from tools.debug import DebugIssueTool from tools.models import FilesNeededRequest, ToolOutput class TestDynamicContextRequests: """Test the dynamic context request mechanism""" @pytest.fixture def analyze_tool(self): return AnalyzeTool() @pytest.fixture def debug_tool(self): return DebugIssueTool() @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") async def test_clarification_request_parsing(self, mock_get_provider, analyze_tool): """Test that tools correctly parse clarification requests""" # Mock model to return a clarification request clarification_json = json.dumps( { "status": "files_required_to_continue", "mandatory_instructions": "I need to see the package.json file to understand dependencies", "files_needed": ["package.json", "package-lock.json"], }, ensure_ascii=False, ) mock_provider = create_mock_provider() mock_provider.get_provider_type.return_value = Mock(value="google") mock_provider.generate_content.return_value = Mock( content=clarification_json, usage={}, model_name="gemini-2.5-flash", metadata={} ) mock_get_provider.return_value = mock_provider result = await analyze_tool.execute( { "step": "Analyze the dependencies used in this project", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial dependency analysis", "relevant_files": ["/absolute/path/src/index.js"], } ) assert len(result) == 1 # Parse the response - analyze tool now uses workflow architecture response_data = json.loads(result[0].text) # Workflow tools may handle provider errors differently than simple tools # They might return error, expert analysis, or clarification requests assert response_data["status"] in ["calling_expert_analysis", "error", "files_required_to_continue"] # Check that expert analysis was performed and contains the clarification if "expert_analysis" in response_data: expert_analysis = response_data["expert_analysis"] # The mock should have returned the clarification JSON if "raw_analysis" in expert_analysis: analysis_content = expert_analysis["raw_analysis"] assert "package.json" in analysis_content assert "dependencies" in analysis_content # For workflow tools, the files_needed logic is handled differently # The test validates that the mocked clarification content was processed assert "step_number" in response_data assert response_data["step_number"] == 1 @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") @patch("utils.conversation_memory.create_thread", return_value="debug-test-uuid") @patch("utils.conversation_memory.add_turn") async def test_normal_response_not_parsed_as_clarification( self, mock_add_turn, mock_create_thread, mock_get_provider, debug_tool ): """Test that normal investigation responses work correctly with new debug tool""" # The new debug tool uses self-investigation pattern result = await debug_tool.execute( { "step": "Investigating NameError: name 'utils' is not defined", "step_number": 1, "total_steps": 3, "next_step_required": True, "findings": "The error indicates 'utils' module is not imported or defined", "files_checked": ["/code/main.py"], "relevant_files": ["/code/main.py"], "hypothesis": "Missing import statement for utils module", "confidence": "high", } ) assert len(result) == 1 # Parse the response - new debug tool returns structured JSON response_data = json.loads(result[0].text) # Debug tool now returns "pause_for_investigation" to force actual investigation assert response_data["status"] == "pause_for_investigation" assert response_data["step_number"] == 1 assert response_data["next_step_required"] is True assert response_data["investigation_status"]["current_confidence"] == "high" assert response_data["investigation_required"] is True assert "required_actions" in response_data @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") async def test_malformed_clarification_request_treated_as_normal(self, mock_get_provider, analyze_tool): """Test that malformed JSON clarification requests are treated as normal responses""" malformed_json = '{"status": "files_required_to_continue", "prompt": "Missing closing brace"' mock_provider = create_mock_provider() mock_provider.get_provider_type.return_value = Mock(value="google") mock_provider.generate_content.return_value = Mock( content=malformed_json, usage={}, model_name="gemini-2.5-flash", metadata={} ) mock_get_provider.return_value = mock_provider result = await analyze_tool.execute( { "step": "What does this do?", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial code analysis", "relevant_files": ["/absolute/path/test.py"], } ) assert len(result) == 1 # Should be treated as normal response due to JSON parse error response_data = json.loads(result[0].text) # Workflow tools may handle provider errors differently than simple tools # They might return error, expert analysis, or clarification requests assert response_data["status"] in ["calling_expert_analysis", "error", "files_required_to_continue"] # The malformed JSON should appear in the expert analysis content if "expert_analysis" in response_data: expert_analysis = response_data["expert_analysis"] if "raw_analysis" in expert_analysis: analysis_content = expert_analysis["raw_analysis"] # The malformed JSON should be included in the analysis assert "files_required_to_continue" in analysis_content or malformed_json in str(response_data) @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") async def test_clarification_with_suggested_action(self, mock_get_provider, analyze_tool): """Test clarification request with suggested next action""" import importlib from providers.registry import ModelProviderRegistry # Ensure deterministic model configuration for this test regardless of previous suites ModelProviderRegistry.reset_for_testing() original_default = os.environ.get("DEFAULT_MODEL") try: os.environ["DEFAULT_MODEL"] = "gemini-2.5-flash" import config importlib.reload(config) clarification_json = json.dumps( { "status": "files_required_to_continue", "mandatory_instructions": "I need to see the database configuration to analyze the connection error", "files_needed": ["config/database.yml", "src/db.py"], "suggested_next_action": { "tool": "analyze", "args": { "prompt": "Analyze database connection timeout issue", "relevant_files": [ "/config/database.yml", "/src/db.py", "/logs/error.log", ], }, }, }, ensure_ascii=False, ) mock_provider = create_mock_provider() mock_provider.get_provider_type.return_value = Mock(value="google") mock_provider.generate_content.return_value = Mock( content=clarification_json, usage={}, model_name="gemini-2.5-flash", metadata={} ) mock_get_provider.return_value = mock_provider result = await analyze_tool.execute( { "step": "Analyze database connection timeout issue", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial database timeout analysis", "relevant_files": ["/absolute/logs/error.log"], } ) assert len(result) == 1 response_data = json.loads(result[0].text) # Workflow tools should either promote clarification status or handle it in expert analysis if response_data["status"] == "files_required_to_continue": # Clarification was properly promoted to main status # Check if mandatory_instructions is at top level or in content if "mandatory_instructions" in response_data: assert "database configuration" in response_data["mandatory_instructions"] assert "files_needed" in response_data assert "config/database.yml" in response_data["files_needed"] assert "src/db.py" in response_data["files_needed"] elif "content" in response_data: # Parse content JSON for workflow tools try: content_json = json.loads(response_data["content"]) assert "mandatory_instructions" in content_json assert ( "database configuration" in content_json["mandatory_instructions"] or "database" in content_json["mandatory_instructions"] ) assert "files_needed" in content_json files_needed_str = str(content_json["files_needed"]) assert ( "config/database.yml" in files_needed_str or "config" in files_needed_str or "database" in files_needed_str ) except json.JSONDecodeError: # Content is not JSON, check if it contains required text content = response_data["content"] assert "database configuration" in content or "config" in content elif response_data["status"] == "calling_expert_analysis": # Clarification may be handled in expert analysis section if "expert_analysis" in response_data: expert_analysis = response_data["expert_analysis"] expert_content = str(expert_analysis) assert ( "database configuration" in expert_content or "config/database.yml" in expert_content or "files_required_to_continue" in expert_content ) else: # Some other status - ensure it's a valid workflow response assert "step_number" in response_data # Check for suggested next action if "suggested_next_action" in response_data: action = response_data["suggested_next_action"] assert action["tool"] == "analyze" finally: if original_default is not None: os.environ["DEFAULT_MODEL"] = original_default else: os.environ.pop("DEFAULT_MODEL", None) import config importlib.reload(config) ModelProviderRegistry.reset_for_testing() def test_tool_output_model_serialization(self): """Test ToolOutput model serialization""" output = ToolOutput( status="success", content="Test content", content_type="markdown", metadata={"tool_name": "test", "execution_time": 1.5}, ) json_str = output.model_dump_json() parsed = json.loads(json_str) assert parsed["status"] == "success" assert parsed["content"] == "Test content" assert parsed["content_type"] == "markdown" assert parsed["metadata"]["tool_name"] == "test" def test_clarification_request_model(self): """Test FilesNeededRequest model""" request = FilesNeededRequest( mandatory_instructions="Need more context", files_needed=["file1.py", "file2.py"], suggested_next_action={"tool": "analyze", "args": {}}, ) assert request.mandatory_instructions == "Need more context" assert len(request.files_needed) == 2 assert request.suggested_next_action["tool"] == "analyze" @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") async def test_error_response_format(self, mock_get_provider, analyze_tool): """Test error response format""" mock_get_provider.side_effect = Exception("API connection failed") result = await analyze_tool.execute( { "step": "Analyze this", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial analysis", "relevant_files": ["/absolute/path/test.py"], } ) assert len(result) == 1 response_data = json.loads(result[0].text) # Workflow tools may handle provider errors differently than simple tools # They might return error, complete analysis, or even clarification requests assert response_data["status"] in ["error", "calling_expert_analysis", "files_required_to_continue"] # If expert analysis was attempted, it may succeed or fail if response_data["status"] == "calling_expert_analysis" and "expert_analysis" in response_data: expert_analysis = response_data["expert_analysis"] # Could be an error or a successful analysis that requests clarification analysis_status = expert_analysis.get("status", "") assert ( analysis_status in ["analysis_error", "analysis_complete"] or "error" in expert_analysis or "files_required_to_continue" in str(expert_analysis) ) elif response_data["status"] == "error": assert "content" in response_data assert response_data["content_type"] == "text" class TestCollaborationWorkflow: """Test complete collaboration workflows""" def teardown_method(self): """Clean up after each test to prevent state pollution.""" # Clear provider registry singleton from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") @patch("tools.workflow.workflow_mixin.BaseWorkflowMixin._call_expert_analysis") async def test_dependency_analysis_triggers_clarification(self, mock_expert_analysis, mock_get_provider): """Test that asking about dependencies without package files triggers clarification""" tool = AnalyzeTool() # Mock Gemini to request package.json when asked about dependencies clarification_json = json.dumps( { "status": "files_required_to_continue", "mandatory_instructions": "I need to see the package.json file to analyze npm dependencies", "files_needed": ["package.json", "package-lock.json"], }, ensure_ascii=False, ) mock_provider = create_mock_provider() mock_provider.get_provider_type.return_value = Mock(value="google") mock_provider.generate_content.return_value = Mock( content=clarification_json, usage={}, model_name="gemini-2.5-flash", metadata={} ) mock_get_provider.return_value = mock_provider # Mock expert analysis to avoid actual API calls mock_expert_analysis.return_value = { "status": "analysis_complete", "raw_analysis": "I need to see the package.json file to analyze npm dependencies", } # Ask about dependencies with only source files (using new workflow format) result = await tool.execute( { "step": "What npm packages and versions does this project use?", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial dependency analysis", "relevant_files": ["/absolute/path/src/index.js"], } ) response = json.loads(result[0].text) # Workflow tools should either promote clarification status or handle it in expert analysis if response["status"] == "files_required_to_continue": # Clarification was properly promoted to main status assert "mandatory_instructions" in response assert "package.json" in response["mandatory_instructions"] assert "files_needed" in response assert "package.json" in response["files_needed"] assert "package-lock.json" in response["files_needed"] elif response["status"] == "calling_expert_analysis": # Clarification may be handled in expert analysis section if "expert_analysis" in response: expert_analysis = response["expert_analysis"] expert_content = str(expert_analysis) assert ( "package.json" in expert_content or "dependencies" in expert_content or "files_required_to_continue" in expert_content ) else: # Some other status - ensure it's a valid workflow response assert "step_number" in response @pytest.mark.asyncio @patch("tools.shared.base_tool.BaseTool.get_model_provider") @patch("tools.workflow.workflow_mixin.BaseWorkflowMixin._call_expert_analysis") async def test_multi_step_collaboration(self, mock_expert_analysis, mock_get_provider): """Test a multi-step collaboration workflow""" tool = AnalyzeTool() # Step 1: Initial request returns clarification needed clarification_json = json.dumps( { "status": "files_required_to_continue", "mandatory_instructions": "I need to see the configuration file to understand the connection settings", "files_needed": ["config.py"], }, ensure_ascii=False, ) mock_provider = create_mock_provider() mock_provider.get_provider_type.return_value = Mock(value="google") mock_provider.generate_content.return_value = Mock( content=clarification_json, usage={}, model_name="gemini-2.5-flash", metadata={} ) mock_get_provider.return_value = mock_provider # Mock expert analysis to avoid actual API calls mock_expert_analysis.return_value = { "status": "analysis_complete", "raw_analysis": "I need to see the configuration file to understand the database connection settings", } result1 = await tool.execute( { "step": "Analyze database connection timeout issue", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial database timeout analysis", "relevant_files": ["/logs/error.log"], } ) response1 = json.loads(result1[0].text) # First call should either return clarification request or handle it in expert analysis if response1["status"] == "files_required_to_continue": # Clarification was properly promoted to main status pass # This is the expected behavior elif response1["status"] == "calling_expert_analysis": # Clarification may be handled in expert analysis section if "expert_analysis" in response1: expert_analysis = response1["expert_analysis"] expert_content = str(expert_analysis) # Should contain some indication of clarification request assert ( "config" in expert_content or "files_required_to_continue" in expert_content or "database" in expert_content ) else: # Some other status - ensure it's a valid workflow response assert "step_number" in response1 # Step 2: Claude would provide additional context and re-invoke # This simulates the second call with more context final_response = """ ## Summary The database connection timeout is caused by incorrect host configuration. ## Hypotheses (Ranked by Likelihood) ### 1. Incorrect Database Host (Confidence: High) **Root Cause:** The config.py file shows the database host is set to 'localhost' but the database is running on a different server. """ mock_provider.generate_content.return_value = Mock( content=final_response, usage={}, model_name="gemini-2.5-flash", metadata={} ) # Update expert analysis mock for second call mock_expert_analysis.return_value = { "status": "analysis_complete", "raw_analysis": final_response, } result2 = await tool.execute( { "step": "Analyze database connection timeout issue with config file", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Analysis with configuration context", "relevant_files": ["/absolute/path/config.py", "/logs/error.log"], # Additional context provided } ) response2 = json.loads(result2[0].text) # Workflow tools should either return expert analysis or handle clarification properly # Accept multiple valid statuses as the workflow can handle the additional context differently # Include 'error' status in case API calls fail in test environment assert response2["status"] in [ "calling_expert_analysis", "files_required_to_continue", "pause_for_analysis", "error", ] # Check that the response contains the expected content regardless of status # If expert analysis was performed, verify content is there if "expert_analysis" in response2: expert_analysis = response2["expert_analysis"] if "raw_analysis" in expert_analysis: analysis_content = expert_analysis["raw_analysis"] assert ( "incorrect host configuration" in analysis_content.lower() or "database" in analysis_content.lower() ) elif response2["status"] == "files_required_to_continue": # If clarification is still being requested, ensure it's reasonable # Since we provided config.py and error.log, workflow tool might still need more context assert "step_number" in response2 # Should be valid workflow response else: # For other statuses, ensure basic workflow structure is maintained assert "step_number" in response2 ================================================ FILE: tests/test_config.py ================================================ """ Tests for configuration """ from config import ( DEFAULT_MODEL, TEMPERATURE_ANALYTICAL, TEMPERATURE_BALANCED, TEMPERATURE_CREATIVE, __author__, __updated__, __version__, ) class TestConfig: """Test configuration values""" def test_version_info(self): """Test version information exists and has correct format""" # Check version format (e.g., "2.4.1") assert isinstance(__version__, str) assert len(__version__.split(".")) == 3 # Major.Minor.Patch # Check author assert __author__ == "Fahad Gilani" # Check updated date exists (don't assert on specific format/value) assert isinstance(__updated__, str) def test_model_config(self): """Test model configuration""" # DEFAULT_MODEL is set in conftest.py for tests assert DEFAULT_MODEL == "gemini-2.5-flash" def test_temperature_defaults(self): """Test temperature constants""" assert TEMPERATURE_ANALYTICAL == 1.0 assert TEMPERATURE_BALANCED == 1.0 assert TEMPERATURE_CREATIVE == 1.0 ================================================ FILE: tests/test_consensus.py ================================================ """ Tests for the Consensus tool using WorkflowTool architecture. """ from unittest.mock import Mock import pytest from tools.consensus import ConsensusRequest, ConsensusTool from tools.models import ToolModelCategory class TestConsensusTool: """Test suite for ConsensusTool using WorkflowTool architecture.""" def test_tool_metadata(self): """Test basic tool metadata and configuration.""" tool = ConsensusTool() assert tool.get_name() == "consensus" assert "consensus" in tool.get_description() assert tool.get_default_temperature() == 1.0 # TEMPERATURE_ANALYTICAL assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING assert tool.requires_model() is False # Consensus manages its own models def test_request_validation_step1(self): """Test Pydantic request model validation for step 1.""" # Valid step 1 request with models step1_request = ConsensusRequest( step="Analyzing the real-time collaboration proposal", step_number=1, total_steps=4, # 1 (Claude) + 2 models + 1 (synthesis) next_step_required=True, findings="Initial assessment shows strong value but technical complexity", confidence="medium", models=[{"model": "flash", "stance": "neutral"}, {"model": "o3-mini", "stance": "for"}], relevant_files=["/proposal.md"], ) assert step1_request.step_number == 1 assert step1_request.confidence == "medium" assert len(step1_request.models) == 2 assert step1_request.models[0]["model"] == "flash" def test_request_validation_missing_models_step1(self): """Test that step 1 requires models field.""" with pytest.raises(ValueError, match="Step 1 requires 'models' field"): ConsensusRequest( step="Test step", step_number=1, total_steps=3, next_step_required=True, findings="Test findings", # Missing models field ) def test_request_validation_later_steps(self): """Test request validation for steps 2+.""" # Step 2+ doesn't require models field step2_request = ConsensusRequest( step="Processing first model response", step_number=2, total_steps=4, next_step_required=True, findings="Model provided supportive perspective", confidence="medium", continuation_id="test-id", current_model_index=1, ) assert step2_request.step_number == 2 assert step2_request.models is None # Not required after step 1 def test_request_validation_duplicate_model_stance(self): """Test that duplicate model+stance combinations are rejected.""" # Valid: same model with different stances valid_request = ConsensusRequest( step="Analyze this proposal", step_number=1, total_steps=1, next_step_required=True, findings="Initial analysis", models=[ {"model": "o3", "stance": "for"}, {"model": "o3", "stance": "against"}, {"model": "flash", "stance": "neutral"}, ], continuation_id="test-id", ) assert len(valid_request.models) == 3 # Invalid: duplicate model+stance combination with pytest.raises(ValueError, match="Duplicate model \\+ stance combination"): ConsensusRequest( step="Analyze this proposal", step_number=1, total_steps=1, next_step_required=True, findings="Initial analysis", models=[ {"model": "o3", "stance": "for"}, {"model": "flash", "stance": "neutral"}, {"model": "o3", "stance": "for"}, # Duplicate! ], continuation_id="test-id", ) def test_input_schema_generation(self): """Test that input schema is generated correctly.""" tool = ConsensusTool() schema = tool.get_input_schema() # Verify consensus workflow fields are present assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "findings" in schema["properties"] # confidence field should be excluded assert "confidence" not in schema["properties"] assert "models" in schema["properties"] # relevant_files should be present as it's used by consensus assert "relevant_files" in schema["properties"] # model field should NOT be present as consensus uses 'models' field instead assert "model" not in schema["properties"] # Verify workflow fields that should NOT be present assert "files_checked" not in schema["properties"] assert "hypothesis" not in schema["properties"] assert "issues_found" not in schema["properties"] assert "temperature" not in schema["properties"] assert "thinking_mode" not in schema["properties"] # Images should be present now assert "images" in schema["properties"] assert schema["properties"]["images"]["type"] == "array" assert schema["properties"]["images"]["items"]["type"] == "string" # Verify field types assert schema["properties"]["step"]["type"] == "string" assert schema["properties"]["step_number"]["type"] == "integer" assert schema["properties"]["models"]["type"] == "array" # Verify models array structure models_items = schema["properties"]["models"]["items"] assert models_items["type"] == "object" assert "model" in models_items["properties"] assert "stance" in models_items["properties"] assert "stance_prompt" in models_items["properties"] def test_get_required_actions(self): """Test required actions for different consensus phases.""" tool = ConsensusTool() # Step 1: Claude's initial analysis actions = tool.get_required_actions(1, "exploring", "Initial findings", 4) assert any("initial analysis" in action for action in actions) assert any("consult other models" in action for action in actions) # Step 2-3: Model consultations actions = tool.get_required_actions(2, "medium", "Model findings", 4) assert any("Review the model response" in action for action in actions) # Final step: Synthesis actions = tool.get_required_actions(4, "high", "All findings", 4) assert any("All models have been consulted" in action for action in actions) assert any("Synthesize all perspectives" in action for action in actions) def test_prepare_step_data(self): """Test step data preparation for consensus workflow.""" tool = ConsensusTool() request = ConsensusRequest( step="Test step", step_number=1, total_steps=3, next_step_required=True, findings="Test findings", confidence="medium", models=[{"model": "test"}], relevant_files=["/test.py"], ) step_data = tool.prepare_step_data(request) # Verify consensus-specific fields assert step_data["step"] == "Test step" assert step_data["findings"] == "Test findings" assert step_data["relevant_files"] == ["/test.py"] # Verify unused workflow fields are empty assert step_data["files_checked"] == [] assert step_data["relevant_context"] == [] assert step_data["issues_found"] == [] assert step_data["hypothesis"] is None def test_stance_enhanced_prompt_generation(self): """Test stance-enhanced prompt generation.""" tool = ConsensusTool() # Test different stances for_prompt = tool._get_stance_enhanced_prompt("for") assert "SUPPORTIVE PERSPECTIVE" in for_prompt against_prompt = tool._get_stance_enhanced_prompt("against") assert "CRITICAL PERSPECTIVE" in against_prompt neutral_prompt = tool._get_stance_enhanced_prompt("neutral") assert "BALANCED PERSPECTIVE" in neutral_prompt # Test custom stance prompt custom = "Focus on specific aspects" custom_prompt = tool._get_stance_enhanced_prompt("for", custom) assert custom in custom_prompt assert "SUPPORTIVE PERSPECTIVE" not in custom_prompt def test_should_call_expert_analysis(self): """Test that consensus workflow doesn't use expert analysis.""" tool = ConsensusTool() assert tool.should_call_expert_analysis({}) is False assert tool.requires_expert_analysis() is False def test_execute_workflow_step1_basic(self): """Test basic workflow validation for step 1.""" tool = ConsensusTool() # Test that step 1 sets up the workflow correctly arguments = { "step": "Initial analysis of proposal", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found pros and cons", "models": [{"model": "flash", "stance": "neutral"}, {"model": "o3-mini", "stance": "for"}], } # Verify models_to_consult is set correctly from step 1 request = tool.get_workflow_request_model()(**arguments) assert len(request.models) == 2 assert request.models[0]["model"] == "flash" assert request.models[1]["model"] == "o3-mini" def test_execute_workflow_total_steps_calculation(self): """Test that total_steps is calculated correctly from models.""" tool = ConsensusTool() # Test with 2 models arguments = { "step": "Initial analysis", "step_number": 1, "total_steps": 4, # This should be corrected to 2 "next_step_required": True, "findings": "Analysis complete", "models": [{"model": "flash", "stance": "neutral"}, {"model": "o3-mini", "stance": "for"}], } request = tool.get_workflow_request_model()(**arguments) # The tool should set total_steps = len(models) = 2 assert len(request.models) == 2 def test_consult_model_basic_structure(self): """Test basic model consultation structure.""" tool = ConsensusTool() # Test that _get_stance_enhanced_prompt works for_prompt = tool._get_stance_enhanced_prompt("for") against_prompt = tool._get_stance_enhanced_prompt("against") neutral_prompt = tool._get_stance_enhanced_prompt("neutral") assert "SUPPORTIVE PERSPECTIVE" in for_prompt assert "CRITICAL PERSPECTIVE" in against_prompt assert "BALANCED PERSPECTIVE" in neutral_prompt def test_model_configuration_validation(self): """Test model configuration validation.""" tool = ConsensusTool() # Test single model config models = [{"model": "flash", "stance": "neutral"}] arguments = { "step": "Test", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Test findings", "models": models, } request = tool.get_workflow_request_model()(**arguments) assert len(request.models) == 1 assert request.models[0]["model"] == "flash" assert request.models[0]["stance"] == "neutral" def test_handle_work_continuation(self): """Test work continuation handling - legacy method for compatibility.""" tool = ConsensusTool() tool.models_to_consult = [{"model": "flash", "stance": "neutral"}, {"model": "o3-mini", "stance": "for"}] # Note: In the new workflow, model consultation happens DURING steps in execute_workflow # This method is kept for compatibility but not actively used in the step-by-step flow # Test after step 1 request = Mock(step_number=1, current_model_index=0) response_data = {} result = tool.handle_work_continuation(response_data, request) # The method still exists but returns legacy status for compatibility assert "status" in result # Test between model consultations request = Mock(step_number=2, current_model_index=1) response_data = {} result = tool.handle_work_continuation(response_data, request) assert "status" in result def test_customize_workflow_response(self): """Test response customization for consensus workflow.""" tool = ConsensusTool() tool.accumulated_responses = [{"model": "test", "response": "data"}] # Test different step numbers (new workflow: 2 models = 2 steps) request = Mock(step_number=1, total_steps=2) response_data = {} result = tool.customize_workflow_response(response_data, request) assert result["consensus_workflow_status"] == "initial_analysis_complete" request = Mock(step_number=2, total_steps=2) response_data = {} result = tool.customize_workflow_response(response_data, request) assert result["consensus_workflow_status"] == "ready_for_synthesis" @pytest.mark.asyncio async def test_consensus_with_relevant_files_model_context_fix(self): """Test that consensus tool properly handles relevant_files without RuntimeError. This is a regression test for the bug where _prepare_file_content_for_prompt was called without model_context parameter, causing RuntimeError: 'Model context not provided for file preparation' Bug details: - Occurred when consensus tool processed requests with relevant_files - _consult_model method called _prepare_file_content_for_prompt without model_context - Method expected model_context parameter but got None (default value) - Runtime validation in base_tool.py threw RuntimeError """ from unittest.mock import AsyncMock, Mock, patch from utils.model_context import ModelContext tool = ConsensusTool() # Create a mock request with relevant_files (the trigger condition) mock_request = Mock() mock_request.relevant_files = ["/test/file1.py", "/test/file2.js"] mock_request.continuation_id = None # Mock model configuration model_config = {"model": "flash", "stance": "neutral"} # Mock the provider and model name resolution with ( patch.object(tool, "get_model_provider") as mock_get_provider, patch.object(tool, "_prepare_file_content_for_prompt") as mock_prepare_files, patch.object(tool, "_get_stance_enhanced_prompt") as mock_get_prompt, patch.object(tool, "get_name", return_value="consensus"), ): # Setup mocks mock_provider = Mock() mock_provider.generate_content = AsyncMock(return_value={"response": "test response"}) mock_get_provider.return_value = mock_provider mock_prepare_files.return_value = ("file content", []) mock_get_prompt.return_value = "system prompt" # Set up the tool's attributes that would be set during normal execution tool.original_proposal = "Test proposal" try: # This should not raise RuntimeError after the fix # The method should create ModelContext and pass it to _prepare_file_content_for_prompt await tool._consult_model(model_config, mock_request) # Verify that _prepare_file_content_for_prompt was called with model_context mock_prepare_files.assert_called_once() call_args = mock_prepare_files.call_args # Check that model_context was passed as keyword argument assert "model_context" in call_args.kwargs, "model_context should be passed as keyword argument" # Verify the model_context is a proper ModelContext instance model_context = call_args.kwargs["model_context"] assert isinstance(model_context, ModelContext), "model_context should be ModelContext instance" # Verify model_context properties are correct assert model_context.model_name == "flash" # Note: provider is accessed lazily, conversation_history and tool_name # are not part of ModelContext constructor in current implementation except RuntimeError as e: if "Model context not provided" in str(e): pytest.fail("The model_context fix is not working. RuntimeError still occurs: " + str(e)) else: # Re-raise if it's a different RuntimeError raise if __name__ == "__main__": import unittest unittest.main() ================================================ FILE: tests/test_consensus_integration.py ================================================ """Integration test for ConsensusTool using OpenAI and Gemini recordings.""" from __future__ import annotations import json import os from pathlib import Path import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from tests.transport_helpers import inject_transport from tools.consensus import ConsensusTool # Directories for recorded HTTP interactions CASSETTE_DIR = Path(__file__).parent / "openai_cassettes" CASSETTE_DIR.mkdir(exist_ok=True) # Mapping of OpenAI model names to their cassette files CONSENSUS_CASSETTES = { "gpt-5": CASSETTE_DIR / "consensus_step1_gpt5_for.json", "gpt-5.2": CASSETTE_DIR / "consensus_step1_gpt52_for.json", } GEMINI_REPLAY_DIR = Path(__file__).parent / "gemini_cassettes" GEMINI_REPLAY_DIR.mkdir(exist_ok=True) GEMINI_REPLAY_ID = "consensus/step2_gemini25_flash_against/mldev" GEMINI_REPLAY_PATH = GEMINI_REPLAY_DIR / "consensus" / "step2_gemini25_flash_against" / "mldev.json" @pytest.mark.integration @pytest.mark.asyncio @pytest.mark.no_mock_provider @pytest.mark.parametrize("openai_model", ["gpt-5", "gpt-5.2"]) async def test_consensus_multi_model_consultations(monkeypatch, openai_model): """Exercise ConsensusTool against OpenAI model (supporting) and gemini-2.5-flash (critical). Tests both gpt-5 and gpt-5.2 to ensure regression coverage for both model families. """ # Get the cassette path for this model consensus_cassette_path = CONSENSUS_CASSETTES[openai_model] env_updates = { "DEFAULT_MODEL": "auto", "OPENAI_API_KEY": os.getenv("OPENAI_API_KEY", ""), "GEMINI_API_KEY": os.getenv("GEMINI_API_KEY", ""), } keys_to_clear = [ "XAI_API_KEY", "OPENROUTER_API_KEY", "ANTHROPIC_API_KEY", "MISTRAL_API_KEY", "CUSTOM_API_KEY", "CUSTOM_API_URL", ] recording_mode = not consensus_cassette_path.exists() or not GEMINI_REPLAY_PATH.exists() if recording_mode: openai_key = env_updates["OPENAI_API_KEY"].strip() gemini_key = env_updates["GEMINI_API_KEY"].strip() if (not openai_key or openai_key.startswith("dummy")) or (not gemini_key or gemini_key.startswith("dummy")): pytest.skip( "Consensus cassette missing and OPENAI_API_KEY/GEMINI_API_KEY " "not configured. Provide real keys to record." ) GEMINI_REPLAY_PATH.parent.mkdir(parents=True, exist_ok=True) with monkeypatch.context() as m: m.setenv("DEFAULT_MODEL", env_updates["DEFAULT_MODEL"]) if recording_mode: m.setenv("OPENAI_API_KEY", env_updates["OPENAI_API_KEY"]) m.setenv("GEMINI_API_KEY", env_updates["GEMINI_API_KEY"]) m.setenv("GOOGLE_GENAI_CLIENT_MODE", "record") else: m.setenv("OPENAI_API_KEY", "dummy-key-for-replay") m.setenv("GEMINI_API_KEY", "dummy-key-for-replay") m.setenv("GOOGLE_GENAI_CLIENT_MODE", "replay") # Ensure restriction policies allow the latest OpenAI models under test m.setenv("OPENAI_ALLOWED_MODELS", openai_model) m.setenv("GOOGLE_GENAI_REPLAYS_DIRECTORY", str(GEMINI_REPLAY_DIR)) m.setenv("GOOGLE_GENAI_REPLAY_ID", GEMINI_REPLAY_ID) for key in keys_to_clear: m.delenv(key, raising=False) # Ensure we use the built-in OpenAI catalogue rather than leftovers from # other tests that patch OPENAI_MODELS_CONFIG_PATH. m.delenv("OPENAI_MODELS_CONFIG_PATH", raising=False) # Reset providers/restrictions and register only OpenAI & Gemini for deterministic behavior ModelProviderRegistry.reset_for_testing() import utils.model_restrictions as model_restrictions model_restrictions._restriction_service = None from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider # Earlier tests may override the OpenAI provider's registry by pointing # OPENAI_MODELS_CONFIG_PATH at fixtures. Force a reload so model # metadata is restored from conf/openai_models.json. OpenAIModelProvider.reload_registry() assert openai_model in OpenAIModelProvider.MODEL_CAPABILITIES ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) # Inject HTTP transport for OpenAI interactions inject_transport(monkeypatch, str(consensus_cassette_path)) tool = ConsensusTool() models_to_consult = [ {"model": openai_model, "stance": "for"}, {"model": "gemini-2.5-flash", "stance": "against"}, ] # Step 1: CLI agent analysis followed by first model consultation step1_arguments = { "step": "Evaluate SwiftUI vs UIKit adoption and recommend ONE word (SwiftUI or UIKit).", "step_number": 1, "total_steps": len(models_to_consult), "next_step_required": True, "findings": "SwiftUI momentum is strong but UIKit remains battle-tested.", "models": models_to_consult, } step1_response = await tool.execute(step1_arguments) assert step1_response and step1_response[0].type == "text" step1_data = json.loads(step1_response[0].text) assert step1_data["status"] == "analysis_and_first_model_consulted" assert step1_data["model_consulted"] == openai_model assert step1_data["model_response"]["status"] == "success" assert step1_data["model_response"]["metadata"]["provider"] == "openai" assert step1_data["model_response"]["verdict"] continuation_offer = step1_data.get("continuation_offer") assert continuation_offer is not None continuation_id = continuation_offer["continuation_id"] # Prepare step 2 inputs using the first model's response summary summary_for_step2 = step1_data["model_response"]["verdict"][:200] step2_arguments = { "step": f"Incorporated {openai_model} perspective: {summary_for_step2}", "step_number": 2, "total_steps": len(models_to_consult), "next_step_required": False, "findings": "Ready to gather opposing stance before synthesis.", "continuation_id": continuation_id, "current_model_index": step1_data.get("current_model_index", 1), "model_responses": step1_data.get("model_responses", []), } step2_response = await tool.execute(step2_arguments) assert step2_response and step2_response[0].type == "text" step2_data = json.loads(step2_response[0].text) assert step2_data["status"] == "consensus_workflow_complete" assert step2_data["model_consulted"] == "gemini-2.5-flash" assert step2_data["model_response"]["metadata"]["provider"] == "google" assert step2_data["model_response"]["verdict"] assert step2_data["complete_consensus"]["models_consulted"] == [ f"{openai_model}:for", "gemini-2.5-flash:against", ] assert step2_data["consensus_complete"] is True continuation_offer_final = step2_data.get("continuation_offer") assert continuation_offer_final is not None assert continuation_offer_final["continuation_id"] == continuation_id # Ensure Gemini replay session is flushed to disk before verification gemini_provider = ModelProviderRegistry.get_provider_for_model("gemini-2.5-flash") if gemini_provider is not None: try: client = gemini_provider.client if hasattr(client, "close"): client.close() finally: if hasattr(gemini_provider, "_client"): gemini_provider._client = None # Ensure cassettes exist for future replays assert consensus_cassette_path.exists() assert GEMINI_REPLAY_PATH.exists() # Clean up provider registry state after test ModelProviderRegistry.reset_for_testing() @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_consensus_auto_mode_with_openrouter_and_gemini(monkeypatch): """Ensure continuation flow resolves to real models instead of leaking 'auto'.""" gemini_key = os.getenv("GEMINI_API_KEY", "").strip() or "dummy-key-for-replay" openrouter_key = os.getenv("OPENROUTER_API_KEY", "").strip() or "dummy-key-for-replay" with monkeypatch.context() as m: m.setenv("DEFAULT_MODEL", "auto") m.setenv("GEMINI_API_KEY", gemini_key) m.setenv("OPENROUTER_API_KEY", openrouter_key) for key in [ "OPENAI_API_KEY", "XAI_API_KEY", "DIAL_API_KEY", "CUSTOM_API_KEY", "CUSTOM_API_URL", ]: m.delenv(key, raising=False) import importlib import config m.setattr(config, "DEFAULT_MODEL", "auto") import server as server_module server = importlib.reload(server_module) m.setattr(server, "DEFAULT_MODEL", "auto", raising=False) ModelProviderRegistry.reset_for_testing() from providers.gemini import GeminiModelProvider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) from utils.storage_backend import get_storage_backend # Clear conversation storage to avoid cross-test leakage storage = get_storage_backend() storage._store.clear() models_to_consult = [ {"model": "claude-3-5-flash-20241022", "stance": "neutral"}, {"model": "gpt-5-mini", "stance": "neutral"}, ] step1_args = { "step": "Evaluate framework options.", "step_number": 1, "total_steps": len(models_to_consult), "next_step_required": True, "findings": "Initial analysis requested.", "models": models_to_consult, } step1_output = await server.handle_call_tool("consensus", step1_args) assert step1_output and step1_output[0].type == "text" step1_payload = json.loads(step1_output[0].text) assert step1_payload["status"] == "analysis_and_first_model_consulted" assert step1_payload["model_consulted"] == "claude-3-5-flash-20241022" assert step1_payload["model_response"]["status"] == "error" assert "claude-3-5-flash-20241022" in step1_payload["model_response"]["error"] continuation_offer = step1_payload.get("continuation_offer") assert continuation_offer is not None continuation_id = continuation_offer["continuation_id"] step2_args = { "step": "Continue consultation sequence.", "step_number": 2, "total_steps": len(models_to_consult), "next_step_required": False, "findings": "Ready for next model.", "continuation_id": continuation_id, "models": models_to_consult, } try: step2_output = await server.handle_call_tool("consensus", step2_args) finally: # Reset provider registry regardless of outcome to avoid cross-test bleed ModelProviderRegistry.reset_for_testing() assert step2_output and step2_output[0].type == "text" step2_payload = json.loads(step2_output[0].text) serialized = json.dumps(step2_payload) assert "auto" not in serialized.lower(), "Auto model leakage should be resolved" assert "gpt-5-mini" in serialized or "claude-3-5-flash-20241022" in serialized # Restore server module to reflect original configuration for other tests import importlib import server as server_module importlib.reload(server_module) ================================================ FILE: tests/test_consensus_schema.py ================================================ """Schema-related tests for ConsensusTool.""" from types import MethodType from tools.consensus import ConsensusTool def test_consensus_models_field_includes_available_models(monkeypatch): """Consensus schema should surface available model guidance like single-model tools.""" tool = ConsensusTool() monkeypatch.setattr( tool, "_get_ranked_model_summaries", MethodType(lambda self, limit=5: (["gemini-2.5-pro (score 100, 1.0M ctx, thinking)"], 1, False), tool), ) monkeypatch.setattr(tool, "_get_restriction_note", MethodType(lambda self: None, tool)) schema = tool.get_input_schema() models_field_description = schema["properties"]["models"]["description"] assert "listmodels" in models_field_description assert "Top models" in models_field_description ================================================ FILE: tests/test_conversation_continuation_integration.py ================================================ """Integration test for conversation continuation persistence.""" from tools.chat import ChatRequest, ChatTool from utils.conversation_memory import get_thread from utils.storage_backend import get_storage_backend def test_first_response_persisted_in_conversation_history(tmp_path): """Ensure the assistant's initial reply is stored for newly created threads.""" # Clear in-memory storage to avoid cross-test contamination storage = get_storage_backend() storage._store.clear() # type: ignore[attr-defined] tool = ChatTool() request = ChatRequest( prompt="First question?", model="local-llama", working_directory_absolute_path=str(tmp_path), ) response_text = "Here is the initial answer." # Mimic the first tool invocation (no continuation_id supplied) continuation_data = tool._create_continuation_offer(request, model_info={"model_name": "local-llama"}) tool._create_continuation_offer_response( response_text, continuation_data, request, {"model_name": "local-llama", "provider": "custom"}, ) thread_id = continuation_data["continuation_id"] thread = get_thread(thread_id) assert thread is not None assert [turn.role for turn in thread.turns] == ["user", "assistant"] assert thread.turns[-1].content == response_text # Cleanup storage for subsequent tests storage._store.clear() # type: ignore[attr-defined] ================================================ FILE: tests/test_conversation_field_mapping.py ================================================ """ Test that conversation history is correctly mapped to tool-specific fields """ from datetime import datetime from unittest.mock import patch import pytest from server import reconstruct_thread_context from utils.conversation_memory import ConversationTurn, ThreadContext @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_conversation_history_field_mapping(): """Test that enhanced prompts are mapped to prompt field for all tools""" # Test data for different tools - all use 'prompt' now test_cases = [ { "tool_name": "analyze", "original_value": "What does this code do?", }, { "tool_name": "chat", "original_value": "Explain this concept", }, { "tool_name": "debug", "original_value": "Getting undefined error", }, { "tool_name": "codereview", "original_value": "Review this implementation", }, { "tool_name": "thinkdeep", "original_value": "My analysis so far", }, ] for test_case in test_cases: # Create real conversation context mock_context = ThreadContext( thread_id="test-thread-123", tool_name=test_case["tool_name"], created_at=datetime.now().isoformat(), last_updated_at=datetime.now().isoformat(), turns=[ ConversationTurn( role="user", content="Previous user message", timestamp=datetime.now().isoformat(), files=["/test/file1.py"], ), ConversationTurn( role="assistant", content="Previous assistant response", timestamp=datetime.now().isoformat(), ), ], initial_context={}, ) # Mock get_thread to return our test context with patch("utils.conversation_memory.get_thread", return_value=mock_context): with patch("utils.conversation_memory.add_turn", return_value=True): # Create arguments with continuation_id and use a test model arguments = { "continuation_id": "test-thread-123", "prompt": test_case["original_value"], "absolute_file_paths": ["/test/file2.py"], "model": "flash", # Use test model to avoid provider errors } # Call reconstruct_thread_context enhanced_args = await reconstruct_thread_context(arguments) # Verify the enhanced prompt is in the prompt field assert "prompt" in enhanced_args enhanced_value = enhanced_args["prompt"] # Should contain conversation history assert "=== CONVERSATION HISTORY" in enhanced_value # Allow for both formats assert "Previous user message" in enhanced_value assert "Previous assistant response" in enhanced_value # Should contain the new user input assert "=== NEW USER INPUT ===" in enhanced_value assert test_case["original_value"] in enhanced_value # Should have token budget assert "_remaining_tokens" in enhanced_args assert enhanced_args["_remaining_tokens"] > 0 @pytest.mark.asyncio @pytest.mark.no_mock_provider async def test_unknown_tool_defaults_to_prompt(): """Test that unknown tools default to using 'prompt' field""" mock_context = ThreadContext( thread_id="test-thread-456", tool_name="unknown_tool", created_at=datetime.now().isoformat(), last_updated_at=datetime.now().isoformat(), turns=[ ConversationTurn( role="user", content="First message", timestamp=datetime.now().isoformat(), ), ConversationTurn( role="assistant", content="First response", timestamp=datetime.now().isoformat(), ), ], initial_context={}, ) with patch("utils.conversation_memory.get_thread", return_value=mock_context): with patch("utils.conversation_memory.add_turn", return_value=True): arguments = { "continuation_id": "test-thread-456", "prompt": "User input", "model": "flash", # Use test model for real integration } enhanced_args = await reconstruct_thread_context(arguments) # Should default to 'prompt' field assert "prompt" in enhanced_args assert "=== CONVERSATION HISTORY" in enhanced_args["prompt"] # Allow for both formats assert "First message" in enhanced_args["prompt"] assert "First response" in enhanced_args["prompt"] assert "User input" in enhanced_args["prompt"] @pytest.mark.asyncio async def test_tool_parameter_standardization(): """Test that workflow tools use standardized investigation pattern""" from tools.analyze import AnalyzeWorkflowRequest from tools.codereview import CodeReviewRequest from tools.debug import DebugInvestigationRequest from tools.precommit import PrecommitRequest from tools.thinkdeep import ThinkDeepWorkflowRequest # Test analyze tool uses workflow pattern analyze = AnalyzeWorkflowRequest( step="What does this do?", step_number=1, total_steps=1, next_step_required=False, findings="Initial analysis", relevant_files=["/test.py"], ) assert analyze.step == "What does this do?" # Debug tool now uses self-investigation pattern with different fields debug = DebugInvestigationRequest( step="Investigating error", step_number=1, total_steps=3, next_step_required=True, findings="Initial error analysis", ) assert debug.step == "Investigating error" assert debug.findings == "Initial error analysis" # Test codereview tool uses workflow fields review = CodeReviewRequest( step="Initial code review investigation", step_number=1, total_steps=2, next_step_required=True, findings="Initial review findings", relevant_files=["/test.py"], ) assert review.step == "Initial code review investigation" assert review.findings == "Initial review findings" # Test thinkdeep tool uses workflow pattern think = ThinkDeepWorkflowRequest( step="My analysis", step_number=1, total_steps=1, next_step_required=False, findings="Initial thinking analysis" ) assert think.step == "My analysis" # Test precommit tool uses workflow fields precommit = PrecommitRequest( step="Validating changes for commit", step_number=1, total_steps=2, next_step_required=True, findings="Initial validation findings", path="/repo", # path only needed for step 1 ) assert precommit.step == "Validating changes for commit" assert precommit.findings == "Initial validation findings" ================================================ FILE: tests/test_conversation_file_features.py ================================================ """ Test suite for conversation memory file management features. This module tests the enhanced conversation memory system including: - File inclusion in conversation history - Token-aware file inclusion planning - Smart file size limiting for conversation history - Cross-tool file context preservation - MCP boundary vs conversation building separation """ import os from unittest.mock import patch from utils.conversation_memory import ( ConversationTurn, ThreadContext, _plan_file_inclusion_by_size, build_conversation_history, get_conversation_file_list, ) class TestConversationFileList: """Test file list extraction from conversation turns""" def test_get_conversation_file_list_basic(self): """Test that files are returned from conversation turns, newest first""" turns = [ ConversationTurn( role="user", content="First turn (older)", timestamp="2023-01-01T00:00:00Z", files=["/project/file1.py", "/project/file2.py"], ), ConversationTurn( role="assistant", content="Second turn (newer)", timestamp="2023-01-01T00:01:00Z", files=["/project/file3.py"], ), ] context = ThreadContext( thread_id="test", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="test", turns=turns, initial_context={}, ) files = get_conversation_file_list(context) # Should contain all unique files, with newest turn files first assert len(files) == 3 assert files[0] == "/project/file3.py" # From newest turn (turn 2) assert "/project/file1.py" in files[1:] # From older turn (turn 1) assert "/project/file2.py" in files[1:] # From older turn (turn 1) def test_get_conversation_file_list_deduplication(self): """Test that duplicate files are removed, prioritizing newer turns""" turns = [ ConversationTurn( role="user", content="First mention (older)", timestamp="2023-01-01T00:00:00Z", files=["/project/file1.py", "/project/shared.py"], ), ConversationTurn( role="assistant", content="Duplicate mention (newer)", timestamp="2023-01-01T00:01:00Z", files=["/project/shared.py", "/project/file2.py"], # shared.py is duplicate ), ] context = ThreadContext( thread_id="test", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="test", turns=turns, initial_context={}, ) files = get_conversation_file_list(context) # Should have unique files only, with newer turn files first assert len(files) == 3 # Files from turn 2 (newer) should come first assert files[0] == "/project/shared.py" # From newer turn (turn 2) assert files[1] == "/project/file2.py" # From newer turn (turn 2) # Files from turn 1 (older) that aren't duplicates assert files[2] == "/project/file1.py" # From older turn (turn 1) class TestFileInclusionPlanning: """Test token-aware file inclusion planning for conversation history""" def test_plan_file_inclusion_within_budget(self, project_path): """Test file inclusion when all files fit within token budget""" # Create small test files small_file1 = os.path.join(project_path, "small1.py") small_file2 = os.path.join(project_path, "small2.py") with open(small_file1, "w") as f: f.write("# Small file 1\nprint('hello')\n") # ~30 chars with open(small_file2, "w") as f: f.write("# Small file 2\nprint('world')\n") # ~30 chars all_files = [small_file1, small_file2] max_tokens = 1000 # Generous budget included, skipped, total_tokens = _plan_file_inclusion_by_size(all_files, max_tokens) assert included == all_files assert skipped == [] assert total_tokens > 0 # Should have estimated some tokens def test_plan_file_inclusion_exceeds_budget(self, project_path): """Test file inclusion when files exceed token budget""" # Create files with different sizes small_file = os.path.join(project_path, "small.py") large_file = os.path.join(project_path, "large.py") with open(small_file, "w") as f: f.write("# Small file\nprint('hello')\n") # ~25 chars with open(large_file, "w") as f: f.write("# Large file\n" + "x = 1\n" * 1000) # Much larger all_files = [small_file, large_file] max_tokens = 50 # Very tight budget included, skipped, total_tokens = _plan_file_inclusion_by_size(all_files, max_tokens) # Should include some files, skip others when budget is tight assert len(included) + len(skipped) == 2 assert total_tokens <= max_tokens def test_plan_file_inclusion_empty_list(self): """Test file inclusion planning with empty file list""" included, skipped, total_tokens = _plan_file_inclusion_by_size([], 1000) assert included == [] assert skipped == [] assert total_tokens == 0 def test_plan_file_inclusion_nonexistent_files(self): """Test file inclusion planning with non-existent files""" nonexistent_files = ["/does/not/exist1.py", "/does/not/exist2.py"] included, skipped, total_tokens = _plan_file_inclusion_by_size(nonexistent_files, 1000) assert included == [] assert skipped == nonexistent_files assert total_tokens == 0 class TestConversationHistoryBuilding: """Test conversation history building with file content embedding""" @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_build_conversation_history_with_file_content(self, project_path): """Test that conversation history includes embedded file content""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() # Create test file with known content test_file = os.path.join(project_path, "test.py") test_content = "# Test file\ndef hello():\n print('Hello, world!')\n" with open(test_file, "w") as f: f.write(test_content) # Create conversation with file reference turns = [ ConversationTurn( role="user", content="Please analyze this file", timestamp="2023-01-01T00:00:00Z", files=[test_file], ) ] context = ThreadContext( thread_id="test-thread", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="analyze", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context) # Verify structure assert "=== CONVERSATION HISTORY (CONTINUATION) ===" in history assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history assert "--- Turn 1 (Agent) ---" in history # Verify file content is embedded assert "--- BEGIN FILE:" in history assert test_file in history assert test_content in history assert "--- END FILE:" in history # Verify turn content assert "Please analyze this file" in history assert f"Files used in this turn: {test_file}" in history @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_build_conversation_history_file_deduplication(self, project_path): """Test that files are embedded only once even if referenced multiple times""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() test_file = os.path.join(project_path, "shared.py") with open(test_file, "w") as f: f.write("# Shared file\nshared_var = 42\n") # Multiple turns referencing the same file turns = [ ConversationTurn( role="user", content="First look at this file", timestamp="2023-01-01T00:00:00Z", files=[test_file], ), ConversationTurn( role="assistant", content="Analysis complete", timestamp="2023-01-01T00:01:00Z", files=[test_file], # Same file referenced again ), ] context = ThreadContext( thread_id="test-thread", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="analyze", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context) # File should appear in embedded section only once file_begin_count = history.count("--- BEGIN FILE:") file_end_count = history.count("--- END FILE:") assert file_begin_count == 1, "File should be embedded exactly once" assert file_end_count == 1, "File should be embedded exactly once" # But should show in both turn references turn_file_refs = history.count(f"Files used in this turn: {test_file}") assert turn_file_refs == 2, "Both turns should show file usage" def test_build_conversation_history_empty_turns(self): """Test conversation history building with no turns""" context = ThreadContext( thread_id="empty-thread", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="test", turns=[], initial_context={}, ) history, tokens = build_conversation_history(context) assert history == "" assert tokens == 0 class TestCrossToolFileContext: """Test cross-tool file context preservation in conversations""" @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_cross_tool_file_context_preservation(self, project_path): """Test that file context is preserved across different tools""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() src_file = os.path.join(project_path, "src.py") test_file = os.path.join(project_path, "test.py") with open(src_file, "w") as f: f.write("def main():\n return 'hello'\n") with open(test_file, "w") as f: f.write("import src\nassert src.main() == 'hello'\n") # Simulate cross-tool conversation with chronological timestamps turns = [ ConversationTurn( role="assistant", content="I've analyzed the source code structure", timestamp="2023-01-01T00:00:00Z", # First turn files=[src_file], tool_name="analyze", model_name="gemini-2.5-flash", model_provider="google", ), ConversationTurn( role="user", content="Now generate tests for it", timestamp="2023-01-01T00:01:00Z", # Second turn (1 minute later) files=[test_file], ), ConversationTurn( role="assistant", content="I've generated comprehensive tests", timestamp="2023-01-01T00:02:00Z", # Third turn (2 minutes later) files=[src_file, test_file], # References both files tool_name="testgen", model_name="gpt-5", model_provider="openai", ), ] context = ThreadContext( thread_id="cross-tool-thread", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:02:00Z", tool_name="testgen", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context) # Verify cross-tool context assert "--- Turn 1 (gemini-2.5-flash using analyze via google) ---" in history assert "--- Turn 2 (Agent) ---" in history assert "--- Turn 3 (gpt-5 using testgen via openai) ---" in history # Verify file context preservation assert "Files used in this turn: " + src_file in history assert "Files used in this turn: " + test_file in history assert f"Files used in this turn: {src_file}, {test_file}" in history # Verify both files are embedded files_section_start = history.find("=== FILES REFERENCED IN THIS CONVERSATION ===") first_file_pos = history.find(src_file, files_section_start) second_file_pos = history.find(test_file, files_section_start) assert first_file_pos > 0 and second_file_pos > 0 class TestLargeConversations: """Test behavior with large conversations, many files, and many turns""" @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_large_conversation_with_many_files(self, project_path): """Test conversation with many files across multiple turns""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() # Create 20 test files test_files = [] for i in range(20): test_file = os.path.join(project_path, f"file{i:02d}.py") with open(test_file, "w") as f: f.write(f"# File {i}\nclass Module{i}:\n def method(self):\n return {i}\n") test_files.append(test_file) # Create 15 conversation turns with files spread across them turns = [] for turn_num in range(15): # Distribute files across turns (some turns have multiple files) if turn_num < 10: turn_files = test_files[turn_num * 2 : (turn_num + 1) * 2] # 2 files per turn else: turn_files = [] # Some turns without files turns.append( ConversationTurn( role="user" if turn_num % 2 == 0 else "assistant", content=f"Turn {turn_num} content - working on modules", timestamp=f"2023-01-01T{turn_num:02d}:00:00Z", files=turn_files, tool_name="analyze" if turn_num % 3 == 0 else None, ) ) context = ThreadContext( thread_id="large-conversation", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T14:00:00Z", tool_name="analyze", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context) # Verify structure assert "=== CONVERSATION HISTORY (CONTINUATION) ===" in history assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history # Should handle large conversation gracefully assert len(history) > 1000 # Should have substantial content assert tokens > 0 # Files from newer turns should be prioritized file_list = get_conversation_file_list(context) assert len(file_list) == 20 # All unique files # Files from turn 9 (newest with files) should come first newest_files = test_files[18:20] # Files from turn 9 assert file_list[0] in newest_files assert file_list[1] in newest_files class TestSmallAndNewConversations: """Test behavior with small/new conversations and edge cases""" def test_empty_conversation(self): """Test completely empty conversation""" context = ThreadContext( thread_id="empty", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="test", turns=[], initial_context={}, ) history, tokens = build_conversation_history(context) assert history == "" assert tokens == 0 @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_single_turn_conversation(self, project_path): """Test conversation with just one turn""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() test_file = os.path.join(project_path, "single.py") with open(test_file, "w") as f: f.write("# Single file\ndef hello():\n return 'world'\n") turns = [ ConversationTurn( role="user", content="Quick question about this file", timestamp="2023-01-01T00:00:00Z", files=[test_file], ) ] context = ThreadContext( thread_id="single-turn", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context) # Should work correctly for single turn assert "=== CONVERSATION HISTORY (CONTINUATION) ===" in history assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history assert "--- Turn 1 (Agent) ---" in history assert "Quick question about this file" in history assert test_file in history assert tokens > 0 class TestFailureScenarios: """Test failure scenarios and error handling""" def test_file_list_with_missing_files(self): """Test conversation with references to missing files""" turns = [ ConversationTurn( role="user", content="Analyze these files", timestamp="2023-01-01T00:00:00Z", files=["/does/not/exist.py", "/also/missing.py"], ) ] context = ThreadContext( thread_id="missing-files", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="analyze", turns=turns, initial_context={}, ) # Should handle missing files gracefully files = get_conversation_file_list(context) assert len(files) == 2 # Still returns file paths assert "/does/not/exist.py" in files assert "/also/missing.py" in files @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_conversation_with_unreadable_files(self, project_path): """Test conversation history building with unreadable files""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() # Create a file that will be treated as missing missing_file = os.path.join(project_path, "nonexistent.py") # Create a readable file for comparison test_file = os.path.join(project_path, "readable.py") with open(test_file, "w") as f: f.write("# Test file\ndef test(): pass\n") turns = [ ConversationTurn( role="user", content="Analyze these files", timestamp="2023-01-01T00:00:00Z", files=[test_file, missing_file], ) ] context = ThreadContext( thread_id="mixed-files", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="analyze", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context) # Should handle gracefully - build history with accessible files assert "=== CONVERSATION HISTORY (CONTINUATION) ===" in history assert "--- Turn 1 (Agent) ---" in history assert "Analyze these files" in history assert tokens > 0 ================================================ FILE: tests/test_conversation_memory.py ================================================ """ Test suite for conversation memory system Tests the Redis-based conversation persistence needed for AI-to-AI multi-turn discussions in stateless MCP environments. """ import os from unittest.mock import Mock, patch import pytest from server import get_follow_up_instructions from utils.conversation_memory import ( CONVERSATION_TIMEOUT_SECONDS, MAX_CONVERSATION_TURNS, ConversationTurn, ThreadContext, add_turn, build_conversation_history, create_thread, get_thread, ) class TestConversationMemory: """Test the conversation memory system for stateless MCP requests""" @patch("utils.conversation_memory.get_storage") def test_create_thread(self, mock_storage): """Test creating a new thread""" mock_client = Mock() mock_storage.return_value = mock_client thread_id = create_thread("chat", {"prompt": "Hello", "absolute_file_paths": ["/test.py"]}) assert thread_id is not None assert len(thread_id) == 36 # UUID4 length # Verify Redis was called mock_client.setex.assert_called_once() call_args = mock_client.setex.call_args assert call_args[0][0] == f"thread:{thread_id}" # key assert call_args[0][1] == CONVERSATION_TIMEOUT_SECONDS # TTL from configuration @patch("utils.conversation_memory.get_storage") def test_get_thread_valid(self, mock_storage): """Test retrieving an existing thread""" mock_client = Mock() mock_storage.return_value = mock_client test_uuid = "12345678-1234-1234-1234-123456789012" # Create valid ThreadContext and serialize it context_obj = ThreadContext( thread_id=test_uuid, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="chat", turns=[], initial_context={"prompt": "test"}, ) mock_client.get.return_value = context_obj.model_dump_json() context = get_thread(test_uuid) assert context is not None assert context.thread_id == test_uuid assert context.tool_name == "chat" mock_client.get.assert_called_once_with(f"thread:{test_uuid}") @patch("utils.conversation_memory.get_storage") def test_get_thread_invalid_uuid(self, mock_storage): """Test handling invalid UUID""" context = get_thread("invalid-uuid") assert context is None @patch("utils.conversation_memory.get_storage") def test_get_thread_not_found(self, mock_storage): """Test handling thread not found""" mock_client = Mock() mock_storage.return_value = mock_client mock_client.get.return_value = None context = get_thread("12345678-1234-1234-1234-123456789012") assert context is None @patch("utils.conversation_memory.get_storage") def test_add_turn_success(self, mock_storage): """Test adding a turn to existing thread""" mock_client = Mock() mock_storage.return_value = mock_client test_uuid = "12345678-1234-1234-1234-123456789012" # Create valid ThreadContext context_obj = ThreadContext( thread_id=test_uuid, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="chat", turns=[], initial_context={"prompt": "test"}, ) mock_client.get.return_value = context_obj.model_dump_json() success = add_turn(test_uuid, "user", "Hello there") assert success is True # Verify Redis get and setex were called mock_client.get.assert_called_once() mock_client.setex.assert_called_once() @patch("utils.conversation_memory.get_storage") def test_add_turn_max_limit(self, mock_storage): """Test turn limit enforcement""" mock_client = Mock() mock_storage.return_value = mock_client test_uuid = "12345678-1234-1234-1234-123456789012" # Create thread with MAX_CONVERSATION_TURNS turns (at limit) turns = [ ConversationTurn(role="user", content=f"Turn {i}", timestamp="2023-01-01T00:00:00Z") for i in range(MAX_CONVERSATION_TURNS) ] context_obj = ThreadContext( thread_id=test_uuid, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="chat", turns=turns, initial_context={"prompt": "test"}, ) mock_client.get.return_value = context_obj.model_dump_json() success = add_turn(test_uuid, "user", "This should fail") assert success is False @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_build_conversation_history(self, project_path): """Test building conversation history format with files and speaker identification""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() # Create real test files to test actual file embedding functionality main_file = project_path / "main.py" readme_file = project_path / "docs" / "readme.md" examples_dir = project_path / "examples" examples_file = examples_dir / "example.py" # Create directories and files readme_file.parent.mkdir(parents=True, exist_ok=True) examples_dir.mkdir(parents=True, exist_ok=True) main_file.write_text("def main():\n print('Hello world')\n") readme_file.write_text("# Project Documentation\nThis is a test project.\n") examples_file.write_text("# Example code\nprint('Example')\n") test_uuid = "12345678-1234-1234-1234-123456789012" turns = [ ConversationTurn( role="user", content="What is Python?", timestamp="2023-01-01T00:00:00Z", files=[str(main_file), str(readme_file)], ), ConversationTurn( role="assistant", content="Python is a programming language", timestamp="2023-01-01T00:01:00Z", files=[str(examples_dir)], # Directory will be expanded to files tool_name="chat", model_name="gpt-5", model_provider="openai", ), ] context = ThreadContext( thread_id=test_uuid, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="chat", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context, model_context=None) # Test basic structure assert "CONVERSATION HISTORY" in history assert f"Thread: {test_uuid}" in history assert "Tool: chat" in history assert f"Turn 2/{MAX_CONVERSATION_TURNS}" in history # Test speaker identification assert "--- Turn 1 (Agent) ---" in history assert "--- Turn 2 (gpt-5 using chat via openai) ---" in history # Test content assert "What is Python?" in history assert "Python is a programming language" in history # Test file tracking # Check that the new file embedding section is included assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history assert "The following files have been shared and analyzed during our conversation." in history # Check that file context from previous turns is included (now shows files used per turn) assert f"Files used in this turn: {main_file}, {readme_file}" in history assert f"Files used in this turn: {examples_dir}" in history # Verify actual file content is embedded assert "def main():" in history assert "Hello world" in history assert "Project Documentation" in history def test_build_conversation_history_empty(self): """Test building history with no turns""" test_uuid = "12345678-1234-1234-1234-123456789012" context = ThreadContext( thread_id=test_uuid, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=[], initial_context={}, ) history, tokens = build_conversation_history(context, model_context=None) assert history == "" assert tokens == 0 class TestConversationFlow: """Test complete conversation flows simulating stateless MCP requests""" @patch("utils.conversation_memory.get_storage") def test_complete_conversation_cycle(self, mock_storage): """Test a complete 5-turn conversation until limit reached""" mock_client = Mock() mock_storage.return_value = mock_client # Simulate independent MCP request cycles # REQUEST 1: Initial request creates thread thread_id = create_thread("chat", {"prompt": "Analyze this code"}) initial_context = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=[], initial_context={"prompt": "Analyze this code"}, ) mock_client.get.return_value = initial_context.model_dump_json() # Add assistant response success = add_turn( thread_id, "assistant", "Code analysis complete", ) assert success is True # REQUEST 2: User responds to follow-up (independent request cycle) # Simulate retrieving updated context from Redis context_after_1 = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="chat", turns=[ ConversationTurn( role="assistant", content="Code analysis complete", timestamp="2023-01-01T00:00:30Z", ) ], initial_context={"prompt": "Analyze this code"}, ) mock_client.get.return_value = context_after_1.model_dump_json() success = add_turn(thread_id, "user", "Yes, check error handling") assert success is True success = add_turn(thread_id, "assistant", "Error handling reviewed") assert success is True # REQUEST 3-5: Continue conversation (simulating independent cycles) # After turn 3 context_after_3 = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:03:00Z", tool_name="chat", turns=[ ConversationTurn( role="assistant", content="Code analysis complete", timestamp="2023-01-01T00:00:30Z", ), ConversationTurn(role="user", content="Yes, check error handling", timestamp="2023-01-01T00:01:30Z"), ConversationTurn( role="assistant", content="Error handling reviewed", timestamp="2023-01-01T00:02:30Z", ), ], initial_context={"prompt": "Analyze this code"}, ) mock_client.get.return_value = context_after_3.model_dump_json() success = add_turn(thread_id, "user", "Yes, check tests") assert success is True success = add_turn(thread_id, "assistant", "Test coverage analyzed") assert success is True # REQUEST 6: Try to exceed MAX_CONVERSATION_TURNS limit - should fail turns_at_limit = [ ConversationTurn( role="assistant" if i % 2 == 0 else "user", content=f"Turn {i + 1}", timestamp="2023-01-01T00:00:30Z" ) for i in range(MAX_CONVERSATION_TURNS) ] context_at_limit = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:05:00Z", tool_name="chat", turns=turns_at_limit, initial_context={"prompt": "Analyze this code"}, ) mock_client.get.return_value = context_at_limit.model_dump_json() # This should fail - conversation has reached limit success = add_turn(thread_id, "user", "This should be rejected") assert success is False # CONVERSATION STOPS HERE @patch("utils.conversation_memory.get_storage") def test_invalid_continuation_id_error(self, mock_storage): """Test that invalid continuation IDs raise proper error for restart""" from server import reconstruct_thread_context mock_client = Mock() mock_storage.return_value = mock_client mock_client.get.return_value = None # Thread not found arguments = {"continuation_id": "invalid-uuid-12345", "prompt": "Continue conversation"} # Should raise ValueError asking to restart with pytest.raises(ValueError) as exc_info: import asyncio asyncio.run(reconstruct_thread_context(arguments)) error_msg = str(exc_info.value) assert "Conversation thread 'invalid-uuid-12345' was not found or has expired" in error_msg assert ( "Please restart the conversation by providing your full question/prompt without the continuation_id" in error_msg ) @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_dynamic_max_turns_configuration(self): """Test that all functions respect MAX_CONVERSATION_TURNS configuration""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() # This test ensures if we change MAX_CONVERSATION_TURNS, everything updates # Test with different max values by patching the constant test_values = [3, 7, 10] for test_max in test_values: # Create turns up to the test limit turns = [ ConversationTurn(role="user", content=f"Turn {i}", timestamp="2023-01-01T00:00:00Z") for i in range(test_max) ] # Test history building respects the limit test_uuid = "12345678-1234-1234-1234-123456789012" context = ThreadContext( thread_id=test_uuid, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=turns, initial_context={}, ) history, tokens = build_conversation_history(context, model_context=None) expected_turn_text = f"Turn {test_max}/{MAX_CONVERSATION_TURNS}" assert expected_turn_text in history def test_follow_up_instructions_dynamic_behavior(self): """Test that follow-up instructions change correctly based on turn count and max setting""" # Test with default MAX_CONVERSATION_TURNS max_turns = MAX_CONVERSATION_TURNS # Test early conversation (should allow follow-ups) early_instructions = get_follow_up_instructions(0, max_turns) assert "CONVERSATION CONTINUATION" in early_instructions assert f"({max_turns - 1} exchanges remaining)" in early_instructions assert "Feel free to ask clarifying questions" in early_instructions # Test mid conversation mid_instructions = get_follow_up_instructions(2, max_turns) assert "CONVERSATION CONTINUATION" in mid_instructions assert f"({max_turns - 3} exchanges remaining)" in mid_instructions assert "Feel free to ask clarifying questions" in mid_instructions # Test approaching limit (should stop follow-ups) limit_instructions = get_follow_up_instructions(max_turns - 1, max_turns) assert "Do NOT include any follow-up questions" in limit_instructions assert "final exchange" in limit_instructions # Test at limit at_limit_instructions = get_follow_up_instructions(max_turns, max_turns) assert "Do NOT include any follow-up questions" in at_limit_instructions # Test with custom max_turns to ensure dynamic behavior custom_max = 3 custom_early = get_follow_up_instructions(0, custom_max) assert f"({custom_max - 1} exchanges remaining)" in custom_early custom_limit = get_follow_up_instructions(custom_max - 1, custom_max) assert "Do NOT include any follow-up questions" in custom_limit def test_follow_up_instructions_defaults_to_config(self): """Test that follow-up instructions use MAX_CONVERSATION_TURNS when max_turns not provided""" instructions = get_follow_up_instructions(0) # No max_turns parameter expected_remaining = MAX_CONVERSATION_TURNS - 1 assert f"({expected_remaining} exchanges remaining)" in instructions @patch("utils.conversation_memory.get_storage") def test_complete_conversation_with_dynamic_turns(self, mock_storage): """Test complete conversation respecting MAX_CONVERSATION_TURNS dynamically""" mock_client = Mock() mock_storage.return_value = mock_client thread_id = create_thread("chat", {"prompt": "Start conversation"}) # Simulate conversation up to MAX_CONVERSATION_TURNS - 1 for turn_num in range(MAX_CONVERSATION_TURNS - 1): # Mock context with current turns turns = [ ConversationTurn( role="user" if i % 2 == 0 else "assistant", content=f"Turn {i + 1}", timestamp="2023-01-01T00:00:00Z", ) for i in range(turn_num) ] context = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=turns, initial_context={"prompt": "Start conversation"}, ) mock_client.get.return_value = context.model_dump_json() # Should succeed success = add_turn(thread_id, "user", f"User turn {turn_num + 1}") assert success is True, f"Turn {turn_num + 1} should succeed" # Now we should be at the limit - create final context final_turns = [ ConversationTurn( role="user" if i % 2 == 0 else "assistant", content=f"Turn {i + 1}", timestamp="2023-01-01T00:00:00Z" ) for i in range(MAX_CONVERSATION_TURNS) ] final_context = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=final_turns, initial_context={"prompt": "Start conversation"}, ) mock_client.get.return_value = final_context.model_dump_json() # This should fail - at the limit success = add_turn(thread_id, "user", "This should fail") assert success is False, f"Turn {MAX_CONVERSATION_TURNS + 1} should fail" @patch("utils.conversation_memory.get_storage") @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_conversation_with_files_and_context_preservation(self, mock_storage): """Test complete conversation flow with file tracking and context preservation""" from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() mock_client = Mock() mock_storage.return_value = mock_client # Start conversation with files using a simple tool thread_id = create_thread("chat", {"prompt": "Analyze this codebase", "absolute_file_paths": ["/project/src/"]}) # Turn 1: Claude provides context with multiple files initial_context = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=[], initial_context={ "prompt": "Analyze this codebase", "absolute_file_paths": ["/project/src/"], }, ) mock_client.get.return_value = initial_context.model_dump_json() # Add Gemini's response success = add_turn( thread_id, "assistant", "I've analyzed your codebase structure.", files=["/project/src/main.py", "/project/src/utils.py"], tool_name="analyze", model_name="gemini-2.5-flash", model_provider="google", ) assert success is True # Turn 2: Claude responds with different files context_turn_1 = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="analyze", turns=[ ConversationTurn( role="assistant", content="I've analyzed your codebase structure.", timestamp="2023-01-01T00:00:30Z", files=["/project/src/main.py", "/project/src/utils.py"], tool_name="analyze", model_name="gemini-2.5-flash", model_provider="google", ) ], initial_context={"prompt": "Analyze this codebase", "relevant_files": ["/project/src/"]}, ) mock_client.get.return_value = context_turn_1.model_dump_json() # User responds with test files success = add_turn( thread_id, "user", "Yes, check the test coverage", files=["/project/tests/", "/project/test_main.py"] ) assert success is True # Turn 3: Gemini analyzes tests context_turn_2 = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:02:00Z", tool_name="analyze", turns=[ ConversationTurn( role="assistant", content="I've analyzed your codebase structure.", timestamp="2023-01-01T00:00:30Z", files=["/project/src/main.py", "/project/src/utils.py"], tool_name="analyze", ), ConversationTurn( role="user", content="Yes, check the test coverage", timestamp="2023-01-01T00:01:30Z", files=["/project/tests/", "/project/test_main.py"], ), ], initial_context={"prompt": "Analyze this codebase", "relevant_files": ["/project/src/"]}, ) mock_client.get.return_value = context_turn_2.model_dump_json() success = add_turn( thread_id, "assistant", "Test coverage analysis complete. Coverage is 85%.", files=["/project/tests/test_utils.py", "/project/coverage.html"], tool_name="analyze", model_name="gemini-2.5-flash", model_provider="google", ) assert success is True # Build conversation history and verify chronological file preservation final_context = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:03:00Z", tool_name="analyze", turns=[ ConversationTurn( role="assistant", content="I've analyzed your codebase structure.", timestamp="2023-01-01T00:00:30Z", files=["/project/src/main.py", "/project/src/utils.py"], tool_name="analyze", model_name="gemini-2.5-flash", model_provider="google", ), ConversationTurn( role="user", content="Yes, check the test coverage", timestamp="2023-01-01T00:01:30Z", files=["/project/tests/", "/project/test_main.py"], ), ConversationTurn( role="assistant", content="Test coverage analysis complete. Coverage is 85%.", timestamp="2023-01-01T00:02:30Z", files=["/project/tests/test_utils.py", "/project/coverage.html"], tool_name="analyze", model_name="gemini-2.5-flash", model_provider="google", ), ], initial_context={"prompt": "Analyze this codebase", "relevant_files": ["/project/src/"]}, ) history, tokens = build_conversation_history(final_context) # Verify chronological order and speaker identification assert "--- Turn 1 (gemini-2.5-flash using analyze via google) ---" in history assert "--- Turn 2 (Agent) ---" in history assert "--- Turn 3 (gemini-2.5-flash using analyze via google) ---" in history # Verify all files are preserved in chronological order turn_1_files = "Files used in this turn: /project/src/main.py, /project/src/utils.py" turn_2_files = "Files used in this turn: /project/tests/, /project/test_main.py" turn_3_files = "Files used in this turn: /project/tests/test_utils.py, /project/coverage.html" assert turn_1_files in history assert turn_2_files in history assert turn_3_files in history # Verify content assert "I've analyzed your codebase structure." in history assert "Yes, check the test coverage" in history assert "Test coverage analysis complete. Coverage is 85%." in history # Verify chronological ordering (turn 1 appears before turn 2, etc.) turn_1_pos = history.find("--- Turn 1 (gemini-2.5-flash using analyze via google) ---") turn_2_pos = history.find("--- Turn 2 (Agent) ---") turn_3_pos = history.find("--- Turn 3 (gemini-2.5-flash using analyze via google) ---") assert turn_1_pos < turn_2_pos < turn_3_pos @patch("utils.conversation_memory.get_storage") def test_stateless_request_isolation(self, mock_storage): """Test that each request cycle is independent but shares context via Redis""" mock_client = Mock() mock_storage.return_value = mock_client # Simulate two different "processes" accessing same thread thread_id = "12345678-1234-1234-1234-123456789012" # Process 1: Creates thread initial_context = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="thinkdeep", turns=[], initial_context={"prompt": "Think about architecture"}, ) mock_client.get.return_value = initial_context.model_dump_json() success = add_turn(thread_id, "assistant", "Architecture analysis") assert success is True # Process 2: Different "request cycle" accesses same thread context_from_redis = ThreadContext( thread_id=thread_id, created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="thinkdeep", turns=[ ConversationTurn( role="assistant", content="Architecture analysis", timestamp="2023-01-01T00:00:30Z", ) ], initial_context={"prompt": "Think about architecture"}, ) mock_client.get.return_value = context_from_redis.model_dump_json() # Verify context continuity across "processes" retrieved_context = get_thread(thread_id) assert retrieved_context is not None assert len(retrieved_context.turns) == 1 @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": ""}, clear=False) def test_token_limit_optimization_in_conversation_history(self): """Test that build_conversation_history efficiently handles token limits""" import os import tempfile from providers.registry import ModelProviderRegistry ModelProviderRegistry.clear_cache() from utils.conversation_memory import build_conversation_history # Create test files with known content sizes with tempfile.TemporaryDirectory() as temp_dir: # Create small and large test files small_file = os.path.join(temp_dir, "small.py") large_file = os.path.join(temp_dir, "large.py") small_content = "# Small file\nprint('hello')\n" large_content = "# Large file\n" + "x = 1\n" * 10000 # Very large file with open(small_file, "w") as f: f.write(small_content) with open(large_file, "w") as f: f.write(large_content) # Create context with files that would exceed token limit context = ThreadContext( thread_id="test-token-limit", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:01:00Z", tool_name="analyze", turns=[ ConversationTurn( role="user", content="Analyze these files", timestamp="2023-01-01T00:00:30Z", files=[small_file, large_file], # Large file should be truncated ) ], initial_context={"prompt": "Analyze code"}, ) # Build conversation history (should handle token limits gracefully) history, tokens = build_conversation_history(context, model_context=None) # Verify the history was built successfully assert "=== CONVERSATION HISTORY" in history assert "=== FILES REFERENCED IN THIS CONVERSATION ===" in history # The small file should be included, but large file might be truncated # At minimum, verify no crashes and history is generated assert len(history) > 0 # If truncation occurred, there should be a note about it if "additional file(s) were truncated due to token limit" in history: assert small_file in history or large_file in history else: # Both files fit within limit assert small_file in history assert large_file in history if __name__ == "__main__": pytest.main([__file__]) ================================================ FILE: tests/test_conversation_missing_files.py ================================================ """ Test conversation memory handling of missing files. Following existing test patterns to ensure conversation memory gracefully handles missing files without crashing. """ from unittest.mock import Mock from utils.conversation_memory import ( ConversationTurn, ThreadContext, build_conversation_history, ) class TestConversationMissingFiles: """Test handling of missing files during conversation memory reconstruction.""" def test_build_conversation_history_handles_missing_files(self): """Test that conversation history building handles missing files gracefully.""" # Create conversation context with missing file reference (following existing test patterns) context = ThreadContext( thread_id="test-thread", created_at="2024-01-01T00:00:00Z", last_updated_at="2024-01-01T00:05:00Z", tool_name="analyze", turns=[ ConversationTurn( role="user", content="Please analyze this file", timestamp="2024-01-01T00:01:00Z", files=["/nonexistent/missing_file.py"], # File that doesn't exist tool_name="analyze", ), ConversationTurn( role="assistant", content="Here's my analysis...", timestamp="2024-01-01T00:02:00Z", tool_name="analyze", ), ], initial_context={"path": "/nonexistent/missing_file.py"}, ) # Mock model context (following existing test patterns) mock_model_context = Mock() mock_model_context.calculate_token_allocation.return_value = Mock(file_tokens=50000, history_tokens=50000) mock_model_context.estimate_tokens.return_value = 100 mock_model_context.model_name = "test-model" # Should not crash, should handle missing file gracefully history, tokens = build_conversation_history(context, mock_model_context) # Should return valid history despite missing file assert isinstance(history, str) assert isinstance(tokens, int) assert len(history) > 0 # Should contain conversation content assert "CONVERSATION HISTORY" in history assert "Please analyze this file" in history assert "Here's my analysis" in history ================================================ FILE: tests/test_custom_openai_temperature_fix.py ================================================ """ Test for custom OpenAI models temperature parameter fix. This test verifies that custom OpenAI models configured through custom_models.json with supports_temperature=false do not send temperature parameters to the API. This addresses issue #245. """ import json import tempfile from pathlib import Path from unittest.mock import Mock, patch from providers.openai import OpenAIModelProvider class TestCustomOpenAITemperatureParameterFix: """Test custom OpenAI model parameter filtering.""" def _create_test_config(self, models_config: list[dict]) -> str: """Create a temporary config file for testing.""" config = {"_README": {"description": "Test config"}, "models": models_config} temp_file = tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) json.dump(config, temp_file, indent=2) temp_file.close() return temp_file.name @patch("utils.model_restrictions.get_restriction_service") @patch("providers.openai_compatible.OpenAI") def test_custom_openai_models_exclude_temperature_from_api_call(self, mock_openai_class, mock_restriction_service): """Test that custom OpenAI models with supports_temperature=false don't send temperature to the API.""" # Create test config with a custom OpenAI model that doesn't support temperature config_models = [ { "model_name": "gpt-5-2025-08-07", "provider": "openai", "context_window": 400000, "max_output_tokens": 128000, "supports_extended_thinking": True, "supports_json_mode": True, "supports_system_prompts": True, "supports_streaming": True, "supports_function_calling": True, "supports_temperature": False, "temperature_constraint": "fixed", "supports_images": True, "max_image_size_mb": 20.0, "reasoning": {"effort": "low"}, "description": "Custom OpenAI GPT-5 test model", } ] config_path = self._create_test_config(config_models) try: # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service # Setup mock client mock_client = Mock() mock_openai_class.return_value = mock_client # Setup mock response mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "gpt-5-2025-08-07" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = Mock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response # Create provider with custom config with patch("providers.registries.openrouter.OpenRouterModelRegistry") as mock_registry_class: # Mock registry to load our test config mock_registry = Mock() mock_registry_class.return_value = mock_registry # Mock get_model_config to return our test model from providers.shared import ModelCapabilities, ProviderType, TemperatureConstraint test_capabilities = ModelCapabilities( provider=ProviderType.OPENAI, model_name="gpt-5-2025-08-07", friendly_name="Custom GPT-5", context_window=400000, max_output_tokens=128000, supports_extended_thinking=True, supports_system_prompts=True, supports_streaming=True, supports_function_calling=True, supports_json_mode=True, supports_images=True, max_image_size_mb=20.0, supports_temperature=False, # This is the key setting temperature_constraint=TemperatureConstraint.create("fixed"), description="Custom OpenAI GPT-5 test model", ) mock_registry.get_model_config.return_value = test_capabilities provider = OpenAIModelProvider(api_key="test-key") # Override model validation to bypass restrictions provider.validate_model_name = lambda name: True # Call generate_content with custom model provider.generate_content( prompt="Test prompt", model_name="gpt-5-2025-08-07", temperature=0.5, max_output_tokens=100 ) # Verify the API call was made without temperature or max_tokens mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] assert ( "temperature" not in call_kwargs ), "Custom OpenAI models with supports_temperature=false should not include temperature parameter" assert ( "max_tokens" not in call_kwargs ), "Custom OpenAI models with supports_temperature=false should not include max_tokens parameter" assert call_kwargs["model"] == "gpt-5-2025-08-07" assert "messages" in call_kwargs finally: # Clean up temp file Path(config_path).unlink(missing_ok=True) @patch("utils.model_restrictions.get_restriction_service") @patch("providers.openai_compatible.OpenAI") def test_custom_openai_models_include_temperature_when_supported(self, mock_openai_class, mock_restriction_service): """Test that custom OpenAI models with supports_temperature=true still send temperature to the API.""" # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service # Setup mock client mock_client = Mock() mock_openai_class.return_value = mock_client # Setup mock response mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "gpt-4-custom" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = Mock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response # Create provider with custom config with patch("providers.registries.openrouter.OpenRouterModelRegistry") as mock_registry_class: # Mock registry to load our test config mock_registry = Mock() mock_registry_class.return_value = mock_registry # Mock get_model_config to return a model that supports temperature from providers.shared import ModelCapabilities, ProviderType, TemperatureConstraint test_capabilities = ModelCapabilities( provider=ProviderType.OPENAI, model_name="gpt-4-custom", friendly_name="Custom GPT-4", context_window=128000, max_output_tokens=32000, supports_extended_thinking=False, supports_system_prompts=True, supports_streaming=True, supports_function_calling=True, supports_json_mode=True, supports_images=True, max_image_size_mb=20.0, supports_temperature=True, # This model DOES support temperature temperature_constraint=TemperatureConstraint.create("range"), description="Custom OpenAI GPT-4 test model", ) mock_registry.get_model_config.return_value = test_capabilities provider = OpenAIModelProvider(api_key="test-key") # Override model validation to bypass restrictions provider.validate_model_name = lambda name: True # Call generate_content with custom model that supports temperature provider.generate_content( prompt="Test prompt", model_name="gpt-4-custom", temperature=0.5, max_output_tokens=100 ) # Verify the API call was made WITH temperature and max_tokens mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] assert ( call_kwargs["temperature"] == 0.5 ), "Custom OpenAI models with supports_temperature=true should include temperature parameter" assert ( call_kwargs["max_tokens"] == 100 ), "Custom OpenAI models with supports_temperature=true should include max_tokens parameter" assert call_kwargs["model"] == "gpt-4-custom" @patch("utils.model_restrictions.get_restriction_service") def test_custom_openai_model_validation(self, mock_restriction_service): """Test that custom OpenAI models are properly validated.""" # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service with patch("providers.registries.openrouter.OpenRouterModelRegistry") as mock_registry_class: # Mock registry to return a custom OpenAI model mock_registry = Mock() mock_registry_class.return_value = mock_registry from providers.shared import ModelCapabilities, ProviderType, TemperatureConstraint test_capabilities = ModelCapabilities( provider=ProviderType.OPENAI, model_name="o3-2025-04-16", friendly_name="Custom O3", context_window=200000, max_output_tokens=65536, supports_extended_thinking=False, supports_system_prompts=True, supports_streaming=True, supports_function_calling=True, supports_json_mode=True, supports_images=True, max_image_size_mb=20.0, supports_temperature=False, temperature_constraint=TemperatureConstraint.create("fixed"), description="Custom OpenAI O3 test model", ) mock_registry.get_model_config.return_value = test_capabilities provider = OpenAIModelProvider(api_key="test-key") # Test that custom model validates successfully assert provider.validate_model_name("o3-2025-04-16") is True # Test that get_capabilities returns the custom config capabilities = provider.get_capabilities("o3-2025-04-16") assert capabilities.supports_temperature is False assert capabilities.model_name == "o3-2025-04-16" assert capabilities.provider == ProviderType.OPENAI @patch("utils.model_restrictions.get_restriction_service") def test_fallback_to_builtin_models_when_registry_fails(self, mock_restriction_service): """Test that provider falls back to built-in models when registry fails.""" # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service with patch("providers.registries.openrouter.OpenRouterModelRegistry") as mock_registry_class: # Mock registry to raise an exception mock_registry_class.side_effect = Exception("Registry not available") provider = OpenAIModelProvider(api_key="test-key") # Test that built-in models still work assert provider.validate_model_name("o3-mini") is True # Test that unsupported models return false assert provider.validate_model_name("unknown-model") is False ================================================ FILE: tests/test_custom_provider.py ================================================ """Tests for CustomProvider functionality.""" import os from unittest.mock import MagicMock, patch import pytest from providers import ModelProviderRegistry from providers.custom import CustomProvider from providers.shared import ProviderType class TestCustomProvider: """Test CustomProvider class functionality.""" def test_provider_initialization_with_params(self): """Test CustomProvider initializes correctly with explicit parameters.""" provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") assert provider.base_url == "http://localhost:11434/v1" assert provider.api_key == "test-key" assert provider.get_provider_type() == ProviderType.CUSTOM def test_provider_initialization_with_env_vars(self): """Test CustomProvider initializes correctly with environment variables.""" with patch.dict(os.environ, {"CUSTOM_API_URL": "http://localhost:8000/v1", "CUSTOM_API_KEY": "env-key"}): provider = CustomProvider() assert provider.base_url == "http://localhost:8000/v1" assert provider.api_key == "env-key" def test_provider_initialization_missing_url(self): """Test CustomProvider raises error when URL is missing.""" with patch.dict(os.environ, {"CUSTOM_API_URL": ""}, clear=False): with pytest.raises(ValueError, match="Custom API URL must be provided"): CustomProvider(api_key="test-key") def test_validate_model_names_always_true(self): """Test CustomProvider validates model names correctly.""" provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") # Known model should validate assert provider.validate_model_name("llama3.2") # For custom provider, unknown models return False when not in registry # This is expected behavior - custom models need to be declared in custom_models.json assert not provider.validate_model_name("unknown-model") assert not provider.validate_model_name("anything") def test_get_capabilities_from_registry(self): """Test get_capabilities returns registry capabilities when available.""" # Save original environment original_env = os.environ.get("OPENROUTER_ALLOWED_MODELS") try: # Clear any restrictions os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") # OpenRouter-backed models should be handled by the OpenRouter provider with pytest.raises(ValueError): provider.get_capabilities("o3") # Test with a custom model from the local registry capabilities = provider.get_capabilities("local-llama") assert capabilities.provider == ProviderType.CUSTOM assert capabilities.context_window > 0 finally: # Restore original environment if original_env is None: os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) else: os.environ["OPENROUTER_ALLOWED_MODELS"] = original_env def test_get_capabilities_generic_fallback(self): """Test get_capabilities raises error for unknown models not in registry.""" provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") # Unknown models should raise ValueError when not in registry with pytest.raises(ValueError, match="Unsupported model 'unknown-model-xyz' for provider custom"): provider.get_capabilities("unknown-model-xyz") def test_model_alias_resolution(self): """Test model alias resolution works correctly.""" provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") # Test that aliases resolve properly # "llama" now resolves to "meta-llama/llama-3-70b" (the OpenRouter model) resolved = provider._resolve_model_name("llama") assert resolved == "meta-llama/llama-3-70b" # Test local model alias resolved_local = provider._resolve_model_name("local-llama") assert resolved_local == "llama3.2" def test_no_thinking_mode_support(self): """Custom provider generic capabilities default to no thinking mode.""" provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") # llama3.2 is a known model that should work assert not provider.get_capabilities("llama3.2").supports_extended_thinking # Unknown models should raise error with pytest.raises(ValueError, match="Unsupported model 'any-model' for provider custom"): provider.get_capabilities("any-model") @patch("providers.custom.OpenAICompatibleProvider.generate_content") def test_generate_content_with_alias_resolution(self, mock_generate): """Test generate_content resolves aliases before calling parent.""" mock_response = MagicMock() mock_generate.return_value = mock_response provider = CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") # Call with an alias result = provider.generate_content( prompt="test prompt", model_name="llama", temperature=0.7, # This is an alias ) # Verify parent method was called with resolved model name mock_generate.assert_called_once() call_args = mock_generate.call_args # The model_name should be either resolved or passed through assert "model_name" in call_args.kwargs assert result == mock_response class TestCustomProviderRegistration: """Test CustomProvider integration with ModelProviderRegistry.""" def setup_method(self): """Clear registry before each test.""" ModelProviderRegistry.clear_cache() ModelProviderRegistry.unregister_provider(ProviderType.CUSTOM) def teardown_method(self): """Clean up after each test.""" ModelProviderRegistry.clear_cache() ModelProviderRegistry.unregister_provider(ProviderType.CUSTOM) def test_custom_provider_factory_registration(self): """Test custom provider can be registered via factory function.""" def custom_provider_factory(api_key=None): return CustomProvider(api_key="test-key", base_url="http://localhost:11434/v1") with patch.dict(os.environ, {"CUSTOM_API_PLACEHOLDER": "configured"}): ModelProviderRegistry.register_provider(ProviderType.CUSTOM, custom_provider_factory) # Verify provider is available available = ModelProviderRegistry.get_available_providers() assert ProviderType.CUSTOM in available # Verify provider can be retrieved provider = ModelProviderRegistry.get_provider(ProviderType.CUSTOM) assert provider is not None assert isinstance(provider, CustomProvider) def test_dual_provider_setup(self): """Test both OpenRouter and Custom providers can coexist.""" from providers.openrouter import OpenRouterProvider # Create factory for custom provider def custom_provider_factory(api_key=None): return CustomProvider(api_key="", base_url="http://localhost:11434/v1") with patch.dict( os.environ, { "OPENROUTER_API_KEY": "test-openrouter-key", "CUSTOM_API_PLACEHOLDER": "configured", "OPENROUTER_ALLOWED_MODELS": "llama,anthropic/claude-opus-4.1", }, clear=True, ): # Register both providers ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) ModelProviderRegistry.register_provider(ProviderType.CUSTOM, custom_provider_factory) # Verify both are available available = ModelProviderRegistry.get_available_providers() assert ProviderType.OPENROUTER in available assert ProviderType.CUSTOM in available # Verify both can be retrieved openrouter_provider = ModelProviderRegistry.get_provider(ProviderType.OPENROUTER) custom_provider = ModelProviderRegistry.get_provider(ProviderType.CUSTOM) assert openrouter_provider is not None assert custom_provider is not None assert isinstance(custom_provider, CustomProvider) def test_provider_priority_selection(self): """Test provider selection prioritizes correctly.""" from providers.openrouter import OpenRouterProvider def custom_provider_factory(api_key=None): return CustomProvider(api_key="", base_url="http://localhost:11434/v1") with patch.dict( os.environ, { "OPENROUTER_API_KEY": "test-openrouter-key", "CUSTOM_API_PLACEHOLDER": "configured", "OPENROUTER_ALLOWED_MODELS": "", }, clear=True, ): import utils.model_restrictions utils.model_restrictions._restriction_service = None custom_provider = custom_provider_factory() openrouter_provider = OpenRouterProvider(api_key="test-openrouter-key") assert not custom_provider.validate_model_name("llama") assert openrouter_provider.validate_model_name("llama") class TestConfigureProvidersFunction: """Test the configure_providers function in server.py.""" def setup_method(self): """Clear environment and registry before each test.""" # Store the original providers to restore them later registry = ModelProviderRegistry() self._original_providers = registry._providers.copy() ModelProviderRegistry.clear_cache() for provider_type in ProviderType: ModelProviderRegistry.unregister_provider(provider_type) def teardown_method(self): """Clean up after each test.""" # Restore the original providers that were registered in conftest.py registry = ModelProviderRegistry() ModelProviderRegistry.clear_cache() registry._providers.clear() registry._providers.update(self._original_providers) def test_configure_providers_custom_only(self): """Test configure_providers with only custom URL set.""" from server import configure_providers with patch.dict( os.environ, { "CUSTOM_API_URL": "http://localhost:11434/v1", "CUSTOM_API_KEY": "", # Clear other API keys "GEMINI_API_KEY": "", "OPENAI_API_KEY": "", "OPENROUTER_API_KEY": "", }, clear=True, ): configure_providers() # Verify only custom provider is available available = ModelProviderRegistry.get_available_providers() assert ProviderType.CUSTOM in available assert ProviderType.OPENROUTER not in available def test_configure_providers_openrouter_only(self): """Test configure_providers with only OpenRouter key set.""" from server import configure_providers with patch.dict( os.environ, { "OPENROUTER_API_KEY": "test-key", # Clear other API keys "GEMINI_API_KEY": "", "OPENAI_API_KEY": "", "CUSTOM_API_URL": "", }, clear=True, ): configure_providers() # Verify only OpenRouter provider is available available = ModelProviderRegistry.get_available_providers() assert ProviderType.OPENROUTER in available assert ProviderType.CUSTOM not in available def test_configure_providers_dual_setup(self): """Test configure_providers with both OpenRouter and Custom configured.""" from server import configure_providers with patch.dict( os.environ, { "OPENROUTER_API_KEY": "test-openrouter-key", "CUSTOM_API_URL": "http://localhost:11434/v1", "CUSTOM_API_KEY": "", # Clear other API keys "GEMINI_API_KEY": "", "OPENAI_API_KEY": "", }, clear=True, ): configure_providers() # Verify both providers are available available = ModelProviderRegistry.get_available_providers() assert ProviderType.OPENROUTER in available assert ProviderType.CUSTOM in available def test_configure_providers_no_valid_keys(self): """Test configure_providers raises error when no valid API keys.""" from server import configure_providers with patch.dict( os.environ, {"GEMINI_API_KEY": "", "OPENAI_API_KEY": "", "OPENROUTER_API_KEY": "", "CUSTOM_API_URL": ""}, clear=True, ): with pytest.raises(ValueError, match="At least one API configuration is required"): configure_providers() ================================================ FILE: tests/test_debug.py ================================================ """ Tests for the debug tool using new WorkflowTool architecture. """ from tools.debug import DebugInvestigationRequest, DebugIssueTool from tools.models import ToolModelCategory class TestDebugTool: """Test suite for DebugIssueTool using new WorkflowTool architecture.""" def test_tool_metadata(self): """Test basic tool metadata and configuration.""" tool = DebugIssueTool() assert tool.get_name() == "debug" assert "debugging and root cause analysis" in tool.get_description() assert tool.get_default_temperature() == 1.0 # TEMPERATURE_ANALYTICAL assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING assert tool.requires_model() is True def test_request_validation(self): """Test Pydantic request model validation.""" # Valid investigation step request step_request = DebugInvestigationRequest( step="Investigating null pointer exception in UserService", step_number=1, total_steps=3, next_step_required=True, findings="Found potential null reference in user authentication flow", files_checked=["/src/UserService.java"], relevant_files=["/src/UserService.java"], relevant_context=["authenticate", "validateUser"], confidence="medium", hypothesis="Null pointer occurs when user object is not properly validated", ) assert step_request.step_number == 1 assert step_request.confidence == "medium" assert len(step_request.relevant_context) == 2 def test_input_schema_generation(self): """Test that input schema is generated correctly.""" tool = DebugIssueTool() schema = tool.get_input_schema() # Verify required investigation fields are present assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "findings" in schema["properties"] assert "relevant_context" in schema["properties"] # Verify field types assert schema["properties"]["step"]["type"] == "string" assert schema["properties"]["step_number"]["type"] == "integer" assert schema["properties"]["next_step_required"]["type"] == "boolean" assert schema["properties"]["relevant_context"]["type"] == "array" def test_model_category_for_debugging(self): """Test that debug tool correctly identifies as extended reasoning category.""" tool = DebugIssueTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_relevant_context_handling(self): """Test that relevant_context is handled correctly.""" request = DebugInvestigationRequest( step="Test investigation", step_number=1, total_steps=2, next_step_required=True, findings="Test findings", relevant_context=["method1", "method2"], ) # Should have relevant_context directly assert request.relevant_context == ["method1", "method2"] # Test step data preparation tool = DebugIssueTool() step_data = tool.prepare_step_data(request) assert step_data["relevant_context"] == ["method1", "method2"] ================================================ FILE: tests/test_deploy_scripts.py ================================================ """ Tests for Docker deployment scripts """ import subprocess from pathlib import Path from unittest.mock import patch import pytest class TestDeploymentScripts: """Test Docker deployment scripts""" @pytest.fixture(autouse=True) def setup(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent self.scripts_dir = self.project_root / "docker" / "scripts" def test_deployment_scripts_exist(self): """Test that deployment scripts exist""" expected_scripts = ["deploy.sh", "deploy.ps1", "build.sh", "build.ps1", "healthcheck.py"] for script in expected_scripts: script_path = self.scripts_dir / script assert script_path.exists(), f"Script {script} must exist" def test_bash_scripts_executable(self): """Test that bash scripts have proper permissions""" bash_scripts = ["deploy.sh", "build.sh"] for script in bash_scripts: script_path = self.scripts_dir / script if script_path.exists(): # Check for shebang content = script_path.read_text() assert content.startswith("#!/"), f"Script {script} must have shebang" def test_powershell_scripts_format(self): """Test PowerShell scripts have proper format""" ps_scripts = ["deploy.ps1", "build.ps1"] for script in ps_scripts: script_path = self.scripts_dir / script if script_path.exists(): content = script_path.read_text() # Check for PowerShell indicators ps_indicators = [ "param(", "Write-Host", "Write-Output", "$", # PowerShell variables ] assert any( indicator in content for indicator in ps_indicators ), f"Script {script} should contain PowerShell syntax" @patch("subprocess.run") def test_deploy_script_docker_commands(self, mock_run): """Test that deploy scripts use proper Docker commands""" mock_run.return_value.returncode = 0 # Expected Docker commands in deployment expected_commands = [["docker", "build"], ["docker-compose", "up"], ["docker", "run"]] for cmd in expected_commands: subprocess.run(cmd, capture_output=True) # Verify subprocess.run was called assert mock_run.call_count >= len(expected_commands) def test_build_script_functionality(self): """Test build script basic functionality""" build_script = self.scripts_dir / "build.sh" if build_script.exists(): content = build_script.read_text() # Should contain Docker build commands assert ( "docker build" in content or "docker-compose build" in content ), "Build script should contain Docker build commands" def test_deploy_script_health_check_integration(self): """Test deploy script includes health check validation""" deploy_scripts = ["deploy.sh", "deploy.ps1"] for script_name in deploy_scripts: script_path = self.scripts_dir / script_name if script_path.exists(): content = script_path.read_text() # Look for health check related content health_check_indicators = ["health", "healthcheck", "docker inspect", "container status"] has_health_check = any(indicator in content.lower() for indicator in health_check_indicators) if not has_health_check: pytest.warns(UserWarning, f"Consider adding health check to {script_name}") def test_script_error_handling(self): """Test that scripts have proper error handling""" scripts = ["deploy.sh", "build.sh"] for script_name in scripts: script_path = self.scripts_dir / script_name if script_path.exists(): content = script_path.read_text() # Check for error handling patterns error_patterns = [ "set -e", # Bash: exit on error "||", # Or operator for error handling "if", # Conditional error checking "exit", # Explicit exit codes ] has_error_handling = any(pattern in content for pattern in error_patterns) if not has_error_handling: pytest.warns(UserWarning, f"Consider adding error handling to {script_name}") @patch("subprocess.run") def test_docker_compose_commands(self, mock_run): """Test Docker Compose command execution""" mock_run.return_value.returncode = 0 # Test various docker-compose commands compose_commands = [ ["docker-compose", "build"], ["docker-compose", "up", "-d"], ["docker-compose", "down"], ["docker-compose", "ps"], ] for cmd in compose_commands: result = subprocess.run(cmd, capture_output=True) assert result.returncode == 0 def test_script_parameter_handling(self): """Test script parameter and option handling""" deploy_ps1 = self.scripts_dir / "deploy.ps1" if deploy_ps1.exists(): content = deploy_ps1.read_text() # PowerShell scripts should handle parameters param_indicators = ["param(", "[Parameter(", "$SkipHealthCheck", "$HealthCheckTimeout"] has_parameters = any(indicator in content for indicator in param_indicators) assert has_parameters, "PowerShell deploy script should handle parameters" def test_environment_preparation(self): """Test that scripts prepare environment correctly""" scripts_to_check = ["deploy.sh", "deploy.ps1"] for script_name in scripts_to_check: script_path = self.scripts_dir / script_name if script_path.exists(): content = script_path.read_text() # Check for environment preparation env_prep_patterns = [".env", "environment", "API_KEY", "mkdir", "logs"] prepares_environment = any(pattern in content for pattern in env_prep_patterns) if not prepares_environment: pytest.warns(UserWarning, f"Consider environment preparation in {script_name}") class TestHealthCheckScript: """Test health check script specifically""" @pytest.fixture(autouse=True) def setup(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent self.healthcheck_script = self.project_root / "docker" / "scripts" / "healthcheck.py" def test_healthcheck_script_syntax(self): """Test health check script has valid Python syntax""" if not self.healthcheck_script.exists(): pytest.skip("healthcheck.py not found") # Try to compile the script try: with open(self.healthcheck_script, encoding="utf-8") as f: content = f.read() compile(content, str(self.healthcheck_script), "exec") except SyntaxError as e: pytest.fail(f"Health check script has syntax errors: {e}") def test_healthcheck_functions_exist(self): """Test that health check functions are defined""" if not self.healthcheck_script.exists(): pytest.skip("healthcheck.py not found") content = self.healthcheck_script.read_text() # Expected functions expected_functions = ["def check_process", "def check_python_imports", "def check_log_directory"] for func in expected_functions: assert func in content, f"Function {func} should be defined" @patch("subprocess.run") def test_healthcheck_process_check(self, mock_run): """Test health check process verification""" # Mock successful process check mock_run.return_value.returncode = 0 mock_run.return_value.stdout = "12345" # Simulate process check result = subprocess.run(["pgrep", "-f", "server.py"], capture_output=True, text=True, timeout=10) assert result.returncode == 0 def test_healthcheck_import_validation(self): """Test health check import validation logic""" # Test critical modules that should be importable critical_modules = ["os", "sys", "subprocess"] for module in critical_modules: try: __import__(module) except ImportError: pytest.fail(f"Critical module {module} should be importable") def test_healthcheck_exit_codes(self): """Test that health check uses proper exit codes""" if not self.healthcheck_script.exists(): pytest.skip("healthcheck.py not found") content = self.healthcheck_script.read_text() # Should have proper exit code handling exit_patterns = [ "sys.exit(0)", # Success "sys.exit(1)", # Failure "exit(0)", "exit(1)", ] has_exit_codes = any(pattern in content for pattern in exit_patterns) assert has_exit_codes, "Health check should use proper exit codes" class TestScriptIntegration: """Test script integration with Docker ecosystem""" def test_scripts_work_with_compose_file(self): """Test that scripts work with docker-compose.yml""" project_root = Path(__file__).parent.parent compose_file = project_root / "docker-compose.yml" if compose_file.exists(): # Scripts should reference the compose file deploy_script = project_root / "docker" / "scripts" / "deploy.sh" if deploy_script.exists(): content = deploy_script.read_text() # Should work with compose file compose_refs = ["docker-compose", "compose.yml", "compose.yaml"] references_compose = any(ref in content for ref in compose_refs) assert ( references_compose or "docker build" in content ), "Deploy script should use either compose or direct Docker" def test_cross_platform_compatibility(self): """Test cross-platform script compatibility""" # Both Unix and Windows scripts should exist unix_deploy = Path(__file__).parent.parent / "docker" / "scripts" / "deploy.sh" windows_deploy = Path(__file__).parent.parent / "docker" / "scripts" / "deploy.ps1" # At least one should exist assert unix_deploy.exists() or windows_deploy.exists(), "At least one deployment script should exist" # If both exist, they should have similar functionality if unix_deploy.exists() and windows_deploy.exists(): unix_content = unix_deploy.read_text() windows_content = windows_deploy.read_text() # Both should reference Docker assert "docker" in unix_content.lower() assert "docker" in windows_content.lower() def test_script_logging_integration(self): """Test that scripts integrate with logging""" scripts_dir = Path(__file__).parent.parent / "docker" / "scripts" scripts = ["deploy.sh", "deploy.ps1", "build.sh", "build.ps1"] for script_name in scripts: script_path = scripts_dir / script_name if script_path.exists(): content = script_path.read_text() # Check for logging/output logging_patterns = ["echo", "Write-Host", "Write-Output", "print", "logger"] has_logging = any(pattern in content for pattern in logging_patterns) if not has_logging: pytest.warns(UserWarning, f"Consider adding logging to {script_name}") ================================================ FILE: tests/test_dial_provider.py ================================================ """Tests for DIAL provider implementation.""" import os from unittest.mock import MagicMock, patch import pytest from providers.dial import DIALModelProvider from providers.shared import ProviderType class TestDIALProvider: """Test DIAL provider functionality.""" @patch.dict(os.environ, {"DIAL_API_KEY": "test-key", "DIAL_API_HOST": "https://test.dialx.ai"}) def test_initialization_with_host(self): """Test provider initialization with custom host.""" provider = DIALModelProvider("test-key") assert provider._dial_api_key == "test-key" # Check internal API key storage assert provider.api_key == "placeholder-not-used" # OpenAI client uses placeholder, auth header removed by hook assert provider.base_url == "https://test.dialx.ai/openai" assert provider.get_provider_type() == ProviderType.DIAL @patch.dict(os.environ, {"DIAL_API_KEY": "test-key", "DIAL_API_HOST": ""}, clear=True) def test_initialization_default_host(self): """Test provider initialization with default host.""" provider = DIALModelProvider("test-key") assert provider._dial_api_key == "test-key" # Check internal API key storage assert provider.api_key == "placeholder-not-used" # OpenAI client uses placeholder, auth header removed by hook assert provider.base_url == "https://core.dialx.ai/openai" def test_initialization_host_normalization(self): """Test that host URL is normalized to include /openai suffix.""" # Test with host missing /openai provider = DIALModelProvider("test-key", base_url="https://custom.dialx.ai") assert provider.base_url == "https://custom.dialx.ai/openai" # Test with host already having /openai provider = DIALModelProvider("test-key", base_url="https://custom.dialx.ai/openai") assert provider.base_url == "https://custom.dialx.ai/openai" @patch.dict(os.environ, {"DIAL_ALLOWED_MODELS": ""}, clear=False) @patch("utils.model_restrictions._restriction_service", None) def test_model_validation(self): """Test model name validation.""" provider = DIALModelProvider("test-key") # Test valid models assert provider.validate_model_name("o3-2025-04-16") is True assert provider.validate_model_name("o3") is True # Shorthand assert provider.validate_model_name("anthropic.claude-opus-4.1-20250805-v1:0") is True assert provider.validate_model_name("opus-4.1") is True # Shorthand assert provider.validate_model_name("gemini-2.5-pro-preview-05-06") is True assert provider.validate_model_name("gemini-2.5-pro") is True # Shorthand # Test invalid model assert provider.validate_model_name("invalid-model") is False def test_resolve_model_name(self): """Test model name resolution for shorthands.""" provider = DIALModelProvider("test-key") # Test shorthand resolution assert provider._resolve_model_name("o3") == "o3-2025-04-16" assert provider._resolve_model_name("o4-mini") == "o4-mini-2025-04-16" assert provider._resolve_model_name("opus-4.1") == "anthropic.claude-opus-4.1-20250805-v1:0" assert provider._resolve_model_name("sonnet-4.1") == "anthropic.claude-sonnet-4.1-20250805-v1:0" assert provider._resolve_model_name("gemini-2.5-pro") == "gemini-2.5-pro-preview-05-06" assert provider._resolve_model_name("gemini-2.5-flash") == "gemini-2.5-flash-preview-05-20" # Test full name passthrough assert provider._resolve_model_name("o3-2025-04-16") == "o3-2025-04-16" assert ( provider._resolve_model_name("anthropic.claude-opus-4.1-20250805-v1:0") == "anthropic.claude-opus-4.1-20250805-v1:0" ) @patch.dict(os.environ, {"DIAL_ALLOWED_MODELS": ""}, clear=False) @patch("utils.model_restrictions._restriction_service", None) def test_get_capabilities(self): """Test getting model capabilities.""" provider = DIALModelProvider("test-key") # Test O3 capabilities capabilities = provider.get_capabilities("o3") assert capabilities.model_name == "o3-2025-04-16" assert capabilities.friendly_name == "DIAL (O3)" assert capabilities.context_window == 200_000 assert capabilities.provider == ProviderType.DIAL assert capabilities.supports_images is True assert capabilities.supports_extended_thinking is False # Test Claude 4.1 capabilities capabilities = provider.get_capabilities("opus-4.1") assert capabilities.model_name == "anthropic.claude-opus-4.1-20250805-v1:0" assert capabilities.context_window == 200_000 assert capabilities.supports_images is True assert capabilities.supports_extended_thinking is False # Test Claude 4.1 with thinking mode capabilities = provider.get_capabilities("opus-4.1-thinking") assert capabilities.model_name == "anthropic.claude-opus-4.1-20250805-v1:0-with-thinking" assert capabilities.context_window == 200_000 assert capabilities.supports_images is True assert capabilities.supports_extended_thinking is True # Test Gemini capabilities capabilities = provider.get_capabilities("gemini-2.5-pro") assert capabilities.model_name == "gemini-2.5-pro-preview-05-06" assert capabilities.context_window == 1_000_000 assert capabilities.supports_images is True # Test temperature constraint assert capabilities.temperature_constraint.min_temp == 0.0 assert capabilities.temperature_constraint.max_temp == 2.0 assert capabilities.temperature_constraint.default_temp == 0.3 @patch.dict(os.environ, {"DIAL_ALLOWED_MODELS": ""}, clear=False) @patch("utils.model_restrictions._restriction_service", None) def test_get_capabilities_invalid_model(self): """Test that get_capabilities raises for invalid models.""" provider = DIALModelProvider("test-key") with pytest.raises(ValueError, match="Unsupported model 'invalid-model' for provider dial"): provider.get_capabilities("invalid-model") @patch("utils.model_restrictions.get_restriction_service") def test_get_capabilities_restricted_model(self, mock_get_restriction): """Test that get_capabilities respects model restrictions.""" provider = DIALModelProvider("test-key") # Mock restriction service to block the model mock_service = MagicMock() mock_service.is_allowed.return_value = False mock_get_restriction.return_value = mock_service with pytest.raises(ValueError, match="not allowed by restriction policy"): provider.get_capabilities("o3") @patch.dict(os.environ, {"DIAL_ALLOWED_MODELS": ""}, clear=False) @patch("utils.model_restrictions._restriction_service", None) def test_supports_vision(self): """Test vision support detection through model capabilities.""" provider = DIALModelProvider("test-key") assert provider.get_capabilities("o3-2025-04-16").supports_images is True assert provider.get_capabilities("o3").supports_images is True # Via resolution assert provider.get_capabilities("anthropic.claude-opus-4.1-20250805-v1:0").supports_images is True assert provider.get_capabilities("gemini-2.5-pro-preview-05-06").supports_images is True with pytest.raises(ValueError): provider.get_capabilities("unknown-model") @patch("openai.OpenAI") # Mock the OpenAI class directly from openai module def test_generate_content_with_alias(self, mock_openai_class): """Test that generate_content properly resolves aliases and uses deployment routing.""" # Create mock client mock_client = MagicMock() mock_response = MagicMock() mock_response.choices = [MagicMock(message=MagicMock(content="Test response"))] mock_response.usage = MagicMock(prompt_tokens=10, completion_tokens=20, total_tokens=30) mock_response.model = "gpt-4" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.choices[0].finish_reason = "stop" mock_client.chat.completions.create.return_value = mock_response mock_openai_class.return_value = mock_client provider = DIALModelProvider("test-key") # Generate content with shorthand response = provider.generate_content(prompt="Test prompt", model_name="o3", temperature=0.7) # Shorthand # Verify OpenAI was instantiated with deployment-specific URL mock_openai_class.assert_called_once() call_args = mock_openai_class.call_args assert "/deployments/o3-2025-04-16" in call_args[1]["base_url"] # Verify the resolved model name was passed to the API mock_client.chat.completions.create.assert_called_once() create_call_args = mock_client.chat.completions.create.call_args assert create_call_args[1]["model"] == "o3-2025-04-16" # Resolved name # Verify response assert response.content == "Test response" assert response.model_name == "o3" # Original name preserved assert response.metadata["model"] == "gpt-4" # API returned model name from mock def test_provider_type(self): """Test provider type identification.""" provider = DIALModelProvider("test-key") assert provider.get_provider_type() == ProviderType.DIAL def test_friendly_name(self): """Test provider friendly name.""" provider = DIALModelProvider("test-key") assert provider.FRIENDLY_NAME == "DIAL" @patch.dict(os.environ, {"DIAL_API_VERSION": "2024-12-01"}) def test_configurable_api_version(self): """Test that API version can be configured via environment variable.""" provider = DIALModelProvider("test-key") # Check that the custom API version is stored assert provider.api_version == "2024-12-01" def test_default_api_version(self): """Test that default API version is used when not configured.""" # Clear any existing DIAL_API_VERSION from environment with patch.dict(os.environ, {}, clear=True): # Keep other env vars but ensure DIAL_API_VERSION is not set if "DIAL_API_VERSION" in os.environ: del os.environ["DIAL_API_VERSION"] provider = DIALModelProvider("test-key") # Check that the default API version is used assert provider.api_version == "2024-12-01-preview" # Check that Api-Key header is set assert provider.DEFAULT_HEADERS["Api-Key"] == "test-key" @patch.dict(os.environ, {"DIAL_ALLOWED_MODELS": "o3-2025-04-16,anthropic.claude-opus-4.1-20250805-v1:0"}) @patch("utils.model_restrictions._restriction_service", None) def test_allowed_models_restriction(self): """Test model allow-list functionality.""" provider = DIALModelProvider("test-key") # These should be allowed assert provider.validate_model_name("o3-2025-04-16") is True assert provider.validate_model_name("o3") is True # Alias for o3-2025-04-16 assert provider.validate_model_name("anthropic.claude-opus-4.1-20250805-v1:0") is True assert provider.validate_model_name("opus-4.1") is True # Resolves to anthropic.claude-opus-4.1-20250805-v1:0 # These should be blocked assert provider.validate_model_name("gemini-2.5-pro-preview-05-06") is False assert provider.validate_model_name("o4-mini-2025-04-16") is False assert provider.validate_model_name("sonnet-4.1") is False # sonnet-4.1 is not in allowed list @patch("httpx.Client") @patch("openai.OpenAI") def test_close_method(self, mock_openai_class, mock_httpx_client_class): """Test that the close method properly closes HTTP clients.""" # Mock the httpx.Client instance that DIALModelProvider will create mock_shared_http_client = MagicMock() mock_httpx_client_class.return_value = mock_shared_http_client # Mock the OpenAI client instances mock_openai_client_1 = MagicMock() mock_openai_client_2 = MagicMock() # Configure side_effect to return different mocks for subsequent calls mock_openai_class.side_effect = [mock_openai_client_1, mock_openai_client_2] provider = DIALModelProvider("test-key") # Mock the superclass's _client attribute directly mock_superclass_client = MagicMock() provider._client = mock_superclass_client # Simulate getting clients for two different deployments to populate _deployment_clients provider._get_deployment_client("model_a") provider._get_deployment_client("model_b") # Now call close provider.close() # Assert that the shared httpx client's close method was called mock_shared_http_client.close.assert_called_once() # Assert that the superclass client's close method was called mock_superclass_client.close.assert_called_once() # Assert that the deployment clients cache is cleared assert not provider._deployment_clients ================================================ FILE: tests/test_directory_expansion_tracking.py ================================================ """ Test for directory expansion tracking in conversation memory This test ensures that when directories are provided to tools, the individual expanded files are properly tracked in conversation history rather than just the directory paths. This prevents file filtering bugs in conversation continuations. """ from pathlib import Path from unittest.mock import Mock, patch import pytest from tests.mock_helpers import create_mock_provider from tools.chat import ChatTool from tools.models import ToolOutput from utils.conversation_memory import add_turn, create_thread class TestDirectoryExpansionTracking: """Test directory expansion tracking in conversation memory""" @pytest.fixture def tool(self): return ChatTool() @pytest.fixture def temp_directory_with_files(self, project_path): """Create a temporary directory with multiple files""" # Create within the project path to avoid security restrictions temp_dir = project_path / "test_temp_dir" temp_dir.mkdir(exist_ok=True) temp_path = temp_dir # Create multiple Swift files (simulating the original bug scenario) files = [] for i in range(5): swift_file = temp_path / f"File{i}.swift" swift_file.write_text( f""" import Foundation class TestClass{i} {{ func testMethod{i}() -> String {{ return "test{i}" }} }} """ ) files.append(str(swift_file)) # Create a Python file as well python_file = temp_path / "helper.py" python_file.write_text( """ def helper_function(): return "helper" """ ) files.append(str(python_file)) try: yield { "directory": str(temp_dir), "absolute_file_paths": files, "swift_files": files[:-1], # All but the Python file "python_file": str(python_file), } finally: # Cleanup import shutil shutil.rmtree(temp_dir, ignore_errors=True) @pytest.mark.asyncio @patch("providers.ModelProviderRegistry.get_provider_for_model") async def test_directory_expansion_tracked_in_conversation_memory( self, mock_get_provider, tool, temp_directory_with_files ): """Test that directory expansion is properly tracked in conversation memory""" # Setup mock provider mock_provider = create_mock_provider() mock_get_provider.return_value = mock_provider directory = temp_directory_with_files["directory"] expected_files = temp_directory_with_files["absolute_file_paths"] # Create a request with the directory (not individual files) request_args = { "prompt": "Analyze this codebase structure", "absolute_file_paths": [directory], # Directory path, not individual files "model": "flash", "working_directory_absolute_path": directory, } # Execute the tool result = await tool.execute(request_args) # Verify the tool executed successfully assert result is not None result_data = result[0].text tool_output = ToolOutput.model_validate_json(result_data) assert tool_output.status in ["success", "continuation_available"] # Verify that the actually processed files were the expanded individual files captured_files = getattr(tool, "_actually_processed_files", []) assert captured_files is not None assert len(captured_files) == len(expected_files) # Convert to sets for comparison (order might differ) # Normalize paths to handle /private prefix differences captured_set = {str(Path(f).resolve()) for f in captured_files} expected_set = {str(Path(f).resolve()) for f in expected_files} assert captured_set == expected_set # Verify that the directory was expanded to individual files assert directory not in captured_files # Directory itself should not be in the list for expected_file in expected_files: # Normalize path for comparison expected_resolved = str(Path(expected_file).resolve()) assert any(str(Path(f).resolve()) == expected_resolved for f in captured_files) @pytest.mark.asyncio @patch("utils.conversation_memory.get_storage") @patch("providers.ModelProviderRegistry.get_provider_for_model") async def test_conversation_continuation_with_directory_files( self, mock_get_provider, mock_storage, tool, temp_directory_with_files ): """Test that conversation continuation works correctly with directory expansion""" # Setup mock Redis client with in-memory storage mock_client = Mock() redis_storage = {} # Simulate Redis storage def mock_get(key): return redis_storage.get(key) def mock_setex(key, ttl, value): redis_storage[key] = value return True mock_client.get.side_effect = mock_get mock_client.setex.side_effect = mock_setex mock_storage.return_value = mock_client # Setup mock provider mock_provider = create_mock_provider() mock_get_provider.return_value = mock_provider directory = temp_directory_with_files["directory"] expected_files = temp_directory_with_files["absolute_file_paths"] # Step 1: Create a conversation thread manually with the expanded files thread_id = create_thread("chat", {"prompt": "Initial analysis", "absolute_file_paths": [directory]}) # Add a turn with the expanded files (simulating what the fix should do) success = add_turn( thread_id, "assistant", "I've analyzed the codebase structure.", files=expected_files, # Individual expanded files, not directory tool_name="chat", ) assert success is True # Step 2: Continue the conversation with the same directory continuation_args = { "prompt": "Now focus on the Swift files specifically", "absolute_file_paths": [directory], # Same directory again "model": "flash", "continuation_id": thread_id, "working_directory_absolute_path": directory, } # Mock to capture file filtering behavior original_filter_new_files = tool.filter_new_files filtered_files = None def capture_filtering_mock(requested_files, continuation_id): nonlocal filtered_files filtered_files = original_filter_new_files(requested_files, continuation_id) return filtered_files with patch.object(tool, "filter_new_files", side_effect=capture_filtering_mock): # Execute continuation - this should not re-embed the same files result = await tool.execute(continuation_args) # Verify the tool executed successfully assert result is not None result_data = result[0].text tool_output = ToolOutput.model_validate_json(result_data) assert tool_output.status in ["success", "continuation_available"] # Verify that file filtering worked correctly # The directory might still be included if it contains files not yet embedded, # but the key point is that we don't re-embed already processed individual files assert filtered_files is not None # This test shows the fix is working - conversation continuation properly filters out # already-embedded files. The exact length depends on whether any new files are found. @patch("utils.conversation_memory.get_storage") def test_get_conversation_embedded_files_with_expanded_files(self, mock_storage, tool, temp_directory_with_files): """Test that get_conversation_embedded_files returns expanded files""" # Setup mock Redis client with in-memory storage mock_client = Mock() redis_storage = {} # Simulate Redis storage def mock_get(key): return redis_storage.get(key) def mock_setex(key, ttl, value): redis_storage[key] = value return True mock_client.get.side_effect = mock_get mock_client.setex.side_effect = mock_setex mock_storage.return_value = mock_client directory = temp_directory_with_files["directory"] expected_files = temp_directory_with_files["absolute_file_paths"] # Create a thread with expanded files thread_id = create_thread("chat", {"prompt": "Initial analysis", "absolute_file_paths": [directory]}) # Add a turn with expanded files success = add_turn( thread_id, "assistant", "Analysis complete.", files=expected_files, # Individual files tool_name="chat", ) assert success is True # Get the embedded files from conversation embedded_files = tool.get_conversation_embedded_files(thread_id) # Verify that we get the individual files, not the directory assert set(embedded_files) == set(expected_files) assert directory not in embedded_files @patch("utils.conversation_memory.get_storage") def test_file_filtering_with_mixed_files_and_directories(self, mock_storage, tool, temp_directory_with_files): """Test file filtering when request contains both individual files and directories""" # Setup mock Redis client with in-memory storage mock_client = Mock() redis_storage = {} # Simulate Redis storage def mock_get(key): return redis_storage.get(key) def mock_setex(key, ttl, value): redis_storage[key] = value return True mock_client.get.side_effect = mock_get mock_client.setex.side_effect = mock_setex mock_storage.return_value = mock_client directory = temp_directory_with_files["directory"] python_file = temp_directory_with_files["python_file"] # Create a thread with some expanded files thread_id = create_thread("chat", {"prompt": "Initial analysis", "absolute_file_paths": [directory]}) # Add a turn with only some of the files (simulate partial embedding) swift_files = temp_directory_with_files["swift_files"] success = add_turn( thread_id, "assistant", "Swift analysis complete.", files=swift_files, # Only Swift files tool_name="chat", ) assert success is True # Request with both directory and individual file mixed_request = [directory, python_file] filtered_files = tool.filter_new_files(mixed_request, thread_id) # The directory should expand to individual files, and since Swift files # are already embedded, only the python file should be new # Note: the filter_new_files method handles directory expansion internally assert python_file in filtered_files # The directory itself might be in the filtered list if it expands to new files # In this case, since we only embedded Swift files, the directory might still be included @pytest.mark.asyncio @patch("providers.ModelProviderRegistry.get_provider_for_model") async def test_actually_processed_files_stored_correctly(self, mock_get_provider, tool, temp_directory_with_files): """Test that _actually_processed_files is stored correctly after file processing""" # Setup mock provider mock_provider = create_mock_provider() mock_get_provider.return_value = mock_provider directory = temp_directory_with_files["directory"] expected_files = temp_directory_with_files["absolute_file_paths"] # Execute the tool request_args = { "prompt": "Analyze this code", "absolute_file_paths": [directory], "model": "flash", "working_directory_absolute_path": directory, } result = await tool.execute(request_args) # Verify the tool executed successfully assert result is not None # Verify that _actually_processed_files was set correctly assert hasattr(tool, "_actually_processed_files") actually_processed = tool._actually_processed_files # Should contain individual files, not the directory # Normalize paths to handle /private prefix differences processed_set = {str(Path(f).resolve()) for f in actually_processed} expected_set = {str(Path(f).resolve()) for f in expected_files} assert processed_set == expected_set assert directory not in actually_processed if __name__ == "__main__": pytest.main([__file__]) ================================================ FILE: tests/test_disabled_tools.py ================================================ """Tests for DISABLED_TOOLS environment variable functionality.""" import logging import os from unittest.mock import patch import pytest from server import ( apply_tool_filter, parse_disabled_tools_env, validate_disabled_tools, ) # Mock the tool classes since we're testing the filtering logic class MockTool: def __init__(self, name): self.name = name class TestDisabledTools: """Test suite for DISABLED_TOOLS functionality.""" def test_parse_disabled_tools_empty(self): """Empty string returns empty set (no tools disabled).""" with patch.dict(os.environ, {"DISABLED_TOOLS": ""}): assert parse_disabled_tools_env() == set() def test_parse_disabled_tools_not_set(self): """Unset variable returns empty set.""" with patch.dict(os.environ, {}, clear=True): # Ensure DISABLED_TOOLS is not in environment if "DISABLED_TOOLS" in os.environ: del os.environ["DISABLED_TOOLS"] assert parse_disabled_tools_env() == set() def test_parse_disabled_tools_single(self): """Single tool name parsed correctly.""" with patch.dict(os.environ, {"DISABLED_TOOLS": "debug"}): assert parse_disabled_tools_env() == {"debug"} def test_parse_disabled_tools_multiple(self): """Multiple tools with spaces parsed correctly.""" with patch.dict(os.environ, {"DISABLED_TOOLS": "debug, analyze, refactor"}): assert parse_disabled_tools_env() == {"debug", "analyze", "refactor"} def test_parse_disabled_tools_extra_spaces(self): """Extra spaces and empty items handled correctly.""" with patch.dict(os.environ, {"DISABLED_TOOLS": " debug , , analyze , "}): assert parse_disabled_tools_env() == {"debug", "analyze"} def test_parse_disabled_tools_duplicates(self): """Duplicate entries handled correctly (set removes duplicates).""" with patch.dict(os.environ, {"DISABLED_TOOLS": "debug,analyze,debug"}): assert parse_disabled_tools_env() == {"debug", "analyze"} def test_tool_filtering_logic(self): """Test the complete filtering logic using the actual server functions.""" # Simulate ALL_TOOLS ALL_TOOLS = { "chat": MockTool("chat"), "debug": MockTool("debug"), "analyze": MockTool("analyze"), "version": MockTool("version"), "listmodels": MockTool("listmodels"), } # Test case 1: No tools disabled disabled_tools = set() enabled_tools = apply_tool_filter(ALL_TOOLS, disabled_tools) assert len(enabled_tools) == 5 # All tools included assert set(enabled_tools.keys()) == set(ALL_TOOLS.keys()) # Test case 2: Disable some regular tools disabled_tools = {"debug", "analyze"} enabled_tools = apply_tool_filter(ALL_TOOLS, disabled_tools) assert len(enabled_tools) == 3 # chat, version, listmodels assert "debug" not in enabled_tools assert "analyze" not in enabled_tools assert "chat" in enabled_tools assert "version" in enabled_tools assert "listmodels" in enabled_tools # Test case 3: Attempt to disable essential tools disabled_tools = {"version", "chat"} enabled_tools = apply_tool_filter(ALL_TOOLS, disabled_tools) assert "version" in enabled_tools # Essential tool not disabled assert "chat" not in enabled_tools # Regular tool disabled assert "listmodels" in enabled_tools # Essential tool included def test_unknown_tools_warning(self, caplog): """Test that unknown tool names generate appropriate warnings.""" ALL_TOOLS = { "chat": MockTool("chat"), "debug": MockTool("debug"), "analyze": MockTool("analyze"), "version": MockTool("version"), "listmodels": MockTool("listmodels"), } disabled_tools = {"chat", "unknown_tool", "another_unknown"} with caplog.at_level(logging.WARNING): validate_disabled_tools(disabled_tools, ALL_TOOLS) assert "Unknown tools in DISABLED_TOOLS: ['another_unknown', 'unknown_tool']" in caplog.text def test_essential_tools_warning(self, caplog): """Test warning when trying to disable essential tools.""" ALL_TOOLS = { "chat": MockTool("chat"), "debug": MockTool("debug"), "analyze": MockTool("analyze"), "version": MockTool("version"), "listmodels": MockTool("listmodels"), } disabled_tools = {"version", "chat", "debug"} with caplog.at_level(logging.WARNING): validate_disabled_tools(disabled_tools, ALL_TOOLS) assert "Cannot disable essential tools: ['version']" in caplog.text @pytest.mark.parametrize( "env_value,expected", [ ("", set()), # Empty string (" ", set()), # Only spaces (",,,", set()), # Only commas ("chat", {"chat"}), # Single tool ("chat,debug", {"chat", "debug"}), # Multiple tools ("chat, debug, analyze", {"chat", "debug", "analyze"}), # With spaces ("chat,debug,chat", {"chat", "debug"}), # Duplicates ], ) def test_parse_disabled_tools_parametrized(self, env_value, expected): """Parametrized tests for various input formats.""" with patch.dict(os.environ, {"DISABLED_TOOLS": env_value}): assert parse_disabled_tools_env() == expected ================================================ FILE: tests/test_docker_claude_desktop_integration.py ================================================ """ Tests for Docker integration with Claude Desktop MCP """ import json import os import tempfile from pathlib import Path import pytest class TestDockerClaudeDesktopIntegration: """Test Docker integration with Claude Desktop""" @pytest.fixture(autouse=True) def setup(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent def test_mcp_config_docker_run_format(self): """Test MCP configuration for direct docker run""" config = { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "/path/to/.env", "-v", "/path/to/logs:/app/logs", "pal-mcp-server:latest", ], } } } # Validate configuration structure assert "mcpServers" in config assert "pal-mcp" in config["mcpServers"] assert config["mcpServers"]["pal-mcp"]["command"] == "docker" args = config["mcpServers"]["pal-mcp"]["args"] assert "run" in args assert "--rm" in args assert "-i" in args assert "--env-file" in args def test_mcp_config_docker_compose_format(self): """Test MCP configuration for docker-compose run""" config = { "mcpServers": { "pal-mcp": { "command": "docker-compose", "args": ["-f", "/path/to/docker-compose.yml", "run", "--rm", "pal-mcp"], } } } # Validate configuration structure assert config["mcpServers"]["pal-mcp"]["command"] == "docker-compose" args = config["mcpServers"]["pal-mcp"]["args"] assert "-f" in args assert "run" in args assert "--rm" in args assert "pal-mcp" in args def test_mcp_config_environment_variables(self): """Test MCP configuration with inline environment variables""" config = { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", "-e", "GEMINI_API_KEY=test_key", "-e", "LOG_LEVEL=INFO", "pal-mcp-server:latest", ], } } } args = config["mcpServers"]["pal-mcp"]["args"] # Check that environment variables are properly formatted env_args = [arg for arg in args if arg.startswith("-e")] assert len(env_args) > 0, "Environment variables should be present" # Check for API key environment variable api_key_present = any("GEMINI_API_KEY=" in args[i + 1] for i, arg in enumerate(args[:-1]) if arg == "-e") assert api_key_present, "API key environment variable should be set" def test_windows_path_format(self): """Test Windows-specific path formatting""" windows_config = { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "C:/Users/User/pal-mcp-server/.env", "-v", "C:/Users/User/pal-mcp-server/logs:/app/logs", "pal-mcp-server:latest", ], } } } args = windows_config["mcpServers"]["pal-mcp"]["args"] # Check Windows path format windows_paths = [arg for arg in args if arg.startswith("C:/")] assert len(windows_paths) > 0, "Windows paths should use forward slashes" for path in windows_paths: assert "\\" not in path, "Windows paths should use forward slashes" def test_mcp_config_validation(self): """Test validation of MCP configuration""" # Valid configuration valid_config = { "mcpServers": {"pal-mcp": {"command": "docker", "args": ["run", "--rm", "-i", "pal-mcp-server:latest"]}} } # Validate JSON serialization config_json = json.dumps(valid_config) loaded_config = json.loads(config_json) assert loaded_config == valid_config def test_mcp_stdio_communication(self): """Test that MCP configuration supports stdio communication""" config = { "mcpServers": { "pal-mcp": { "command": "docker", "args": [ "run", "--rm", "-i", # Interactive mode for stdio "pal-mcp-server:latest", ], } } } args = config["mcpServers"]["pal-mcp"]["args"] # Check for interactive mode assert "-i" in args, "Interactive mode required for stdio communication" # Should not expose network ports for stdio communication port_args = [arg for arg in args if arg.startswith("-p")] assert len(port_args) == 0, "No ports should be exposed for stdio mode" def test_docker_image_reference(self): """Test that Docker image is properly referenced""" configs = [ {"image": "pal-mcp-server:latest"}, {"image": "pal-mcp-server:v1.0.0"}, {"image": "registry/pal-mcp-server:latest"}, ] for config in configs: image = config["image"] # Basic image format validation assert ":" in image, "Image should have a tag" assert len(image.split(":")) == 2, "Image should have exactly one tag" @pytest.fixture def temp_mcp_config(self): """Create temporary MCP configuration file""" config = { "mcpServers": { "pal-mcp": { "command": "docker", "args": ["run", "--rm", "-i", "--env-file", "/tmp/.env", "pal-mcp-server:latest"], } } } with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False, encoding="utf-8") as f: json.dump(config, f, indent=2) temp_file_path = f.name yield temp_file_path os.unlink(temp_file_path) def test_mcp_config_file_parsing(self, temp_mcp_config): """Test parsing of MCP configuration file""" # Read and parse the temporary config file with open(temp_mcp_config, encoding="utf-8") as f: config = json.load(f) assert "mcpServers" in config assert "pal-mcp" in config["mcpServers"] def test_environment_file_integration(self): """Test integration with .env file""" # Test .env file format expected by Docker env_content = """GEMINI_API_KEY=test_key OPENAI_API_KEY=test_key_2 LOG_LEVEL=INFO DEFAULT_MODEL=auto """ # Parse environment content env_vars = {} for line in env_content.strip().split("\n"): if "=" in line and not line.startswith("#"): key, value = line.split("=", 1) env_vars[key] = value # Validate required environment variables assert "GEMINI_API_KEY" in env_vars assert len(env_vars["GEMINI_API_KEY"]) > 0 def test_docker_volume_mount_paths(self): """Test Docker volume mount path configurations""" mount_configs = [ {"host": "./logs", "container": "/app/logs"}, {"host": "/absolute/path/logs", "container": "/app/logs"}, {"host": "C:/Windows/path/logs", "container": "/app/logs"}, ] for config in mount_configs: mount_arg = f"{config['host']}:{config['container']}" # Validate mount format assert ":" in mount_arg parts = mount_arg.split(":") assert len(parts) >= 2 assert parts[-1].startswith("/"), "Container path should be absolute" class TestDockerMCPErrorHandling: """Test error handling for Docker MCP integration""" def test_missing_docker_image_handling(self): """Test handling of missing Docker image""" # This would test what happens when the image doesn't exist # In practice, Claude Desktop would show an error nonexistent_config = { "mcpServers": {"pal-mcp": {"command": "docker", "args": ["run", "--rm", "-i", "nonexistent:latest"]}} } # Configuration should be valid even if image doesn't exist assert "pal-mcp" in nonexistent_config["mcpServers"] def test_invalid_env_file_path(self): """Test handling of invalid .env file path""" config_with_invalid_env = { "mcpServers": { "pal-mcp": { "command": "docker", "args": ["run", "--rm", "-i", "--env-file", "/nonexistent/.env", "pal-mcp-server:latest"], } } } # Configuration structure should still be valid args = config_with_invalid_env["mcpServers"]["pal-mcp"]["args"] assert "--env-file" in args def test_docker_permission_issues(self): """Test configuration for potential Docker permission issues""" # On some systems, Docker requires specific permissions # The configuration should work with both cases configs = [ # Regular Docker command {"command": "docker"}, # Sudo Docker command (if needed) {"command": "sudo", "extra_args": ["docker"]}, ] for config in configs: assert len(config["command"]) > 0 def test_resource_limit_configurations(self): """Test Docker resource limit configurations""" config_with_limits = { "mcpServers": { "pal-mcp": { "command": "docker", "args": ["run", "--rm", "-i", "--memory=512m", "--cpus=1.0", "pal-mcp-server:latest"], } } } args = config_with_limits["mcpServers"]["pal-mcp"]["args"] # Check for resource limits memory_limit = any("--memory" in arg for arg in args) cpu_limit = any("--cpus" in arg for arg in args) assert memory_limit or cpu_limit, "Resource limits should be configurable" ================================================ FILE: tests/test_docker_config_complete.py ================================================ """ Complete configuration test for Docker MCP """ import os from pathlib import Path from unittest.mock import patch import pytest class TestDockerMCPConfiguration: """Docker MCP configuration tests""" def test_dockerfile_configuration(self): """Test Dockerfile configuration""" project_root = Path(__file__).parent.parent dockerfile = project_root / "Dockerfile" if not dockerfile.exists(): pytest.skip("Dockerfile not found") content = dockerfile.read_text() # Essential checks assert "FROM python:" in content assert "COPY" in content or "ADD" in content assert "server.py" in content # Recommended security checks security_checks = [ "USER " in content, # Non-root user "WORKDIR" in content, # Defined working directory ] # At least one security practice should be present if any(security_checks): assert True, "Security best practices detected" def test_environment_file_template(self): """Test environment file template""" project_root = Path(__file__).parent.parent env_example = project_root / ".env.example" if env_example.exists(): content = env_example.read_text() # Essential variables essential_vars = ["GEMINI_API_KEY", "OPENAI_API_KEY", "LOG_LEVEL"] for var in essential_vars: assert f"{var}=" in content, f"Variable {var} missing" # Docker-specific variables should also be present docker_vars = ["COMPOSE_PROJECT_NAME", "TZ", "LOG_MAX_SIZE"] for var in docker_vars: assert f"{var}=" in content, f"Docker variable {var} missing" def test_logs_directory_setup(self): """Test logs directory setup""" project_root = Path(__file__).parent.parent logs_dir = project_root / "logs" # The logs directory should exist or be creatable if not logs_dir.exists(): try: logs_dir.mkdir(exist_ok=True) created = True except Exception: created = False assert created, "Logs directory should be creatable" else: assert logs_dir.is_dir(), "logs should be a directory" class TestDockerCommandValidation: """Docker command validation tests""" @patch("subprocess.run") def test_docker_build_command(self, mock_run): """Test docker build command""" mock_run.return_value.returncode = 0 # Standard build command build_cmd = ["docker", "build", "-t", "pal-mcp-server:latest", "."] import subprocess subprocess.run(build_cmd, capture_output=True) mock_run.assert_called_once() @patch("subprocess.run") def test_docker_run_mcp_command(self, mock_run): """Test docker run command for MCP""" mock_run.return_value.returncode = 0 # Run command for MCP run_cmd = [ "docker", "run", "--rm", "-i", "--env-file", ".env", "-v", "logs:/app/logs", "pal-mcp-server:latest", "python", "server.py", ] import subprocess subprocess.run(run_cmd, capture_output=True) mock_run.assert_called_once() def test_docker_command_structure(self): """Test Docker command structure""" # Recommended MCP command mcp_cmd = [ "docker", "run", "--rm", "-i", "--env-file", "/path/to/.env", "-v", "/path/to/logs:/app/logs", "pal-mcp-server:latest", "python", "server.py", ] # Structure checks assert mcp_cmd[0] == "docker" assert "run" in mcp_cmd assert "--rm" in mcp_cmd # Automatic cleanup assert "-i" in mcp_cmd # Interactive mode assert "--env-file" in mcp_cmd # Environment variables assert "pal-mcp-server:latest" in mcp_cmd # Image class TestIntegrationChecks: """Integration checks""" def test_complete_setup_checklist(self): """Test complete setup checklist""" project_root = Path(__file__).parent.parent # Checklist for essential files essential_files = { "Dockerfile": project_root / "Dockerfile", "server.py": project_root / "server.py", "requirements.txt": project_root / "requirements.txt", "docker-compose.yml": project_root / "docker-compose.yml", } missing_files = [] for name, path in essential_files.items(): if not path.exists(): missing_files.append(name) # Allow some missing files for flexibility critical_files = ["Dockerfile", "server.py"] missing_critical = [f for f in missing_files if f in critical_files] assert not missing_critical, f"Critical files missing: {missing_critical}" def test_mcp_integration_readiness(self): """Test MCP integration readiness""" project_root = Path(__file__).parent.parent # MCP integration checks checks = { "dockerfile": (project_root / "Dockerfile").exists(), "server_script": (project_root / "server.py").exists(), "logs_dir": (project_root / "logs").exists() or True, } # At least critical elements must be present critical_checks = ["dockerfile", "server_script"] missing_critical = [k for k in critical_checks if not checks[k]] assert not missing_critical, f"Critical elements missing: {missing_critical}" # Readiness score ready_score = sum(checks.values()) / len(checks) assert ready_score >= 0.75, f"Insufficient readiness score: {ready_score:.2f}" class TestErrorHandling: """Error handling tests""" def test_missing_api_key_handling(self): """Test handling of missing API key""" # Simulate environment without API keys with patch.dict(os.environ, {}, clear=True): api_keys = [os.getenv("GEMINI_API_KEY"), os.getenv("OPENAI_API_KEY"), os.getenv("XAI_API_KEY")] has_api_key = any(key for key in api_keys) # No key should be present assert not has_api_key, "No API key detected (expected for test)" # System should handle this gracefully error_handled = True # Simulate error handling assert error_handled, "API key error handling implemented" def test_docker_not_available_handling(self): """Test handling of Docker not available""" @patch("subprocess.run") def simulate_docker_unavailable(mock_run): # Simulate Docker not available mock_run.side_effect = FileNotFoundError("docker: command not found") try: import subprocess subprocess.run(["docker", "--version"], capture_output=True) docker_available = True except FileNotFoundError: docker_available = False # Docker is not available - expected error assert not docker_available, "Docker unavailable (simulation)" # System should provide a clear error message error_message_clear = True # Simulation assert error_message_clear, "Clear Docker error message" simulate_docker_unavailable() if __name__ == "__main__": pytest.main([__file__, "-v"]) ================================================ FILE: tests/test_docker_healthcheck.py ================================================ """ Tests for Docker health check functionality """ import os import subprocess from pathlib import Path from unittest.mock import patch import pytest class TestDockerHealthCheck: """Test Docker health check implementation""" @pytest.fixture(autouse=True) def setup(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent self.healthcheck_script = self.project_root / "docker" / "scripts" / "healthcheck.py" def test_healthcheck_script_exists(self): """Test that health check script exists""" assert self.healthcheck_script.exists(), "healthcheck.py must exist" def test_healthcheck_script_executable(self): """Test that health check script is executable""" if not self.healthcheck_script.exists(): pytest.skip("healthcheck.py not found") # Check if script has Python shebang content = self.healthcheck_script.read_text() assert content.startswith("#!/usr/bin/env python"), "Health check script must have Python shebang" @patch("subprocess.run") def test_process_check_success(self, mock_run): """Test successful process check""" # Mock successful pgrep command mock_run.return_value.returncode = 0 mock_run.return_value.stdout = "12345\n" # Import and test the function (if we can access it) # This would require the healthcheck module to be importable result = subprocess.run(["pgrep", "-f", "server.py"], capture_output=True, text=True, timeout=10) assert result.returncode == 0 @patch("subprocess.run") def test_process_check_failure(self, mock_run): """Test failed process check""" # Mock failed pgrep command mock_run.return_value.returncode = 1 mock_run.return_value.stderr = "No such process" result = subprocess.run(["pgrep", "-f", "server.py"], capture_output=True, text=True, timeout=10) assert result.returncode == 1 def test_critical_modules_import(self): """Test that critical modules can be imported""" critical_modules = ["json", "os", "sys", "pathlib"] for module_name in critical_modules: try: __import__(module_name) except ImportError: pytest.fail(f"Critical module {module_name} cannot be imported") def test_optional_modules_graceful_failure(self): """Test graceful handling of optional module import failures""" optional_modules = ["mcp", "google.genai", "openai"] for module_name in optional_modules: try: __import__(module_name) except ImportError: # This is expected in test environment pass def test_log_directory_check(self): """Test log directory health check logic""" # Test with existing directory test_dir = self.project_root / "logs" if test_dir.exists(): assert os.access(test_dir, os.W_OK), "Logs directory must be writable" def test_health_check_timeout_handling(self): """Test that health checks handle timeouts properly""" timeout_duration = 10 # Mock a command that would timeout with patch("subprocess.run") as mock_run: mock_run.side_effect = subprocess.TimeoutExpired(["test"], timeout_duration) with pytest.raises(subprocess.TimeoutExpired): subprocess.run(["sleep", "20"], capture_output=True, text=True, timeout=timeout_duration) def test_health_check_docker_configuration(self): """Test health check configuration in Docker setup""" compose_file = self.project_root / "docker-compose.yml" if compose_file.exists(): content = compose_file.read_text() # Check for health check configuration assert "healthcheck:" in content, "Health check must be configured" assert "healthcheck.py" in content, "Health check script must be referenced" assert "interval:" in content, "Health check interval must be set" assert "timeout:" in content, "Health check timeout must be set" class TestDockerHealthCheckIntegration: """Integration tests for Docker health checks""" def test_dockerfile_health_check_setup(self): """Test that Dockerfile includes health check setup""" project_root = Path(__file__).parent.parent dockerfile = project_root / "Dockerfile" if dockerfile.exists(): content = dockerfile.read_text() # Check that health check script is copied script_copied = ("COPY" in content and "healthcheck.py" in content) or "COPY . ." in content assert script_copied, "Health check script must be copied to container" def test_health_check_failure_scenarios(self): """Test various health check failure scenarios""" failure_scenarios = [ {"type": "process_not_found", "expected": False}, {"type": "import_error", "expected": False}, {"type": "permission_error", "expected": False}, {"type": "timeout_error", "expected": False}, ] for scenario in failure_scenarios: # Each scenario should result in health check failure assert scenario["expected"] is False def test_health_check_recovery(self): """Test health check recovery after transient failures""" # Test that health checks can recover from temporary issues recovery_scenarios = [ {"initial_state": "failing", "final_state": "healthy"}, {"initial_state": "timeout", "final_state": "healthy"}, ] for scenario in recovery_scenarios: assert scenario["final_state"] == "healthy" @patch.dict(os.environ, {}, clear=True) def test_health_check_with_missing_env_vars(self): """Test health check behavior with missing environment variables""" # Health check should still work even without API keys # (it tests system health, not API connectivity) required_vars = ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY"] # Verify no API keys are set for var in required_vars: assert os.getenv(var) is None def test_health_check_performance(self): """Test that health checks complete within reasonable time""" # Health checks should be fast to avoid impacting container startup max_execution_time = 30 # seconds # Mock a health check execution import time start_time = time.time() # Simulate health check operations time.sleep(0.1) # Simulate actual work execution_time = time.time() - start_time assert ( execution_time < max_execution_time ), f"Health check took {execution_time}s, should be < {max_execution_time}s" ================================================ FILE: tests/test_docker_implementation.py ================================================ """ Unit tests for Docker configuration and implementation of PAL MCP Server This module tests: - Docker and MCP configuration - Environment variable validation - Docker commands - Integration with Claude Desktop - stdio communication """ import json import os import subprocess import sys import tempfile from pathlib import Path from unittest.mock import patch import pytest # Import project modules sys.path.insert(0, str(Path(__file__).parent.parent)) class TestDockerConfiguration: """Tests for Docker configuration of PAL MCP Server""" def setup_method(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent self.docker_compose_path = self.project_root / "docker-compose.yml" self.dockerfile_path = self.project_root / "Dockerfile" def test_dockerfile_exists(self): """Test that Dockerfile exists and is valid""" assert self.dockerfile_path.exists(), "Dockerfile must exist" # Check Dockerfile content content = self.dockerfile_path.read_text() assert "FROM python:" in content, "Dockerfile must have a Python base" # Dockerfile uses COPY . . to copy all code assert "COPY . ." in content or "COPY --chown=" in content, "Dockerfile must copy source code" assert "CMD" in content, "Dockerfile must have a default command" assert "server.py" in content, "Dockerfile must reference server.py" def test_docker_compose_configuration(self): """Test that docker-compose.yml is properly configured""" assert self.docker_compose_path.exists(), "docker-compose.yml must exist" # Basic YAML syntax check content = self.docker_compose_path.read_text() assert "services:" in content, "docker-compose.yml must have services" assert "pal-mcp" in content, "Service pal-mcp must be defined" assert "build:" in content, "Build configuration must be present" def test_environment_file_template(self): """Test that an .env file template exists""" env_example_path = self.project_root / ".env.example" if env_example_path.exists(): content = env_example_path.read_text() assert "GEMINI_API_KEY=" in content, "Template must contain GEMINI_API_KEY" assert "OPENAI_API_KEY=" in content, "Template must contain OPENAI_API_KEY" assert "LOG_LEVEL=" in content, "Template must contain LOG_LEVEL" class TestDockerCommands: """Tests for Docker commands""" def setup_method(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent @patch("subprocess.run") def test_docker_build_command(self, mock_run): """Test that the docker build command works""" mock_run.return_value.returncode = 0 mock_run.return_value.stdout = "Successfully built" # Simulate docker build subprocess.run( ["docker", "build", "-t", "pal-mcp-server:latest", str(self.project_root)], capture_output=True, text=True ) mock_run.assert_called_once() @patch("subprocess.run") def test_docker_run_command_structure(self, mock_run): """Test that the docker run command has the correct structure""" mock_run.return_value.returncode = 0 # Recommended MCP command cmd = [ "docker", "run", "--rm", "-i", "--env-file", ".env", "-v", "logs:/app/logs", "pal-mcp-server:latest", "python", "server.py", ] # Check command structure assert cmd[0] == "docker", "First command must be docker" assert "run" in cmd, "Must contain run" assert "--rm" in cmd, "Must contain --rm for cleanup" assert "-i" in cmd, "Must contain -i for stdio" assert "--env-file" in cmd, "Must contain --env-file" assert "pal-mcp-server:latest" in cmd, "Must reference the image" @patch("subprocess.run") def test_docker_health_check(self, mock_run): """Test Docker health check""" mock_run.return_value.returncode = 0 mock_run.return_value.stdout = "Health check passed" # Simulate health check subprocess.run( ["docker", "run", "--rm", "pal-mcp-server:latest", "python", "/usr/local/bin/healthcheck.py"], capture_output=True, text=True, ) mock_run.assert_called_once() class TestEnvironmentValidation: """Tests for environment variable validation""" def test_required_api_keys_validation(self): """Test that API key validation works""" # Test with valid API key with patch.dict(os.environ, {"GEMINI_API_KEY": "test_key"}): # Here we should have a function that validates the keys # Let's simulate the validation logic has_api_key = bool(os.getenv("GEMINI_API_KEY") or os.getenv("OPENAI_API_KEY") or os.getenv("XAI_API_KEY")) assert has_api_key, "At least one API key must be present" # Test without API key with patch.dict(os.environ, {}, clear=True): has_api_key = bool(os.getenv("GEMINI_API_KEY") or os.getenv("OPENAI_API_KEY") or os.getenv("XAI_API_KEY")) assert not has_api_key, "No API key should be present" def test_environment_file_parsing(self): """Test parsing of the .env file""" # Create a temporary .env file env_content = """ # Test environment file GEMINI_API_KEY=test_gemini_key OPENAI_API_KEY=test_openai_key LOG_LEVEL=INFO DEFAULT_MODEL=auto """ with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False) as f: f.write(env_content) env_file_path = f.name try: # Simulate parsing of the .env file env_vars = {} with open(env_file_path) as f: for line in f: line = line.strip() if line and not line.startswith("#") and "=" in line: key, value = line.split("=", 1) env_vars[key] = value assert "GEMINI_API_KEY" in env_vars, "GEMINI_API_KEY must be parsed" assert env_vars["GEMINI_API_KEY"] == "test_gemini_key", "Value must be correct" assert env_vars["LOG_LEVEL"] == "INFO", "LOG_LEVEL must be parsed" finally: os.unlink(env_file_path) class TestMCPIntegration: """Tests for MCP integration with Claude Desktop""" def test_mcp_configuration_generation(self): """Test MCP configuration generation""" # Expected MCP configuration expected_config = { "servers": { "pal-docker": { "command": "docker", "args": [ "run", "--rm", "-i", "--env-file", "/path/to/.env", "-v", "/path/to/logs:/app/logs", "pal-mcp-server:latest", "python", "server.py", ], "env": {"DOCKER_BUILDKIT": "1"}, } } } # Check structure assert "servers" in expected_config pal_docker = expected_config["servers"]["pal-docker"] assert pal_docker["command"] == "docker" assert "run" in pal_docker["args"] assert "--rm" in pal_docker["args"] assert "-i" in pal_docker["args"] def test_stdio_communication_structure(self): """Test structure of stdio communication""" # Simulate an MCP message mcp_message = {"jsonrpc": "2.0", "method": "initialize", "params": {}, "id": 1} # Check that the message is valid JSON json_str = json.dumps(mcp_message) parsed = json.loads(json_str) assert parsed["jsonrpc"] == "2.0" assert "method" in parsed assert "id" in parsed class TestDockerSecurity: """Tests for Docker security""" def test_non_root_user_configuration(self): """Test that the container uses a non-root user""" dockerfile_path = Path(__file__).parent.parent / "Dockerfile" if dockerfile_path.exists(): content = dockerfile_path.read_text() # Check that a non-root user is configured assert "USER " in content or "useradd" in content, "Dockerfile should configure a non-root user" def test_readonly_filesystem_configuration(self): """Test read-only filesystem configuration""" # This configuration should be in docker-compose.yml or Dockerfile docker_compose_path = Path(__file__).parent.parent / "docker-compose.yml" if docker_compose_path.exists(): content = docker_compose_path.read_text() # Look for security configurations security_indicators = ["read_only", "tmpfs", "security_opt", "cap_drop"] # At least one security indicator should be present # Note: This test can be adjusted according to the actual implementation security_found = any(indicator in content for indicator in security_indicators) assert security_found or True # Flexible test def test_environment_variable_security(self): """Test that sensitive environment variables are not hardcoded""" dockerfile_path = Path(__file__).parent.parent / "Dockerfile" if dockerfile_path.exists(): content = dockerfile_path.read_text() # Check that no API keys are hardcoded sensitive_patterns = ["API_KEY=sk-", "API_KEY=gsk_", "API_KEY=xai-"] for pattern in sensitive_patterns: assert pattern not in content, f"Sensitive API key detected in Dockerfile: {pattern}" class TestDockerPerformance: """Tests for Docker performance""" def test_image_size_optimization(self): """Test that the Docker image is not excessively large""" # This test would require docker to be executed # Simulate size check expected_max_size_mb = 500 # 500MB max # In production, we would do: # result = subprocess.run(['docker', 'images', '--format', '{{.Size}}', 'pal-mcp-server:latest']) # Here we simulate simulated_size = "294MB" # Current observed size size_mb = float(simulated_size.replace("MB", "")) assert size_mb <= expected_max_size_mb, f"Image too large: {size_mb}MB > {expected_max_size_mb}MB" def test_startup_time_expectations(self): """Test startup time expectations""" # Conceptual test - in production we would measure actual time expected_startup_time_seconds = 10 # Simulate a startup time measurement simulated_startup_time = 3 # seconds assert ( simulated_startup_time <= expected_startup_time_seconds ), f"Startup time too long: {simulated_startup_time}s" @pytest.fixture def temp_project_dir(): """Fixture to create a temporary project directory""" with tempfile.TemporaryDirectory() as temp_dir: temp_path = Path(temp_dir) # Create base structure (temp_path / "logs").mkdir() # Create base files (temp_path / "server.py").write_text("# Mock server.py") (temp_path / "Dockerfile").write_text( """ FROM python:3.11-slim COPY server.py /app/ CMD ["python", "/app/server.py"] """ ) yield temp_path class TestIntegration: """Integration tests for the entire Docker setup""" def test_complete_docker_setup_validation(self, temp_project_dir): """Test complete integration of Docker setup""" # Create an .env file env_content = """ GEMINI_API_KEY=test_key LOG_LEVEL=INFO """ (temp_project_dir / ".env").write_text(env_content) # Validate that everything is in place assert (temp_project_dir / ".env").exists() assert (temp_project_dir / "Dockerfile").exists() assert (temp_project_dir / "logs").exists() # Validate basic Docker command structure docker_cmd = [ "docker", "run", "--rm", "-i", "--env-file", ".env", "pal-mcp-server:latest", "python", "server.py", ] # Basic structure checks assert docker_cmd[0] == "docker" assert "run" in docker_cmd assert "--rm" in docker_cmd assert "--env-file" in docker_cmd if __name__ == "__main__": # Run tests pytest.main([__file__, "-v", "--tb=short"]) ================================================ FILE: tests/test_docker_mcp_validation.py ================================================ """ Validation test for Docker MCP implementation """ import json import os import subprocess import sys import tempfile from pathlib import Path from unittest.mock import patch import pytest # Add project root to path sys.path.insert(0, str(Path(__file__).parent.parent)) class TestDockerMCPValidation: """Validation tests for Docker MCP""" @pytest.fixture(autouse=True) def setup(self): """Automatic setup for each test""" self.project_root = Path(__file__).parent.parent self.dockerfile_path = self.project_root / "Dockerfile" def test_dockerfile_exists_and_valid(self): """Test Dockerfile existence and validity""" assert self.dockerfile_path.exists(), "Missing Dockerfile" content = self.dockerfile_path.read_text() assert "FROM python:" in content, "Python base required" assert "server.py" in content, "server.py must be copied" @patch("subprocess.run") def test_docker_command_validation(self, mock_run): """Test Docker command validation""" mock_run.return_value.returncode = 0 # Standard Docker MCP command cmd = ["docker", "run", "--rm", "-i", "--env-file", ".env", "pal-mcp-server:latest", "python", "server.py"] subprocess.run(cmd, capture_output=True) mock_run.assert_called_once_with(cmd, capture_output=True) def test_environment_variables_validation(self): """Test environment variables validation""" required_vars = ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY"] # Test with variable present with patch.dict(os.environ, {"GEMINI_API_KEY": "test"}): has_key = any(os.getenv(var) for var in required_vars) assert has_key, "At least one API key required" # Test without variables with patch.dict(os.environ, {}, clear=True): has_key = any(os.getenv(var) for var in required_vars) assert not has_key, "No key should be present" def test_docker_security_configuration(self): """Test Docker security configuration""" if not self.dockerfile_path.exists(): pytest.skip("Dockerfile not found") content = self.dockerfile_path.read_text() # Check non-root user has_user_config = "USER " in content or "useradd" in content or "adduser" in content # Note: The test can be adjusted according to implementation if has_user_config: assert True, "User configuration found" else: # Warning instead of failure for flexibility pytest.warns(UserWarning, "Consider adding a non-root user") class TestDockerIntegration: """Docker-MCP integration tests""" @pytest.fixture def temp_env_file(self): """Fixture for temporary .env file""" content = """GEMINI_API_KEY=test_key LOG_LEVEL=INFO DEFAULT_MODEL=auto """ with tempfile.NamedTemporaryFile(mode="w", suffix=".env", delete=False, encoding="utf-8") as f: f.write(content) temp_file_path = f.name # File is now closed, can yield yield temp_file_path os.unlink(temp_file_path) def test_env_file_parsing(self, temp_env_file): """Test .env file parsing""" env_vars = {} with open(temp_env_file, encoding="utf-8") as f: for line in f: line = line.strip() if line and not line.startswith("#") and "=" in line: key, value = line.split("=", 1) env_vars[key] = value assert "GEMINI_API_KEY" in env_vars assert env_vars["GEMINI_API_KEY"] == "test_key" assert env_vars["LOG_LEVEL"] == "INFO" def test_mcp_message_structure(self): """Test MCP message structure""" message = {"jsonrpc": "2.0", "method": "initialize", "params": {}, "id": 1} # Check JSON serialization json_str = json.dumps(message) parsed = json.loads(json_str) assert parsed["jsonrpc"] == "2.0" assert "method" in parsed assert "id" in parsed class TestDockerPerformance: """Docker performance tests""" def test_image_size_expectation(self): """Test expected image size""" # Maximum expected size (in MB) max_size_mb = 500 # Simulation - in reality, Docker would be queried simulated_size = 294 # MB observed assert simulated_size <= max_size_mb, f"Image too large: {simulated_size}MB > {max_size_mb}MB" def test_startup_performance(self): """Test startup performance""" max_startup_seconds = 10 simulated_startup = 3 # seconds assert simulated_startup <= max_startup_seconds, f"Startup too slow: {simulated_startup}s" @pytest.mark.integration class TestFullIntegration: """Full integration tests""" def test_complete_setup_simulation(self): """Simulate complete setup""" # Simulate all required components components = { "dockerfile": True, "mcp_config": True, "env_template": True, "documentation": True, } # Check that all components are present missing = [k for k, v in components.items() if not v] assert not missing, f"Missing components: {missing}" def test_docker_mcp_workflow(self): """Test complete Docker-MCP workflow""" # Workflow steps workflow_steps = [ "build_image", "create_env_file", "configure_mcp_json", "test_docker_run", "validate_mcp_communication", ] # Simulate each step for step in workflow_steps: # In reality, each step would be tested individually assert step is not None, f"Step {step} not defined" if __name__ == "__main__": # Run tests with pytest pytest.main([__file__, "-v"]) ================================================ FILE: tests/test_docker_security.py ================================================ """ Tests for Docker security configuration and best practices """ import os from pathlib import Path from unittest.mock import patch import pytest class TestDockerSecurity: """Test Docker security configuration""" @pytest.fixture(autouse=True) def setup(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent self.dockerfile_path = self.project_root / "Dockerfile" self.compose_path = self.project_root / "docker-compose.yml" def test_non_root_user_configuration(self): """Test that container runs as non-root user""" if not self.dockerfile_path.exists(): pytest.skip("Dockerfile not found") content = self.dockerfile_path.read_text() # Check for user creation or switching user_indicators = ["USER " in content, "useradd" in content, "adduser" in content, "RUN addgroup" in content] assert any(user_indicators), "Container should run as non-root user" def test_no_unnecessary_privileges(self): """Test that container doesn't request unnecessary privileges""" if not self.compose_path.exists(): pytest.skip("docker-compose.yml not found") content = self.compose_path.read_text() # Check that dangerous options are not used dangerous_options = ["privileged: true", "--privileged", "cap_add:", "SYS_ADMIN"] for option in dangerous_options: assert option not in content, f"Dangerous option {option} should not be used" def test_read_only_filesystem(self): """Test read-only filesystem configuration where applicable""" if not self.compose_path.exists(): pytest.skip("docker-compose.yml not found") content = self.compose_path.read_text() # Check for read-only configurations if "read_only:" in content: assert "read_only: true" in content, "Read-only filesystem should be properly configured" def test_environment_variable_security(self): """Test secure handling of environment variables""" # Ensure sensitive data is not hardcoded sensitive_patterns = ["password", "secret", "key", "token"] for file_path in [self.dockerfile_path, self.compose_path]: if not file_path.exists(): continue content = file_path.read_text().lower() # Check that we don't have hardcoded secrets for pattern in sensitive_patterns: # Allow variable names but not actual values lines = content.split("\n") for line in lines: if f"{pattern}=" in line and not line.strip().startswith("#"): # Check if it looks like a real value vs variable name if '"' in line or "'" in line: value_part = line.split("=")[1].strip() if len(value_part) > 10 and not value_part.startswith("$"): pytest.fail(f"Potential hardcoded secret in {file_path}: {line.strip()}") def test_network_security(self): """Test network security configuration""" if not self.compose_path.exists(): pytest.skip("docker-compose.yml not found") content = self.compose_path.read_text() # Check for custom network (better than default bridge) if "networks:" in content: assert ( "driver: bridge" in content or "external:" in content ), "Custom networks should use bridge driver or be external" def test_volume_security(self): """Test volume security configuration""" if not self.compose_path.exists(): pytest.skip("docker-compose.yml not found") content = self.compose_path.read_text() # Check that sensitive host paths are not mounted dangerous_mounts = ["/:/", "/var/run/docker.sock:", "/etc/passwd:", "/etc/shadow:", "/root:"] for mount in dangerous_mounts: assert mount not in content, f"Dangerous mount {mount} should not be used" def test_secret_management(self): """Test that secrets are properly managed""" # Check for Docker secrets usage in compose file if self.compose_path.exists(): content = self.compose_path.read_text() # If secrets are used, they should be properly configured if "secrets:" in content: assert "external: true" in content or "file:" in content, "Secrets should be external or file-based" def test_container_capabilities(self): """Test container capabilities are properly restricted""" if not self.compose_path.exists(): pytest.skip("docker-compose.yml not found") content = self.compose_path.read_text() # Check for capability restrictions if "cap_drop:" in content: assert "ALL" in content, "Should drop all capabilities by default" # If capabilities are added, they should be minimal if "cap_add:" in content: dangerous_caps = ["SYS_ADMIN", "NET_ADMIN", "SYS_PTRACE"] for cap in dangerous_caps: assert cap not in content, f"Dangerous capability {cap} should not be added" class TestDockerSecretsHandling: """Test Docker secrets and API key handling""" def test_env_file_not_in_image(self): """Test that .env files are not copied into Docker image""" project_root = Path(__file__).parent.parent dockerfile = project_root / "Dockerfile" if dockerfile.exists(): content = dockerfile.read_text() # .env files should not be copied assert "COPY .env" not in content, ".env file should not be copied into image" def test_dockerignore_for_sensitive_files(self): """Test that .dockerignore excludes sensitive files""" project_root = Path(__file__).parent.parent dockerignore = project_root / ".dockerignore" if dockerignore.exists(): content = dockerignore.read_text() sensitive_files = [".env", "*.key", "*.pem", ".git"] for file_pattern in sensitive_files: if file_pattern not in content: # Warning rather than failure for flexibility import warnings warnings.warn(f"Consider adding {file_pattern} to .dockerignore", UserWarning, stacklevel=2) @patch.dict(os.environ, {}, clear=True) def test_no_default_api_keys(self): """Test that no default API keys are present""" # Ensure no API keys are set by default api_key_vars = ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "ANTHROPIC_API_KEY"] for var in api_key_vars: assert os.getenv(var) is None, f"{var} should not have a default value" def test_api_key_format_validation(self): """Test API key format validation if implemented""" # Test cases for API key validation test_cases = [ {"key": "", "valid": False}, {"key": "test", "valid": False}, # Too short {"key": "sk-" + "x" * 40, "valid": True}, # OpenAI format {"key": "AIza" + "x" * 35, "valid": True}, # Google format ] for case in test_cases: # This would test actual validation if implemented # For now, just check the test structure assert isinstance(case["valid"], bool) assert isinstance(case["key"], str) class TestDockerComplianceChecks: """Test Docker configuration compliance with security standards""" def test_dockerfile_best_practices(self): """Test Dockerfile follows security best practices""" project_root = Path(__file__).parent.parent dockerfile = project_root / "Dockerfile" if not dockerfile.exists(): pytest.skip("Dockerfile not found") content = dockerfile.read_text() # Check for multi-stage builds (reduces attack surface) if "FROM" in content: from_count = content.count("FROM") if from_count > 1: assert "AS" in content, "Multi-stage builds should use named stages" # Check for specific user ID (better than name-only) if "USER" in content: user_lines = [line for line in content.split("\n") if line.strip().startswith("USER")] for line in user_lines: # Could be improved to check for numeric UID assert len(line.strip()) > 5, "USER directive should be specific" def test_container_security_context(self): """Test container security context configuration""" project_root = Path(__file__).parent.parent compose_file = project_root / "docker-compose.yml" if compose_file.exists(): content = compose_file.read_text() # Check for security context if configured security_options = ["security_opt:", "no-new-privileges:", "read_only:"] # At least one security option should be present security_configured = any(opt in content for opt in security_options) if not security_configured: import warnings warnings.warn("Consider adding security options to docker-compose.yml", UserWarning, stacklevel=2) ================================================ FILE: tests/test_docker_volume_persistence.py ================================================ """ Tests for Docker volume persistence functionality """ import json import os import subprocess from pathlib import Path from unittest.mock import patch import pytest class TestDockerVolumePersistence: """Test Docker volume persistence for configuration and logs""" @pytest.fixture(autouse=True) def setup(self): """Setup for each test""" self.project_root = Path(__file__).parent.parent self.docker_compose_path = self.project_root / "docker-compose.yml" def test_docker_compose_volumes_configuration(self): """Test that docker-compose.yml has proper volume configuration""" if not self.docker_compose_path.exists(): pytest.skip("docker-compose.yml not found") content = self.docker_compose_path.read_text() # Check for named volume definition assert "pal-mcp-config:" in content, "pal-mcp-config volume must be defined" assert "driver: local" in content, "Named volume must use local driver" # Check for volume mounts in service assert "./logs:/app/logs" in content, "Logs volume mount required" assert "pal-mcp-config:/app/conf" in content, "Config volume mount required" def test_persistent_volume_creation(self): """Test that persistent volumes are created correctly""" # This test checks that the volume configuration is valid # In a real environment, you might want to test actual volume creation volume_name = "pal-mcp-config" # Mock Docker command to check volume exists with patch("subprocess.run") as mock_run: mock_run.return_value.returncode = 0 mock_run.return_value.stdout = f"{volume_name}\n" # Simulate docker volume ls command result = subprocess.run(["docker", "volume", "ls", "--format", "{{.Name}}"], capture_output=True, text=True) assert volume_name in result.stdout def test_configuration_persistence_between_runs(self): """Test that configuration persists between container runs""" # This is a conceptual test - in practice you'd need a real Docker environment config_data = {"test_key": "test_value", "persistent": True} # Simulate writing config to persistent volume with patch("json.dump") as mock_dump: json.dump(config_data, mock_dump) # Simulate container restart and config retrieval with patch("json.load") as mock_load: mock_load.return_value = config_data loaded_config = json.load(mock_load) assert loaded_config == config_data assert loaded_config["persistent"] is True def test_log_persistence_configuration(self): """Test that log persistence is properly configured""" log_mount = "./logs:/app/logs" if self.docker_compose_path.exists(): content = self.docker_compose_path.read_text() assert log_mount in content, f"Log mount {log_mount} must be configured" def test_volume_backup_restore_capability(self): """Test that volumes can be backed up and restored""" # Test backup command structure backup_cmd = [ "docker", "run", "--rm", "-v", "pal-mcp-config:/data", "-v", "$(pwd):/backup", "alpine", "tar", "czf", "/backup/config-backup.tar.gz", "-C", "/data", ".", ] # Verify command structure is valid assert "pal-mcp-config:/data" in backup_cmd assert "tar" in backup_cmd assert "czf" in backup_cmd def test_volume_permissions(self): """Test that volume permissions are properly set""" # Check that logs directory has correct permissions logs_dir = self.project_root / "logs" if logs_dir.exists(): # Check that directory is writable assert os.access(logs_dir, os.W_OK), "Logs directory must be writable" # Test creating a temporary file test_file = logs_dir / "test_write_permission.tmp" try: test_file.write_text("test") assert test_file.exists() finally: if test_file.exists(): test_file.unlink() class TestDockerVolumeIntegration: """Integration tests for Docker volumes with MCP functionality""" def test_mcp_config_persistence(self): """Test that MCP configuration persists in named volume""" mcp_config = {"models": ["gemini-2.0-flash", "gpt-4"], "default_model": "auto", "thinking_mode": "high"} # Test config serialization/deserialization config_str = json.dumps(mcp_config) loaded_config = json.loads(config_str) assert loaded_config == mcp_config assert "models" in loaded_config def test_docker_compose_run_volume_usage(self): """Test that docker-compose run uses volumes correctly""" # Verify that docker-compose run inherits volume configuration # This is more of a configuration validation test compose_run_cmd = ["docker-compose", "run", "--rm", "pal-mcp"] # The command should work with the existing volume configuration assert "docker-compose" in compose_run_cmd assert "run" in compose_run_cmd assert "--rm" in compose_run_cmd def test_volume_data_isolation(self): """Test that different container instances share volume data correctly""" shared_data = {"instance_count": 0, "shared_state": "active"} # Simulate multiple container instances accessing shared volume for _ in range(3): shared_data["instance_count"] += 1 assert shared_data["shared_state"] == "active" assert shared_data["instance_count"] == 3 ================================================ FILE: tests/test_file_protection.py ================================================ """ Test file protection mechanisms to ensure MCP doesn't scan: 1. Its own directory 2. User's home directory root 3. Excluded directories """ from pathlib import Path from unittest.mock import patch from utils.file_utils import ( expand_paths, get_user_home_directory, is_home_directory_root, is_mcp_directory, ) class TestMCPDirectoryDetection: """Test MCP self-detection to prevent scanning its own code.""" def test_detect_mcp_directory_dynamically(self, tmp_path): """Test dynamic MCP directory detection based on script location.""" # The is_mcp_directory function now uses __file__ to detect MCP location # It checks if the given path is a subdirectory of the MCP server from pathlib import Path import utils.file_utils # Get the actual MCP server directory mcp_server_dir = Path(utils.file_utils.__file__).parent.parent.resolve() # Test that the MCP server directory itself is detected assert is_mcp_directory(mcp_server_dir) is True # Test that a subdirectory of MCP is also detected if (mcp_server_dir / "tools").exists(): assert is_mcp_directory(mcp_server_dir / "tools") is True def test_no_detection_on_non_mcp_directory(self, tmp_path): """Test no detection on directories outside MCP.""" # Any directory outside the MCP server should not be detected non_mcp_dir = tmp_path / "some_other_project" non_mcp_dir.mkdir() assert is_mcp_directory(non_mcp_dir) is False def test_no_detection_on_regular_directory(self, tmp_path): """Test no detection on regular project directories.""" # Create some random Python files (tmp_path / "app.py").touch() (tmp_path / "main.py").touch() (tmp_path / "utils.py").touch() assert is_mcp_directory(tmp_path) is False def test_no_detection_on_file(self, tmp_path): """Test no detection when path is a file, not directory.""" file_path = tmp_path / "test.py" file_path.touch() assert is_mcp_directory(file_path) is False def test_mcp_directory_excluded_from_scan(self, tmp_path): """Test that MCP directories are excluded during path expansion.""" # For this test, we need to mock is_mcp_directory since we can't # actually create the MCP directory structure in tmp_path from unittest.mock import patch as mock_patch # Create a project with a subdirectory we'll pretend is MCP project_root = tmp_path / "my_project" project_root.mkdir() # Add some project files (project_root / "app.py").write_text("# My app") (project_root / "config.py").write_text("# Config") # Create a subdirectory that we'll mock as MCP fake_mcp_dir = project_root / "gemini-mcp-server" fake_mcp_dir.mkdir() (fake_mcp_dir / "server.py").write_text("# MCP server") (fake_mcp_dir / "test.py").write_text("# Should not be included") # Mock is_mcp_directory to return True for our fake MCP dir def mock_is_mcp(path): return str(path).endswith("gemini-mcp-server") # Scan the project with mocked MCP detection with mock_patch("utils.file_utils.is_mcp_directory", side_effect=mock_is_mcp): files = expand_paths([str(project_root)]) # Verify project files are included but MCP files are not file_names = [Path(f).name for f in files] assert "app.py" in file_names assert "config.py" in file_names assert "test.py" not in file_names # From MCP dir assert "server.py" not in file_names # From MCP dir class TestHomeDirectoryProtection: """Test protection against scanning user's home directory root.""" def test_detect_exact_home_directory(self): """Test detection of exact home directory path.""" with patch("utils.file_utils.get_user_home_directory") as mock_home: mock_home.return_value = Path("/Users/testuser") assert is_home_directory_root(Path("/Users/testuser")) is True assert is_home_directory_root(Path("/Users/testuser/")) is True def test_allow_home_subdirectories(self): """Test that subdirectories of home are allowed.""" with patch("utils.file_utils.get_user_home_directory") as mock_home: mock_home.return_value = Path("/Users/testuser") assert is_home_directory_root(Path("/Users/testuser/projects")) is False assert is_home_directory_root(Path("/Users/testuser/Documents/code")) is False def test_detect_home_patterns_macos(self): """Test detection of macOS home directory patterns.""" # Test various macOS home patterns assert is_home_directory_root(Path("/Users/john")) is True assert is_home_directory_root(Path("/Users/jane")) is True # But subdirectories should be allowed assert is_home_directory_root(Path("/Users/john/projects")) is False def test_detect_home_patterns_linux(self): """Test detection of Linux home directory patterns.""" assert is_home_directory_root(Path("/home/ubuntu")) is True assert is_home_directory_root(Path("/home/user")) is True # But subdirectories should be allowed assert is_home_directory_root(Path("/home/ubuntu/code")) is False def test_detect_home_patterns_windows(self): """Test detection of Windows home directory patterns.""" assert is_home_directory_root(Path("C:\\Users\\John")) is True assert is_home_directory_root(Path("C:/Users/Jane")) is True # But subdirectories should be allowed assert is_home_directory_root(Path("C:\\Users\\John\\Documents")) is False def test_home_directory_excluded_from_scan(self, tmp_path): """Test that home directory root is excluded during path expansion.""" with patch("utils.file_utils.get_user_home_directory") as mock_home: mock_home.return_value = tmp_path # Try to scan home directory files = expand_paths([str(tmp_path)]) # Should return empty as home root is skipped assert files == [] class TestUserHomeEnvironmentVariable: """Test USER_HOME environment variable handling.""" def test_user_home_from_pathlib(self): """Test that get_user_home_directory uses Path.home().""" with patch("pathlib.Path.home") as mock_home: mock_home.return_value = Path("/Users/testuser") home = get_user_home_directory() assert home == Path("/Users/testuser") def test_get_home_directory_uses_pathlib(self): """Test that get_user_home_directory always uses Path.home().""" with patch("pathlib.Path.home") as mock_home: mock_home.return_value = Path("/home/testuser") home = get_user_home_directory() assert home == Path("/home/testuser") # Verify Path.home() was called mock_home.assert_called_once() def test_home_directory_on_different_platforms(self): """Test home directory detection on different platforms.""" # Test different platform home directories test_homes = [ Path("/Users/john"), # macOS Path("/home/ubuntu"), # Linux Path("C:\\Users\\John"), # Windows ] for test_home in test_homes: with patch("pathlib.Path.home") as mock_home: mock_home.return_value = test_home home = get_user_home_directory() assert home == test_home class TestExcludedDirectories: """Test that excluded directories are properly filtered.""" def test_excluded_dirs_not_scanned(self, tmp_path): """Test that directories in EXCLUDED_DIRS are skipped.""" # Create a project with various directories project = tmp_path / "project" project.mkdir() # Create some allowed files (project / "main.py").write_text("# Main") (project / "app.py").write_text("# App") # Create excluded directories with files for excluded in ["node_modules", ".git", "build", "__pycache__", ".venv"]: excluded_dir = project / excluded excluded_dir.mkdir() (excluded_dir / "test.py").write_text("# Should not be included") (excluded_dir / "data.json").write_text("{}") # Create a nested allowed directory src = project / "src" src.mkdir() (src / "utils.py").write_text("# Utils") files = expand_paths([str(project)]) file_names = [Path(f).name for f in files] # Check allowed files are included assert "main.py" in file_names assert "app.py" in file_names assert "utils.py" in file_names # Check excluded files are not included assert "test.py" not in file_names assert "data.json" not in file_names def test_new_excluded_directories(self, tmp_path): """Test newly added excluded directories like .next, .nuxt, etc.""" project = tmp_path / "webapp" project.mkdir() # Create files in new excluded directories for excluded in [".next", ".nuxt", "bower_components", ".expo"]: excluded_dir = project / excluded excluded_dir.mkdir() (excluded_dir / "generated.js").write_text("// Generated") # Create an allowed file (project / "index.js").write_text("// Index") files = expand_paths([str(project)]) file_names = [Path(f).name for f in files] assert "index.js" in file_names assert "generated.js" not in file_names class TestIntegrationScenarios: """Test realistic integration scenarios.""" def test_project_with_mcp_clone_inside(self, tmp_path): """Test scanning a project that has MCP cloned inside it.""" # Setup: User project with MCP cloned as a tool user_project = tmp_path / "my-awesome-project" user_project.mkdir() # User's project files (user_project / "README.md").write_text("# My Project") (user_project / "main.py").write_text("print('Hello')") src = user_project / "src" src.mkdir() (src / "app.py").write_text("# App code") # MCP cloned inside the project mcp = user_project / "tools" / "gemini-mcp-server" mcp.mkdir(parents=True) # Create typical MCP files (mcp / "server.py").write_text("# MCP server code") (mcp / "config.py").write_text("# MCP config") tools_dir = mcp / "tools" tools_dir.mkdir() (tools_dir / "chat.py").write_text("# Chat tool") (mcp / "LICENSE").write_text("MIT License") (mcp / "README.md").write_text("# Gemini MCP") # Also add node_modules (should be excluded) node_modules = user_project / "node_modules" node_modules.mkdir() (node_modules / "package.json").write_text("{}") # Mock is_mcp_directory for this test def mock_is_mcp(path): return "gemini-mcp-server" in str(path) with patch("utils.file_utils.is_mcp_directory", side_effect=mock_is_mcp): files = expand_paths([str(user_project)]) file_paths = [str(f) for f in files] # User files should be included assert any("my-awesome-project/README.md" in p for p in file_paths) assert any("my-awesome-project/main.py" in p for p in file_paths) assert any("src/app.py" in p for p in file_paths) # MCP files should NOT be included assert not any("gemini-mcp-server" in p for p in file_paths) assert not any("server.py" in p for p in file_paths) # node_modules should NOT be included assert not any("node_modules" in p for p in file_paths) def test_security_without_workspace_root(self, tmp_path): """Test that security still works with the new security model.""" # The system now relies on is_dangerous_path and is_home_directory_root # for security protection # Test that we can scan regular project directories project_dir = tmp_path / "my_project" project_dir.mkdir() (project_dir / "app.py").write_text("# App") files = expand_paths([str(project_dir)]) assert len(files) == 1 assert "app.py" in files[0] # Test that home directory root is still protected with patch("utils.file_utils.get_user_home_directory") as mock_home: mock_home.return_value = tmp_path # Scanning home root should return empty files = expand_paths([str(tmp_path)]) assert files == [] ================================================ FILE: tests/test_gemini_token_usage.py ================================================ """Tests for Gemini provider token usage extraction.""" import unittest from unittest.mock import Mock from providers.gemini import GeminiModelProvider class TestGeminiTokenUsage(unittest.TestCase): """Test Gemini provider token usage handling.""" def setUp(self): """Set up test fixtures.""" self.provider = GeminiModelProvider("test-key") def test_extract_usage_with_valid_tokens(self): """Test token extraction with valid token counts.""" response = Mock() response.usage_metadata = Mock() response.usage_metadata.prompt_token_count = 100 response.usage_metadata.candidates_token_count = 50 usage = self.provider._extract_usage(response) self.assertEqual(usage["input_tokens"], 100) self.assertEqual(usage["output_tokens"], 50) self.assertEqual(usage["total_tokens"], 150) def test_extract_usage_with_none_input_tokens(self): """Test token extraction when input_tokens is None (regression test for bug).""" response = Mock() response.usage_metadata = Mock() response.usage_metadata.prompt_token_count = None # This was causing crashes response.usage_metadata.candidates_token_count = 50 usage = self.provider._extract_usage(response) # Should not include input_tokens when None self.assertNotIn("input_tokens", usage) self.assertEqual(usage["output_tokens"], 50) # Should not calculate total_tokens when input is None self.assertNotIn("total_tokens", usage) def test_extract_usage_with_none_output_tokens(self): """Test token extraction when output_tokens is None (regression test for bug).""" response = Mock() response.usage_metadata = Mock() response.usage_metadata.prompt_token_count = 100 response.usage_metadata.candidates_token_count = None # This was causing crashes usage = self.provider._extract_usage(response) self.assertEqual(usage["input_tokens"], 100) # Should not include output_tokens when None self.assertNotIn("output_tokens", usage) # Should not calculate total_tokens when output is None self.assertNotIn("total_tokens", usage) def test_extract_usage_with_both_none_tokens(self): """Test token extraction when both token counts are None.""" response = Mock() response.usage_metadata = Mock() response.usage_metadata.prompt_token_count = None response.usage_metadata.candidates_token_count = None usage = self.provider._extract_usage(response) # Should return empty dict when all tokens are None self.assertEqual(usage, {}) def test_extract_usage_without_usage_metadata(self): """Test token extraction when response has no usage_metadata.""" response = Mock(spec=[]) usage = self.provider._extract_usage(response) # Should return empty dict self.assertEqual(usage, {}) def test_extract_usage_with_zero_tokens(self): """Test token extraction with zero token counts.""" response = Mock() response.usage_metadata = Mock() response.usage_metadata.prompt_token_count = 0 response.usage_metadata.candidates_token_count = 0 usage = self.provider._extract_usage(response) self.assertEqual(usage["input_tokens"], 0) self.assertEqual(usage["output_tokens"], 0) self.assertEqual(usage["total_tokens"], 0) def test_extract_usage_missing_attributes(self): """Test token extraction when metadata lacks token count attributes.""" response = Mock() response.usage_metadata = Mock(spec=[]) usage = self.provider._extract_usage(response) # Should return empty dict when attributes are missing self.assertEqual(usage, {}) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_image_support_integration.py ================================================ """ Integration tests for native image support feature. Tests the complete image support pipeline: - Conversation memory integration with images - Tool request validation and schema support - Provider image processing capabilities - Cross-tool image context preservation """ import os import tempfile import uuid from unittest.mock import Mock, patch import pytest from tools.chat import ChatTool from tools.debug import DebugIssueTool from tools.shared.exceptions import ToolExecutionError from utils.conversation_memory import ( ConversationTurn, ThreadContext, add_turn, create_thread, get_conversation_image_list, get_thread, ) from utils.model_context import ModelContext @pytest.mark.no_mock_provider class TestImageSupportIntegration: """Integration tests for the complete image support feature.""" def test_conversation_turn_includes_images(self): """Test that ConversationTurn can store and track images.""" turn = ConversationTurn( role="user", content="Please analyze this diagram", timestamp="2025-01-01T00:00:00Z", files=["code.py"], images=["diagram.png", "flowchart.jpg"], tool_name="chat", ) assert turn.images == ["diagram.png", "flowchart.jpg"] assert turn.files == ["code.py"] assert turn.content == "Please analyze this diagram" def test_get_conversation_image_list_newest_first(self): """Test that image list prioritizes newest references.""" # Create thread context with multiple turns context = ThreadContext( thread_id=str(uuid.uuid4()), created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:00:00Z", tool_name="chat", turns=[ ConversationTurn( role="user", content="Turn 1", timestamp="2025-01-01T00:00:00Z", images=["old_diagram.png", "shared.png"], ), ConversationTurn( role="assistant", content="Turn 2", timestamp="2025-01-01T01:00:00Z", images=["middle.png"] ), ConversationTurn( role="user", content="Turn 3", timestamp="2025-01-01T02:00:00Z", images=["shared.png", "new_diagram.png"], # shared.png appears again ), ], initial_context={}, ) image_list = get_conversation_image_list(context) # Should prioritize newest first, with duplicates removed (newest wins) expected = ["shared.png", "new_diagram.png", "middle.png", "old_diagram.png"] assert image_list == expected @patch("utils.conversation_memory.get_storage") def test_add_turn_with_images(self, mock_storage): """Test adding a conversation turn with images.""" mock_client = Mock() mock_storage.return_value = mock_client # Mock the Redis operations to return success mock_client.set.return_value = True thread_id = create_thread("test_tool", {"initial": "context"}) # Set up initial thread context for add_turn to find initial_context = ThreadContext( thread_id=thread_id, created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:00:00Z", tool_name="test_tool", turns=[], # Empty initially initial_context={"initial": "context"}, ) mock_client.get.return_value = initial_context.model_dump_json() success = add_turn( thread_id=thread_id, role="user", content="Analyze these screenshots", files=["app.py"], images=["screenshot1.png", "screenshot2.png"], tool_name="debug", ) assert success # Mock thread context for get_thread call updated_context = ThreadContext( thread_id=thread_id, created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:00:00Z", tool_name="test_tool", turns=[ ConversationTurn( role="user", content="Analyze these screenshots", timestamp="2025-01-01T00:00:00Z", files=["app.py"], images=["screenshot1.png", "screenshot2.png"], tool_name="debug", ) ], initial_context={"initial": "context"}, ) mock_client.get.return_value = updated_context.model_dump_json() # Retrieve and verify the thread context = get_thread(thread_id) assert context is not None assert len(context.turns) == 1 turn = context.turns[0] assert turn.images == ["screenshot1.png", "screenshot2.png"] assert turn.files == ["app.py"] assert turn.content == "Analyze these screenshots" def test_chat_tool_schema_includes_images(self): """Test that ChatTool schema includes images field.""" tool = ChatTool() schema = tool.get_input_schema() assert "images" in schema["properties"] images_field = schema["properties"]["images"] assert images_field["type"] == "array" assert images_field["items"]["type"] == "string" assert "visual context" in images_field["description"].lower() def test_debug_tool_schema_includes_images(self): """Test that DebugIssueTool schema includes images field.""" tool = DebugIssueTool() schema = tool.get_input_schema() assert "images" in schema["properties"] images_field = schema["properties"]["images"] assert images_field["type"] == "array" assert images_field["items"]["type"] == "string" assert "screenshots" in images_field["description"].lower() def test_tool_image_validation_limits(self): """Test that tools validate image size limits using real provider resolution.""" tool = ChatTool() # Create small test images (each 0.5MB, total 1MB) small_images = [] for _ in range(2): with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file: # Write 0.5MB of data temp_file.write(b"\x00" * (512 * 1024)) small_images.append(temp_file.name) try: # Test with an invalid model name that doesn't exist in any provider # Use model_context parameter name (not positional) result = tool._validate_image_limits(small_images, model_context=ModelContext("non-existent-model-12345")) # Should return error because model not available or doesn't support images assert result is not None assert result["status"] == "error" assert "is not available" in result["content"] or "does not support image processing" in result["content"] # Test that empty/None images always pass regardless of model result = tool._validate_image_limits([], model_context=ModelContext("gemini-2.5-pro")) assert result is None result = tool._validate_image_limits(None, model_context=ModelContext("gemini-2.5-pro")) assert result is None finally: # Clean up temp files for img_path in small_images: if os.path.exists(img_path): os.unlink(img_path) def test_image_validation_model_specific_limits(self): """Test that different models have appropriate size limits using real provider resolution.""" tool = ChatTool() # Test with Gemini model which has better image support in test environment # Create 15MB image (under default limits) small_image_path = None large_image_path = None try: # Create 15MB image with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file: temp_file.write(b"\x00" * (15 * 1024 * 1024)) # 15MB small_image_path = temp_file.name # Test with the default model from test environment (gemini-2.5-flash) result = tool._validate_image_limits([small_image_path], ModelContext("gemini-2.5-flash")) assert result is None # Should pass for Gemini models # Create 150MB image (over typical limits) with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file: temp_file.write(b"\x00" * (150 * 1024 * 1024)) # 150MB large_image_path = temp_file.name result = tool._validate_image_limits([large_image_path], ModelContext("gemini-2.5-flash")) # Large images should fail validation assert result is not None assert result["status"] == "error" assert "Image size limit exceeded" in result["content"] finally: # Clean up temp files if small_image_path and os.path.exists(small_image_path): os.unlink(small_image_path) if large_image_path and os.path.exists(large_image_path): os.unlink(large_image_path) @pytest.mark.asyncio async def test_chat_tool_execution_with_images(self): """Test that ChatTool can execute with images parameter using real provider resolution.""" import importlib # Create a temporary image file for testing with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as temp_file: # Write a simple PNG header (minimal valid PNG) png_header = b"\x89PNG\r\n\x1a\n\x00\x00\x00\rIHDR\x00\x00\x00\x01\x00\x00\x00\x01\x08\x06\x00\x00\x00\x1f\x15\xc4\x89\x00\x00\x00\rIDATx\x9cc\x00\x01\x00\x00\x05\x00\x01\r\n-\xdb\x00\x00\x00\x00IEND\xaeB`\x82" temp_file.write(png_header) temp_image_path = temp_file.name # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for real provider resolution os.environ["OPENAI_API_KEY"] = "sk-test-key-images-test-not-real" os.environ["DEFAULT_MODEL"] = "gpt-4o" # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = ChatTool() # Test with real provider resolution with tempfile.TemporaryDirectory() as working_directory: with pytest.raises(ToolExecutionError) as exc_info: await tool.execute( { "prompt": "What do you see in this image?", "images": [temp_image_path], "model": "gpt-4o", "working_directory_absolute_path": working_directory, } ) error_msg = exc_info.value.payload if hasattr(exc_info.value, "payload") else str(exc_info.value) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error (API key or network) assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection", "401", "403"] ) finally: # Clean up temp file os.unlink(temp_image_path) # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None @patch("utils.conversation_memory.get_storage") def test_cross_tool_image_context_preservation(self, mock_storage): """Test that images are preserved across different tools in conversation.""" mock_client = Mock() mock_storage.return_value = mock_client # Mock the Redis operations to return success mock_client.set.return_value = True # Create initial thread with chat tool thread_id = create_thread("chat", {"initial": "context"}) # Set up initial thread context for add_turn to find initial_context = ThreadContext( thread_id=thread_id, created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:00:00Z", tool_name="chat", turns=[], # Empty initially initial_context={"initial": "context"}, ) mock_client.get.return_value = initial_context.model_dump_json() # Add turn with images from chat tool add_turn( thread_id=thread_id, role="user", content="Here's my UI design", images=["design.png", "mockup.jpg"], tool_name="chat", ) add_turn( thread_id=thread_id, role="assistant", content="I can see your design. It looks good!", tool_name="chat" ) # Add turn with different images from debug tool add_turn( thread_id=thread_id, role="user", content="Now I'm getting this error", images=["error_screen.png"], files=["error.log"], tool_name="debug", ) # Mock complete thread context for get_thread call complete_context = ThreadContext( thread_id=thread_id, created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:05:00Z", tool_name="chat", turns=[ ConversationTurn( role="user", content="Here's my UI design", timestamp="2025-01-01T00:01:00Z", images=["design.png", "mockup.jpg"], tool_name="chat", ), ConversationTurn( role="assistant", content="I can see your design. It looks good!", timestamp="2025-01-01T00:02:00Z", tool_name="chat", ), ConversationTurn( role="user", content="Now I'm getting this error", timestamp="2025-01-01T00:03:00Z", images=["error_screen.png"], files=["error.log"], tool_name="debug", ), ], initial_context={"initial": "context"}, ) mock_client.get.return_value = complete_context.model_dump_json() # Retrieve thread and check image preservation context = get_thread(thread_id) assert context is not None # Get conversation image list (should prioritize newest first) image_list = get_conversation_image_list(context) expected = ["error_screen.png", "design.png", "mockup.jpg"] assert image_list == expected # Verify each turn has correct images assert context.turns[0].images == ["design.png", "mockup.jpg"] assert context.turns[1].images is None # Assistant turn without images assert context.turns[2].images == ["error_screen.png"] def test_tool_request_base_class_has_images(self): """Test that base ToolRequest class includes images field.""" from tools.shared.base_models import ToolRequest # Create request with images request = ToolRequest(images=["test.png", "test2.jpg"]) assert request.images == ["test.png", "test2.jpg"] # Test default value request_no_images = ToolRequest() assert request_no_images.images is None def test_data_url_image_format_support(self): """Test that tools can handle data URL format images.""" tool = ChatTool() # Test with data URL (base64 encoded 1x1 transparent PNG) data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNk+M9QDwADhgGAWjR9awAAAABJRU5ErkJggg==" images = [data_url] # Test with a dummy model that doesn't exist in any provider result = tool._validate_image_limits(images, ModelContext("test-dummy-model-name")) # Should return error because model not available or doesn't support images assert result is not None assert result["status"] == "error" assert "is not available" in result["content"] or "does not support image processing" in result["content"] # Test with another non-existent model to check error handling result = tool._validate_image_limits(images, ModelContext("another-dummy-model")) # Should return error because model not available assert result is not None assert result["status"] == "error" def test_empty_images_handling(self): """Test that tools handle empty images lists gracefully.""" tool = ChatTool() # Empty list should not fail validation (no need for provider setup) result = tool._validate_image_limits([], ModelContext("gemini-2.5-pro")) assert result is None # None should not fail validation (no need for provider setup) result = tool._validate_image_limits(None, ModelContext("gemini-2.5-pro")) assert result is None @patch("utils.conversation_memory.get_storage") def test_conversation_memory_thread_chaining_with_images(self, mock_storage): """Test that images work correctly with conversation thread chaining.""" mock_client = Mock() mock_storage.return_value = mock_client # Mock the Redis operations to return success mock_client.set.return_value = True # Create parent thread with images parent_thread_id = create_thread("chat", {"parent": "context"}) # Set up initial parent thread context for add_turn to find parent_context = ThreadContext( thread_id=parent_thread_id, created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:00:00Z", tool_name="chat", turns=[], # Empty initially initial_context={"parent": "context"}, ) mock_client.get.return_value = parent_context.model_dump_json() add_turn( thread_id=parent_thread_id, role="user", content="Parent thread with images", images=["parent1.png", "shared.png"], tool_name="chat", ) # Create child thread linked to parent using a simple tool child_thread_id = create_thread("chat", {"prompt": "child context"}, parent_thread_id=parent_thread_id) add_turn( thread_id=child_thread_id, role="user", content="Child thread with more images", images=["child1.png", "shared.png"], # shared.png appears again (should prioritize newer) tool_name="chat", ) # Mock child thread context for get_thread call child_context = ThreadContext( thread_id=child_thread_id, created_at="2025-01-01T00:00:00Z", last_updated_at="2025-01-01T00:02:00Z", tool_name="debug", turns=[ ConversationTurn( role="user", content="Child thread with more images", timestamp="2025-01-01T00:02:00Z", images=["child1.png", "shared.png"], tool_name="debug", ) ], initial_context={"child": "context"}, parent_thread_id=parent_thread_id, ) mock_client.get.return_value = child_context.model_dump_json() # Get child thread and verify image collection works across chain child_context = get_thread(child_thread_id) assert child_context is not None assert child_context.parent_thread_id == parent_thread_id # Test image collection for child thread only child_images = get_conversation_image_list(child_context) assert child_images == ["child1.png", "shared.png"] ================================================ FILE: tests/test_image_validation.py ================================================ """Tests for image validation utility helpers.""" import base64 import os import tempfile from unittest.mock import Mock, patch import pytest from utils.image_utils import DEFAULT_MAX_IMAGE_SIZE_MB, validate_image class TestImageValidation: """Test suite for image validation functionality.""" def test_validate_data_url_valid(self) -> None: """Test validation of valid data URL.""" # Create a small test image (1x1 PNG) test_image_data = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" ) data_url = f"data:image/png;base64,{base64.b64encode(test_image_data).decode()}" image_bytes, mime_type = validate_image(data_url) assert image_bytes == test_image_data assert mime_type == "image/png" @pytest.mark.parametrize( "invalid_url,expected_error", [ ("data:image/png", "Invalid data URL format"), # Missing base64 part ("data:image/png;base64", "Invalid data URL format"), # Missing data ("data:text/plain;base64,dGVzdA==", "Unsupported image type"), # Not an image ], ) def test_validate_data_url_invalid_format(self, invalid_url: str, expected_error: str) -> None: """Test validation of malformed data URL.""" with pytest.raises(ValueError) as excinfo: validate_image(invalid_url) assert expected_error in str(excinfo.value) def test_non_data_url_treated_as_file_path(self) -> None: """Test that non-data URLs are treated as file paths.""" # Test case that's not a data URL at all with pytest.raises(ValueError) as excinfo: validate_image("image/png;base64,abc123") assert "Image file not found" in str(excinfo.value) # Treated as file path def test_validate_data_url_unsupported_type(self) -> None: """Test validation of unsupported image type in data URL.""" data_url = "data:image/bmp;base64,Qk0=" # BMP format with pytest.raises(ValueError) as excinfo: validate_image(data_url) assert "Unsupported image type: image/bmp" in str(excinfo.value) def test_validate_data_url_invalid_base64(self) -> None: """Test validation of data URL with invalid base64.""" data_url = "data:image/png;base64,@@@invalid@@@" with pytest.raises(ValueError) as excinfo: validate_image(data_url) assert "Invalid base64 data" in str(excinfo.value) def test_validate_large_data_url(self) -> None: """Test validation of large data URL to ensure size limits work.""" # Create a large image (21MB) large_data = b"x" * (21 * 1024 * 1024) # 21MB # Encode as base64 and create data URL import base64 encoded_data = base64.b64encode(large_data).decode() data_url = f"data:image/png;base64,{encoded_data}" # Should fail with default 20MB limit with pytest.raises(ValueError) as excinfo: validate_image(data_url) assert f"Image too large: 21.0MB (max: {DEFAULT_MAX_IMAGE_SIZE_MB:.1f}MB)" in str(excinfo.value) # Should succeed with higher limit image_bytes, mime_type = validate_image(data_url, max_size_mb=25.0) assert len(image_bytes) == len(large_data) assert mime_type == "image/png" def test_validate_file_path_valid(self) -> None: """Test validation of valid image file.""" # Create a temporary image file with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_file: # Write a small test PNG test_image_data = base64.b64decode( "iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" ) tmp_file.write(test_image_data) tmp_file_path = tmp_file.name try: image_bytes, mime_type = validate_image(tmp_file_path) assert image_bytes == test_image_data assert mime_type == "image/png" finally: os.unlink(tmp_file_path) def test_validate_file_path_not_found(self) -> None: """Test validation of non-existent file.""" with pytest.raises(ValueError) as excinfo: validate_image("/path/to/nonexistent/image.png") assert "Image file not found" in str(excinfo.value) def test_validate_file_path_unsupported_extension(self) -> None: """Test validation of file with unsupported extension.""" with tempfile.NamedTemporaryFile(suffix=".bmp", delete=False) as tmp_file: tmp_file.write(b"dummy data") tmp_file_path = tmp_file.name try: with pytest.raises(ValueError) as excinfo: validate_image(tmp_file_path) assert "Unsupported image format: .bmp" in str(excinfo.value) finally: os.unlink(tmp_file_path) def test_validate_file_path_read_error(self) -> None: """Test validation when file cannot be read.""" with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_file: tmp_file_path = tmp_file.name # Remove the file but keep the path os.unlink(tmp_file_path) with pytest.raises(ValueError) as excinfo: validate_image(tmp_file_path) assert "Image file not found" in str(excinfo.value) def test_validate_image_size_limit(self) -> None: """Test validation of image size limits.""" # Create a large "image" (just random data) large_data = b"x" * (21 * 1024 * 1024) # 21MB with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_file: tmp_file.write(large_data) tmp_file_path = tmp_file.name try: with pytest.raises(ValueError) as excinfo: validate_image(tmp_file_path, max_size_mb=20.0) assert "Image too large: 21.0MB (max: 20.0MB)" in str(excinfo.value) finally: os.unlink(tmp_file_path) def test_validate_image_custom_size_limit(self) -> None: """Test validation with custom size limit.""" # Create a 2MB "image" data = b"x" * (2 * 1024 * 1024) with tempfile.NamedTemporaryFile(suffix=".png", delete=False) as tmp_file: tmp_file.write(data) tmp_file_path = tmp_file.name try: # Should fail with 1MB limit with pytest.raises(ValueError) as excinfo: validate_image(tmp_file_path, max_size_mb=1.0) assert "Image too large: 2.0MB (max: 1.0MB)" in str(excinfo.value) # Should succeed with 3MB limit image_bytes, mime_type = validate_image(tmp_file_path, max_size_mb=3.0) assert len(image_bytes) == len(data) assert mime_type == "image/png" finally: os.unlink(tmp_file_path) def test_validate_image_default_size_limit(self) -> None: """Test validation with default size limit (None).""" # Create a small image that's under the default limit data = b"x" * (1024 * 1024) # 1MB with tempfile.NamedTemporaryFile(suffix=".jpg", delete=False) as tmp_file: tmp_file.write(data) tmp_file_path = tmp_file.name try: # Should succeed with default limit (20MB) image_bytes, mime_type = validate_image(tmp_file_path) assert len(image_bytes) == len(data) assert mime_type == "image/jpeg" # Should also succeed when explicitly passing None image_bytes, mime_type = validate_image(tmp_file_path, max_size_mb=None) assert len(image_bytes) == len(data) assert mime_type == "image/jpeg" finally: os.unlink(tmp_file_path) def test_validate_all_supported_formats(self) -> None: """Test validation of all supported image formats.""" supported_formats = { ".png": "image/png", ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".gif": "image/gif", ".webp": "image/webp", } for ext, expected_mime in supported_formats.items(): with tempfile.NamedTemporaryFile(suffix=ext, delete=False) as tmp_file: tmp_file.write(b"dummy image data") tmp_file_path = tmp_file.name try: image_bytes, mime_type = validate_image(tmp_file_path) assert mime_type == expected_mime assert image_bytes == b"dummy image data" finally: os.unlink(tmp_file_path) class TestProviderIntegration: """Test image validation integration with different providers.""" @patch("providers.gemini.logger") def test_gemini_provider_uses_validation(self, mock_logger: Mock) -> None: """Test that Gemini provider uses the base validation.""" from providers.gemini import GeminiModelProvider # Create a provider instance provider = GeminiModelProvider(api_key="test-key") # Test with non-existent file result = provider._process_image("/nonexistent/image.png") assert result is None mock_logger.warning.assert_called_with("Image file not found: /nonexistent/image.png") @patch("providers.openai_compatible.logging") def test_openai_compatible_provider_uses_validation(self, mock_logging: Mock) -> None: """Test that OpenAI-compatible providers use the base validation.""" from providers.xai import XAIModelProvider # Create a provider instance (XAI inherits from OpenAICompatibleProvider) provider = XAIModelProvider(api_key="test-key") # Test with non-existent file result = provider._process_image("/nonexistent/image.png") assert result is None mock_logging.warning.assert_called_with("Image file not found: /nonexistent/image.png") def test_data_url_preservation(self) -> None: """Test that data URLs are properly preserved through validation.""" from providers.xai import XAIModelProvider provider = XAIModelProvider(api_key="test-key") # Valid data URL data_url = "data:image/png;base64,iVBORw0KGgoAAAANSUhEUgAAAAEAAAABCAYAAAAfFcSJAAAADUlEQVR42mNkYPhfDwAChwGA60e6kgAAAABJRU5ErkJggg==" result = provider._process_image(data_url) assert result is not None assert result["type"] == "image_url" assert result["image_url"]["url"] == data_url ================================================ FILE: tests/test_integration_utf8.py ================================================ """ Full integration test script to validate UTF-8 implementation and French localization. This script runs all unit tests and checks full integration. """ import json import os import subprocess import sys import tempfile from pathlib import Path def run_utf8_integration_tests(): """Run UTF-8 integration tests.""" print("🚀 Starting UTF-8 integration tests") print("=" * 60) # Test environment setup os.environ["LOCALE"] = "fr-FR" os.environ["GEMINI_API_KEY"] = "dummy-key-for-tests" os.environ["OPENAI_API_KEY"] = "dummy-key-for-tests" # Test 1: Validate UTF-8 characters in json.dumps print("\n1️⃣ UTF-8 encoding test with json.dumps") test_utf8_json_encoding() # Test 2: Validate language instruction generation print("\n2️⃣ Language instruction generation test") test_language_instruction_generation() # Test 3: Validate UTF-8 file handling print("\n3️⃣ UTF-8 file handling test") test_file_utf8_handling() # Test 4: Validate MCP tools integration print("\n4️⃣ MCP tools integration test") test_mcp_tools_integration() # Test 5: Run unit tests print("\n5️⃣ Running unit tests") run_unit_tests() print("\n✅ All UTF-8 integration tests completed!") print("🇫🇷 French localization works correctly!") def test_utf8_json_encoding(): """Test UTF-8 encoding with json.dumps(ensure_ascii=False).""" print(" Testing UTF-8 JSON encoding...") # Test data with French characters and emojis test_data = { "analyse": { "statut": "terminée", "résultat": "Aucun problème critique détecté", "recommandations": [ "Améliorer la documentation", "Optimiser les performances", "Ajouter des tests unitaires", ], "métadonnées": { "créé_par": "Développeur Principal", "date_création": "2024-01-01", "dernière_modification": "2024-01-15", }, "émojis_status": { "critique": "🔴", "élevé": "🟠", "moyen": "🟡", "faible": "🟢", "succès": "✅", "erreur": "❌", }, }, "outils": [ {"nom": "analyse", "description": "Analyse architecturale avancée"}, {"nom": "révision", "description": "Révision de code automatisée"}, {"nom": "génération", "description": "Génération de documentation"}, ], } # Test with ensure_ascii=False json_correct = json.dumps(test_data, ensure_ascii=False, indent=2) # Checks utf8_terms = [ "terminée", "résultat", "détecté", "Améliorer", "créé_par", "Développeur", "création", "métadonnées", "dernière", "émojis_status", "élevé", "révision", "génération", ] emojis = ["🔴", "🟠", "🟡", "🟢", "✅", "❌"] for term in utf8_terms: assert term in json_correct, f"Missing UTF-8 term: {term}" for emoji in emojis: assert emoji in json_correct, f"Missing emoji: {emoji}" # Check for escaped characters assert "\\u" not in json_correct, "Escaped Unicode characters detected!" # Test parsing parsed = json.loads(json_correct) assert parsed["analyse"]["statut"] == "terminée" assert parsed["analyse"]["émojis_status"]["critique"] == "🔴" print(" ✅ UTF-8 JSON encoding: SUCCESS") def test_language_instruction_generation(): """Test language instruction generation.""" print(" Testing language instruction generation...") # Simulation of get_language_instruction def get_language_instruction(): locale = os.getenv("LOCALE", "").strip() if not locale: return "" return f"Always respond in {locale}.\n\n" # Test with different locales test_locales = [ ("fr-FR", "French"), ("en-US", "English"), ("es-ES", "Spanish"), ("de-DE", "German"), ("", "none"), ] for locale, description in test_locales: os.environ["LOCALE"] = locale instruction = get_language_instruction() if locale: assert locale in instruction, f"Missing {locale} in instruction" assert instruction.endswith("\n\n"), "Incorrect instruction format" print(f" 📍 {description}: {instruction.strip()}") else: assert instruction == "", "Empty instruction expected for empty locale" print(f" 📍 {description}: (empty)") # Restore French locale os.environ["LOCALE"] = "fr-FR" print(" ✅ Language instruction generation: SUCCESS") def test_file_utf8_handling(): """Test handling of files with UTF-8 content.""" print(" Testing UTF-8 file handling...") # File content with French characters french_content = '''#!/usr/bin/env python3 """ Module de gestion des préférences utilisateur. Développé par: Équipe Technique Date de création: 15 décembre 2024 """ import json from typing import Dict, Optional class GestionnairePreferences: """Gestionnaire des préférences utilisateur avec support UTF-8.""" def __init__(self): self.données = {} self.historique = [] def définir_préférence(self, clé: str, valeur) -> bool: """ Définit une préférence utilisateur. Args: clé: Identifiant de la préférence valeur: Valeur à enregistrer Returns: True si la préférence a été définie avec succès """ try: self.données[clé] = valeur self.historique.append({ "action": "définition", "clé": clé, "horodatage": "2024-01-01T12:00:00Z" }) return True except Exception as e: print(f"Error setting preference: {e}") return False def obtenir_préférence(self, clé: str) -> Optional: """Récupère une préférence par sa clé.""" return self.données.get(clé) def exporter_données(self) -> str: """Exporte les données en JSON UTF-8.""" return json.dumps(self.données, ensure_ascii=False, indent=2) # Configuration par défaut avec caractères UTF-8 CONFIG_DÉFAUT = { "langue": "français", "région": "France", "thème": "sombre", "notifications": "activées" } def créer_gestionnaire() -> GestionnairePreferences: """Crée une instance du gestionnaire.""" gestionnaire = GestionnairePreferences() # Application de la configuration par défaut for clé, valeur in CONFIG_DÉFAUT.items(): gestionnaire.définir_préférence(clé, valeur) return gestionnaire if __name__ == "__main__": # Test d'utilisation gestionnaire = créer_gestionnaire() print("Gestionnaire créé avec succès! 🎉") print(f"Données: {gestionnaire.exporter_données()}") ''' # Test writing and reading UTF-8 with tempfile.NamedTemporaryFile(mode="w", encoding="utf-8", suffix=".py", delete=False) as f: f.write(french_content) temp_file = f.name try: # Test reading with open(temp_file, encoding="utf-8") as f: read_content = f.read() # Checks assert read_content == french_content, "Altered UTF-8 content" # Check specific terms utf8_terms = [ "préférences", "Développé", "Équipe", "création", "données", "définir_préférence", "horodatage", "Récupère", "français", "activées", "créer_gestionnaire", "succès", ] for term in utf8_terms: assert term in read_content, f"Missing UTF-8 term: {term}" print(" ✅ UTF-8 file handling: SUCCESS") finally: # Cleanup os.unlink(temp_file) def test_mcp_tools_integration(): """Test MCP tools integration with UTF-8.""" print(" Testing MCP tools integration...") # Simulation of MCP tool response def simulate_mcp_tool_response(): """Simulate MCP tool response with UTF-8 content.""" response_data = { "status": "success", "content_type": "markdown", "content": """# Analyse Terminée avec Succès ✅ ## Résumé de l'Analyse L'analyse architecturale du projet a été **terminée** avec succès. Voici les principaux résultats : ### 🎯 Objectifs Atteints - ✅ Révision complète du code - ✅ Identification des problèmes de performance - ✅ Recommandations d'amélioration générées ### 📊 Métriques Analysées | Métrique | Valeur | Statut | |----------|--------|--------| | Complexité cyclomatique | 12 | 🟡 Acceptable | | Couverture de tests | 85% | 🟢 Bon | | Dépendances externes | 23 | 🟠 À réviser | ### 🔍 Problèmes Identifiés #### 🔴 Critique Aucun problème critique détecté. #### 🟠 Élevé 1. **Performance des requêtes** : Optimisation nécessaire 2. **Gestion mémoire** : Fuites potentielles détectées #### 🟡 Moyen 1. **Documentation** : Certaines fonctions manquent de commentaires 2. **Tests unitaires** : Couverture à améliorer ### � Détails de l'Analyse Pour plus de détails sur chaque problème identifié, consultez les recommandations ci-dessous. ### �🚀 Recommandations Prioritaires 1. **Optimisation DB** : Implémenter un cache Redis 2. **Refactoring** : Séparer les responsabilités 3. **Documentation** : Ajouter les docstrings manquantes 4. **Tests** : Augmenter la couverture à 90%+ ### 📈 Prochaines Étapes - [ ] Implémenter le système de cache - [ ] Refactorer les modules identifiés - [ ] Compléter la documentation - [ ] Exécuter les tests de régression --- *Analyse générée automatiquement par MCP PAL* 🤖 """, "metadata": { "tool_name": "analyze", "execution_time": 2.5, "locale": "fr-FR", "timestamp": "2024-01-01T12:00:00Z", "analysis_summary": { "files_analyzed": 15, "issues_found": 4, "recommendations": 4, "overall_score": "B+ (Good level)", }, }, "continuation_offer": { "continuation_id": "analysis-123", "note": "In-depth analysis available with more details", }, } # Serialization with ensure_ascii=False json_response = json.dumps(response_data, ensure_ascii=False, indent=2) # UTF-8 checks utf8_checks = [ "Terminée", "Succès", "Résumé", "terminée", "Atteints", "Révision", "problèmes", "générées", "Métriques", "Identifiés", "détecté", "Élevé", "nécessaire", "détectées", "améliorer", "Prioritaires", "responsabilités", "Étapes", "régression", "générée", "détails", ] for term in utf8_checks: assert term in json_response, f"Missing UTF-8 term: {term}" # Emoji check emojis = ["✅", "🎯", "📊", "🟡", "🟢", "🟠", "🔍", "🔴", "🚀", "📈", "🤖"] for emoji in emojis: assert emoji in json_response, f"Missing emoji: {emoji}" # Test parsing parsed = json.loads(json_response) assert parsed["status"] == "success" assert "Terminée" in parsed["content"] assert parsed["metadata"]["locale"] == "fr-FR" return json_response # Test simulation response = simulate_mcp_tool_response() assert len(response) > 1000, "MCP response too short" print(" ✅ MCP tools integration: SUCCESS") def run_unit_tests(): """Run unit tests.""" print(" Running unit tests...") # List of test files to run test_files = ["test_utf8_localization.py", "test_provider_utf8.py", "test_workflow_utf8.py"] current_dir = Path(__file__).parent test_results = [] for test_file in test_files: test_path = current_dir / test_file if test_path.exists(): print(f" 📝 Running {test_file}...") try: # Test execution result = subprocess.run( [sys.executable, "-m", "unittest", test_file.replace(".py", ""), "-v"], cwd=current_dir, capture_output=True, text=True, timeout=60, ) if result.returncode == 0: print(f" ✅ {test_file}: SUCCESS") test_results.append((test_file, "SUCCESS")) else: print(f" ❌ {test_file}: FAILURE") print(f" Error: {result.stderr[:200]}...") test_results.append((test_file, "FAILURE")) except subprocess.TimeoutExpired: print(f" ⏰ {test_file}: TIMEOUT") test_results.append((test_file, "TIMEOUT")) except Exception as e: print(f" 💥 {test_file}: ERROR - {e}") test_results.append((test_file, "ERROR")) else: print(f" ⚠️ {test_file}: NOT FOUND") test_results.append((test_file, "NOT FOUND")) # Test summary print("\n 📋 Unit test summary:") for test_file, status in test_results: status_emoji = {"SUCCESS": "✅", "FAILURE": "❌", "TIMEOUT": "⏰", "ERROR": "💥", "NOT FOUND": "⚠️"}.get( status, "❓" ) print(f" {status_emoji} {test_file}: {status}") def main(): """Main function.""" print("🇫🇷 UTF-8 Integration Test - PAL MCP Server") print("=" * 60) try: run_utf8_integration_tests() print("\n🎉 SUCCESS: All UTF-8 integration tests passed!") print("🚀 PAL MCP Server fully supports French localization!") return 0 except AssertionError as e: print(f"\n❌ FAILURE: Assertion test failed: {e}") return 1 except Exception as e: print(f"\n💥 ERROR: Unexpected exception: {e}") return 1 if __name__ == "__main__": sys.exit(main()) ================================================ FILE: tests/test_intelligent_fallback.py ================================================ """ Test suite for intelligent auto mode fallback logic Tests the new dynamic model selection based on available API keys """ import os from unittest.mock import Mock, patch import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType class TestIntelligentFallback: """Test intelligent model fallback logic""" def setup_method(self): """Setup for each test - clear registry and reset providers""" # Store original providers for restoration registry = ModelProviderRegistry() self._original_providers = registry._providers.copy() self._original_initialized = registry._initialized_providers.copy() # Clear registry completely ModelProviderRegistry._instance = None def teardown_method(self): """Cleanup after each test - restore original providers""" # Restore original registry state registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() registry._providers.update(self._original_providers) registry._initialized_providers.update(self._original_initialized) @patch.dict(os.environ, {"OPENAI_API_KEY": "sk-test-key", "GEMINI_API_KEY": ""}, clear=False) def test_prefers_openai_o3_mini_when_available(self): """Test that gpt-5.2 is preferred when OpenAI API key is available (based on new preference order)""" # Register only OpenAI provider for this test from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) fallback_model = ModelProviderRegistry.get_preferred_fallback_model() assert fallback_model == "gpt-5.2" # Based on new preference order: gpt-5.2 before o4-mini @patch.dict(os.environ, {"OPENAI_API_KEY": "", "GEMINI_API_KEY": "test-gemini-key"}, clear=False) def test_prefers_gemini_flash_when_openai_unavailable(self): """Test that gemini-2.5-flash is used when only Gemini API key is available""" # Register only Gemini provider for this test from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) fallback_model = ModelProviderRegistry.get_preferred_fallback_model() assert fallback_model == "gemini-2.5-flash" @patch.dict(os.environ, {"OPENAI_API_KEY": "sk-test-key", "GEMINI_API_KEY": "test-gemini-key"}, clear=False) def test_prefers_openai_when_both_available(self): """Test that OpenAI is preferred when both API keys are available""" # Register both OpenAI and Gemini providers from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) fallback_model = ModelProviderRegistry.get_preferred_fallback_model() assert fallback_model == "gemini-2.5-flash" # Gemini has priority now (based on new PROVIDER_PRIORITY_ORDER) @patch.dict(os.environ, {"OPENAI_API_KEY": "", "GEMINI_API_KEY": ""}, clear=False) def test_fallback_when_no_keys_available(self): """Test fallback behavior when no API keys are available""" # Register providers but with no API keys available from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) fallback_model = ModelProviderRegistry.get_preferred_fallback_model() assert fallback_model == "gemini-2.5-flash" # Default fallback def test_available_providers_with_keys(self): """Test the get_available_providers_with_keys method""" from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider with patch.dict(os.environ, {"OPENAI_API_KEY": "sk-test-key", "GEMINI_API_KEY": ""}, clear=False): # Clear and register providers ModelProviderRegistry._instance = None ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) available = ModelProviderRegistry.get_available_providers_with_keys() assert ProviderType.OPENAI in available assert ProviderType.GOOGLE not in available with patch.dict(os.environ, {"OPENAI_API_KEY": "", "GEMINI_API_KEY": "test-key"}, clear=False): # Clear and register providers ModelProviderRegistry._instance = None ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) available = ModelProviderRegistry.get_available_providers_with_keys() assert ProviderType.GOOGLE in available assert ProviderType.OPENAI not in available def test_auto_mode_conversation_memory_integration(self): """Test that conversation memory uses intelligent fallback in auto mode""" from utils.conversation_memory import ThreadContext, build_conversation_history # Mock auto mode - patch the config module where these values are defined with ( patch("config.IS_AUTO_MODE", True), patch("config.DEFAULT_MODEL", "auto"), patch.dict(os.environ, {"OPENAI_API_KEY": "sk-test-key", "GEMINI_API_KEY": ""}, clear=False), ): # Register only OpenAI provider for this test from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Create a context with at least one turn so it doesn't exit early from utils.conversation_memory import ConversationTurn context = ThreadContext( thread_id="test-123", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="chat", turns=[ConversationTurn(role="user", content="Test message", timestamp="2023-01-01T00:00:30Z")], initial_context={}, ) # This should use o4-mini for token calculations since OpenAI is available with patch("utils.model_context.ModelContext") as mock_context_class: mock_context_instance = Mock() mock_context_class.return_value = mock_context_instance mock_context_instance.calculate_token_allocation.return_value = Mock( file_tokens=10000, history_tokens=5000 ) # Mock estimate_tokens to return integers for proper summing mock_context_instance.estimate_tokens.return_value = 100 history, tokens = build_conversation_history(context, model_context=None) # Verify that ModelContext was called with gpt-5.2 (the intelligent fallback based on new preference order) mock_context_class.assert_called_once_with("gpt-5.2") def test_auto_mode_with_gemini_only(self): """Test auto mode behavior when only Gemini API key is available""" from utils.conversation_memory import ThreadContext, build_conversation_history with ( patch("config.IS_AUTO_MODE", True), patch("config.DEFAULT_MODEL", "auto"), patch.dict(os.environ, {"OPENAI_API_KEY": "", "GEMINI_API_KEY": "test-key"}, clear=False), ): # Register only Gemini provider for this test from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) from utils.conversation_memory import ConversationTurn context = ThreadContext( thread_id="test-456", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="analyze", turns=[ConversationTurn(role="assistant", content="Test response", timestamp="2023-01-01T00:00:30Z")], initial_context={}, ) with patch("utils.model_context.ModelContext") as mock_context_class: mock_context_instance = Mock() mock_context_class.return_value = mock_context_instance mock_context_instance.calculate_token_allocation.return_value = Mock( file_tokens=10000, history_tokens=5000 ) # Mock estimate_tokens to return integers for proper summing mock_context_instance.estimate_tokens.return_value = 100 history, tokens = build_conversation_history(context, model_context=None) # Should use gemini-2.5-flash when only Gemini is available mock_context_class.assert_called_once_with("gemini-2.5-flash") def test_non_auto_mode_unchanged(self): """Test that non-auto mode behavior is unchanged""" from utils.conversation_memory import ThreadContext, build_conversation_history with patch("config.IS_AUTO_MODE", False), patch("config.DEFAULT_MODEL", "gemini-2.5-pro"): from utils.conversation_memory import ConversationTurn context = ThreadContext( thread_id="test-789", created_at="2023-01-01T00:00:00Z", last_updated_at="2023-01-01T00:00:00Z", tool_name="thinkdeep", turns=[ ConversationTurn(role="user", content="Test in non-auto mode", timestamp="2023-01-01T00:00:30Z") ], initial_context={}, ) with patch("utils.model_context.ModelContext") as mock_context_class: mock_context_instance = Mock() mock_context_class.return_value = mock_context_instance mock_context_instance.calculate_token_allocation.return_value = Mock( file_tokens=10000, history_tokens=5000 ) # Mock estimate_tokens to return integers for proper summing mock_context_instance.estimate_tokens.return_value = 100 history, tokens = build_conversation_history(context, model_context=None) # Should use the configured DEFAULT_MODEL, not the intelligent fallback mock_context_class.assert_called_once_with("gemini-2.5-pro") if __name__ == "__main__": pytest.main([__file__]) ================================================ FILE: tests/test_issue_245_simple.py ================================================ """ Simple test to verify GitHub issue #245 is fixed. Issue: Custom OpenAI models (gpt-5, o3) use temperature despite the config having supports_temperature: false """ from unittest.mock import Mock, patch from providers.openai import OpenAIModelProvider def test_issue_245_custom_openai_temperature_ignored(): """Test that reproduces and validates the fix for issue #245.""" with patch("utils.model_restrictions.get_restriction_service") as mock_restriction: with patch("providers.openai_compatible.OpenAI") as mock_openai: with patch("providers.registries.openrouter.OpenRouterModelRegistry") as mock_registry_class: # Mock restriction service mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction.return_value = mock_service # Mock OpenAI client mock_client = Mock() mock_openai.return_value = mock_client mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "gpt-5-2025-08-07" mock_response.id = "test" mock_response.created = 123 mock_response.usage = Mock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response # Mock registry with user's custom config (the issue scenario) mock_registry = Mock() mock_registry_class.return_value = mock_registry from providers.shared import ModelCapabilities, ProviderType, TemperatureConstraint # This is what the user configured in their custom_models.json custom_config = ModelCapabilities( provider=ProviderType.OPENAI, model_name="gpt-5-2025-08-07", friendly_name="Custom GPT-5", context_window=400000, max_output_tokens=128000, supports_extended_thinking=True, supports_json_mode=True, supports_system_prompts=True, supports_streaming=True, supports_function_calling=True, supports_temperature=False, # User set this to false! temperature_constraint=TemperatureConstraint.create("fixed"), supports_images=True, max_image_size_mb=20.0, description="Custom OpenAI GPT-5", ) mock_registry.get_model_config.return_value = custom_config # Create provider and test provider = OpenAIModelProvider(api_key="test-key") provider.validate_model_name = lambda name: True # This is what was causing the 400 error before the fix provider.generate_content( prompt="Test", model_name="gpt-5-2025-08-07", temperature=0.2 # This should be ignored! ) # Verify the fix: NO temperature should be sent to the API call_kwargs = mock_client.chat.completions.create.call_args[1] assert "temperature" not in call_kwargs, "Fix failed: temperature still being sent!" ================================================ FILE: tests/test_large_prompt_handling.py ================================================ """ Tests for large prompt handling functionality. This test module verifies that the MCP server correctly handles prompts that exceed the 50,000 character limit by requesting Claude to save them to a file and resend. """ import json import os import shutil import tempfile from unittest.mock import MagicMock, patch import pytest from config import MCP_PROMPT_SIZE_LIMIT from tools.chat import ChatTool from tools.codereview import CodeReviewTool from tools.shared.exceptions import ToolExecutionError # from tools.debug import DebugIssueTool # Commented out - debug tool refactored class TestLargePromptHandling: """Test suite for large prompt handling across all tools.""" def teardown_method(self): """Clean up after each test to prevent state pollution.""" # Clear provider registry singleton from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None @pytest.fixture def large_prompt(self): """Create a prompt larger than MCP_PROMPT_SIZE_LIMIT characters.""" return "x" * (MCP_PROMPT_SIZE_LIMIT + 1000) @pytest.fixture def normal_prompt(self): """Create a normal-sized prompt.""" return "This is a normal prompt that should work fine." @pytest.fixture def temp_prompt_file(self, large_prompt): """Create a temporary prompt.txt file with large content.""" # Create temp file with exact name "prompt.txt" temp_dir = tempfile.mkdtemp() file_path = os.path.join(temp_dir, "prompt.txt") with open(file_path, "w") as f: f.write(large_prompt) return file_path @pytest.mark.asyncio async def test_chat_large_prompt_detection(self, large_prompt): """Test that chat tool detects large prompts.""" tool = ChatTool() temp_dir = tempfile.mkdtemp() temp_dir = tempfile.mkdtemp() try: with pytest.raises(ToolExecutionError) as exc_info: await tool.execute({"prompt": large_prompt, "working_directory_absolute_path": temp_dir}) finally: shutil.rmtree(temp_dir, ignore_errors=True) output = json.loads(exc_info.value.payload) assert output["status"] == "resend_prompt" assert f"{MCP_PROMPT_SIZE_LIMIT:,} characters" in output["content"] # The prompt size should match the user input since we check at MCP transport boundary before adding internal content assert output["metadata"]["prompt_size"] == len(large_prompt) assert output["metadata"]["limit"] == MCP_PROMPT_SIZE_LIMIT @pytest.mark.asyncio async def test_chat_normal_prompt_works(self, normal_prompt): """Test that chat tool works normally with regular prompts.""" tool = ChatTool() temp_dir = tempfile.mkdtemp() # This test runs in the test environment which uses dummy keys # The chat tool will return an error for dummy keys, which is expected try: try: result = await tool.execute( {"prompt": normal_prompt, "model": "gemini-2.5-flash", "working_directory_absolute_path": temp_dir} ) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: assert len(result) == 1 output = json.loads(result[0].text) finally: shutil.rmtree(temp_dir, ignore_errors=True) # Whether provider succeeds or fails, we should not hit the resend_prompt branch assert output["status"] != "resend_prompt" @pytest.mark.asyncio async def test_chat_prompt_file_handling(self): """Test that chat tool correctly handles prompt.txt files with reasonable size.""" tool = ChatTool() # Use a smaller prompt that won't exceed limit when combined with system prompt reasonable_prompt = "This is a reasonable sized prompt for testing prompt.txt file handling." # Create a temp file with reasonable content temp_dir = tempfile.mkdtemp() temp_prompt_file = os.path.join(temp_dir, "prompt.txt") with open(temp_prompt_file, "w") as f: f.write(reasonable_prompt) try: try: result = await tool.execute( { "prompt": "", "absolute_file_paths": [temp_prompt_file], "model": "gemini-2.5-flash", "working_directory_absolute_path": temp_dir, } ) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: assert len(result) == 1 output = json.loads(result[0].text) # The test may fail with dummy API keys, which is expected behavior. # We're mainly testing that the tool processes prompt files correctly without size errors. assert output["status"] != "resend_prompt" finally: # Cleanup shutil.rmtree(temp_dir) @pytest.mark.asyncio async def test_codereview_large_focus(self, large_prompt): """Test that codereview tool detects large focus_on field using real integration testing.""" import importlib import os tool = CodeReviewTool() # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for real provider resolution os.environ["OPENAI_API_KEY"] = "sk-test-key-large-focus-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None # Test with real provider resolution try: args = { "step": "initial review setup", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial testing", "relevant_files": ["/some/file.py"], "files_checked": ["/some/file.py"], "focus_on": large_prompt, "prompt": "Test code review for validation purposes", "model": "o3-mini", } try: result = await tool.execute(args) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: assert len(result) == 1 output = json.loads(result[0].text) # The large focus_on may trigger the resend_prompt guard before provider access. # When the guard does not trigger, auto-mode falls back to provider selection and # returns an error about the unavailable model. Both behaviors are acceptable for this test. if output.get("status") == "resend_prompt": assert output["metadata"]["prompt_size"] == len(large_prompt) else: assert output.get("status") == "error" assert "Model" in output.get("content", "") except Exception as e: # If we get an unexpected exception, ensure it's not a mock artifact error_msg = str(e) assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error (API, authentication, etc.) assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None # NOTE: Precommit test has been removed because the precommit tool has been # refactored to use a workflow-based pattern instead of accepting simple prompt/path fields. # The new precommit tool requires workflow fields like: step, step_number, total_steps, # next_step_required, findings, etc. See simulator_tests/test_precommitworkflow_validation.py # for comprehensive workflow testing including large prompt handling. # NOTE: Debug tool tests have been commented out because the debug tool has been # refactored to use a self-investigation pattern instead of accepting a prompt field. # The new debug tool requires fields like: step, step_number, total_steps, next_step_required, findings # and doesn't have the "resend_prompt" functionality for large prompts. # @pytest.mark.asyncio # async def test_debug_large_error_description(self, large_prompt): # """Test that debug tool detects large error_description.""" # tool = DebugIssueTool() # result = await tool.execute({"prompt": large_prompt}) # # assert len(result) == 1 # output = json.loads(result[0].text) # assert output["status"] == "resend_prompt" # @pytest.mark.asyncio # async def test_debug_large_error_context(self, large_prompt, normal_prompt): # """Test that debug tool detects large error_context.""" # tool = DebugIssueTool() # result = await tool.execute({"prompt": normal_prompt, "error_context": large_prompt}) # # assert len(result) == 1 # output = json.loads(result[0].text) # assert output["status"] == "resend_prompt" # Removed: test_analyze_large_question - workflow tool handles large prompts differently @pytest.mark.asyncio async def test_multiple_files_with_prompt_txt(self, temp_prompt_file): """Test handling of prompt.txt alongside other files.""" tool = ChatTool() other_file = "/some/other/file.py" with ( patch("utils.model_context.ModelContext") as mock_model_context_cls, patch.object(tool, "handle_prompt_file") as mock_handle_prompt, patch.object(tool, "_prepare_file_content_for_prompt") as mock_prepare_files, ): mock_provider = MagicMock() mock_provider.get_provider_type.return_value = MagicMock(value="google") mock_provider.generate_content.return_value = MagicMock( content="Success", usage={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, model_name="gemini-2.5-flash", metadata={"finish_reason": "STOP"}, ) from utils.model_context import TokenAllocation mock_model_context = MagicMock() mock_model_context.model_name = "gemini-2.5-flash" mock_model_context.provider = mock_provider mock_model_context.capabilities = MagicMock(supports_extended_thinking=False) mock_model_context.calculate_token_allocation.return_value = TokenAllocation( total_tokens=1_000_000, content_tokens=800_000, response_tokens=200_000, file_tokens=320_000, history_tokens=320_000, ) mock_model_context_cls.return_value = mock_model_context # Return the prompt content and updated files list (without prompt.txt) mock_handle_prompt.return_value = ("Large prompt content from file", [other_file]) # Mock the centralized file preparation method mock_prepare_files.return_value = ("File content", [other_file]) # Use a small prompt to avoid triggering size limit await tool.execute( { "prompt": "Test prompt", "absolute_file_paths": [temp_prompt_file, other_file], "working_directory_absolute_path": os.path.dirname(temp_prompt_file), } ) # Verify handle_prompt_file was called with the original files list mock_handle_prompt.assert_called_once_with([temp_prompt_file, other_file]) # Verify _prepare_file_content_for_prompt was called with the updated files list (without prompt.txt) mock_prepare_files.assert_called_once() files_arg = mock_prepare_files.call_args[0][0] assert len(files_arg) == 1 assert files_arg[0] == other_file temp_dir = os.path.dirname(temp_prompt_file) shutil.rmtree(temp_dir) @pytest.mark.asyncio async def test_boundary_case_exactly_at_limit(self): """Test prompt exactly at MCP_PROMPT_SIZE_LIMIT characters (should pass with the fix).""" tool = ChatTool() exact_prompt = "x" * MCP_PROMPT_SIZE_LIMIT # Mock the model provider to avoid real API calls with patch.object(tool, "get_model_provider") as mock_get_provider: mock_provider = MagicMock() mock_provider.get_provider_type.return_value = MagicMock(value="google") mock_provider.get_capabilities.return_value = MagicMock(supports_extended_thinking=False) mock_provider.generate_content.return_value = MagicMock( content="Response to the large prompt", usage={"input_tokens": 12000, "output_tokens": 10, "total_tokens": 12010}, model_name="gemini-2.5-flash", metadata={"finish_reason": "STOP"}, ) mock_get_provider.return_value = mock_provider # With the fix, this should now pass because we check at MCP transport boundary before adding internal content temp_dir = tempfile.mkdtemp() try: try: result = await tool.execute({"prompt": exact_prompt, "working_directory_absolute_path": temp_dir}) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: output = json.loads(result[0].text) finally: shutil.rmtree(temp_dir, ignore_errors=True) assert output["status"] != "resend_prompt" @pytest.mark.asyncio async def test_boundary_case_just_over_limit(self): """Test prompt just over MCP_PROMPT_SIZE_LIMIT characters (should trigger file request).""" tool = ChatTool() over_prompt = "x" * (MCP_PROMPT_SIZE_LIMIT + 1) temp_dir = tempfile.mkdtemp() try: try: result = await tool.execute({"prompt": over_prompt, "working_directory_absolute_path": temp_dir}) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: output = json.loads(result[0].text) finally: shutil.rmtree(temp_dir, ignore_errors=True) assert output["status"] == "resend_prompt" @pytest.mark.asyncio async def test_empty_prompt_no_file(self): """Test empty prompt without prompt.txt file.""" tool = ChatTool() with patch.object(tool, "get_model_provider") as mock_get_provider: mock_provider = MagicMock() mock_provider.get_provider_type.return_value = MagicMock(value="google") mock_provider.get_capabilities.return_value = MagicMock(supports_extended_thinking=False) mock_provider.generate_content.return_value = MagicMock( content="Success", usage={"input_tokens": 10, "output_tokens": 20, "total_tokens": 30}, model_name="gemini-2.5-flash", metadata={"finish_reason": "STOP"}, ) mock_get_provider.return_value = mock_provider temp_dir = tempfile.mkdtemp() try: try: result = await tool.execute({"prompt": "", "working_directory_absolute_path": temp_dir}) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: output = json.loads(result[0].text) finally: shutil.rmtree(temp_dir, ignore_errors=True) assert output["status"] != "resend_prompt" @pytest.mark.asyncio async def test_prompt_file_read_error(self): """Test handling when prompt.txt can't be read.""" from tests.mock_helpers import create_mock_provider tool = ChatTool() bad_file = "/nonexistent/prompt.txt" with ( patch.object(tool, "get_model_provider") as mock_get_provider, patch("utils.model_context.ModelContext") as mock_model_context_class, ): mock_provider = create_mock_provider(model_name="gemini-2.5-flash", context_window=1_048_576) mock_provider.generate_content.return_value.content = "Success" mock_get_provider.return_value = mock_provider # Mock ModelContext to avoid the comparison issue from utils.model_context import TokenAllocation mock_model_context = MagicMock() mock_model_context.model_name = "gemini-2.5-flash" mock_model_context.calculate_token_allocation.return_value = TokenAllocation( total_tokens=1_048_576, content_tokens=838_861, response_tokens=209_715, file_tokens=335_544, history_tokens=335_544, ) mock_model_context_class.return_value = mock_model_context # Should continue with empty prompt when file can't be read temp_dir = tempfile.mkdtemp() try: try: result = await tool.execute( {"prompt": "", "absolute_file_paths": [bad_file], "working_directory_absolute_path": temp_dir} ) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: output = json.loads(result[0].text) finally: shutil.rmtree(temp_dir, ignore_errors=True) assert output["status"] != "resend_prompt" @pytest.mark.asyncio async def test_large_file_context_does_not_trigger_mcp_prompt_limit(self, tmp_path): """Large context files should not be blocked by MCP prompt limit enforcement.""" from tests.mock_helpers import create_mock_provider from utils.model_context import TokenAllocation tool = ChatTool() # Create a file significantly larger than MCP_PROMPT_SIZE_LIMIT characters large_content = "A" * (MCP_PROMPT_SIZE_LIMIT * 5) large_file = tmp_path / "huge_context.txt" large_file.write_text(large_content) mock_provider = create_mock_provider(model_name="flash") class DummyModelContext: def __init__(self, provider): self.model_name = "flash" self._provider = provider self.capabilities = provider.get_capabilities("flash") @property def provider(self): return self._provider def calculate_token_allocation(self): return TokenAllocation( total_tokens=1_048_576, content_tokens=838_861, response_tokens=209_715, file_tokens=335_544, history_tokens=335_544, ) dummy_context = DummyModelContext(mock_provider) with patch.object(tool, "get_model_provider", return_value=mock_provider): result = await tool.execute( { "prompt": "Summarize the design decisions", "absolute_file_paths": [str(large_file)], "model": "flash", "working_directory_absolute_path": str(tmp_path), "_model_context": dummy_context, } ) output = json.loads(result[0].text) assert output["status"] != "resend_prompt" @pytest.mark.asyncio async def test_mcp_boundary_with_large_internal_context(self): """ Critical test: Ensure MCP_PROMPT_SIZE_LIMIT only applies to user input (MCP boundary), NOT to internal context like conversation history, system prompts, or file content. This test verifies that even if our internal prompt (with system prompts, history, etc.) exceeds MCP_PROMPT_SIZE_LIMIT, it should still work as long as the user's input is small. """ tool = ChatTool() # Small user input that should pass MCP boundary check small_user_prompt = "What is the weather like?" # Mock a huge conversation history that would exceed MCP limits if incorrectly checked huge_history = "x" * (MCP_PROMPT_SIZE_LIMIT * 2) # 100K chars = way over 50K limit temp_dir = tempfile.mkdtemp() original_prepare_prompt = tool.prepare_prompt try: with ( patch.object(tool, "get_model_provider") as mock_get_provider, patch("utils.model_context.ModelContext") as mock_model_context_class, ): from tests.mock_helpers import create_mock_provider from utils.model_context import TokenAllocation mock_provider = create_mock_provider(model_name="flash") mock_get_provider.return_value = mock_provider mock_model_context = MagicMock() mock_model_context.model_name = "flash" mock_model_context.provider = mock_provider mock_model_context.calculate_token_allocation.return_value = TokenAllocation( total_tokens=1_048_576, content_tokens=838_861, response_tokens=209_715, file_tokens=335_544, history_tokens=335_544, ) mock_model_context_class.return_value = mock_model_context async def mock_prepare_prompt(request): normal_prompt = await original_prepare_prompt(request) huge_internal_prompt = f"{normal_prompt}\n\n=== HUGE INTERNAL CONTEXT ===\n{huge_history}" assert len(huge_internal_prompt) > MCP_PROMPT_SIZE_LIMIT return huge_internal_prompt tool.prepare_prompt = mock_prepare_prompt result = await tool.execute( {"prompt": small_user_prompt, "model": "flash", "working_directory_absolute_path": temp_dir} ) output = json.loads(result[0].text) assert output["status"] != "resend_prompt" mock_provider.generate_content.assert_called_once() call_kwargs = mock_provider.generate_content.call_args[1] actual_prompt = call_kwargs.get("prompt") assert len(actual_prompt) > MCP_PROMPT_SIZE_LIMIT assert huge_history in actual_prompt assert small_user_prompt in actual_prompt finally: tool.prepare_prompt = original_prepare_prompt shutil.rmtree(temp_dir, ignore_errors=True) @pytest.mark.asyncio async def test_mcp_boundary_vs_internal_processing_distinction(self): """ Test that clearly demonstrates the distinction between: 1. MCP transport boundary (user input - SHOULD be limited) 2. Internal processing (system prompts, files, history - should NOT be limited) """ tool = ChatTool() # Test case 1: Large user input should fail at MCP boundary large_user_input = "x" * (MCP_PROMPT_SIZE_LIMIT + 1000) temp_dir = tempfile.mkdtemp() try: try: result = await tool.execute( {"prompt": large_user_input, "model": "flash", "working_directory_absolute_path": temp_dir} ) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: output = json.loads(result[0].text) assert output["status"] == "resend_prompt" # Should fail assert "too large for MCP's token limits" in output["content"] # Test case 2: Small user input should succeed even with huge internal processing small_user_input = "Hello" try: result = await tool.execute( { "prompt": small_user_input, "model": "gemini-2.5-flash", "working_directory_absolute_path": temp_dir, } ) except ToolExecutionError as exc: output = json.loads(exc.payload if hasattr(exc, "payload") else str(exc)) else: output = json.loads(result[0].text) # The test will fail with dummy API keys, which is expected behavior # We're mainly testing that the tool processes small prompts correctly without size errors assert output["status"] != "resend_prompt" finally: shutil.rmtree(temp_dir, ignore_errors=True) @pytest.mark.asyncio async def test_continuation_with_huge_conversation_history(self): """ Test that continuation calls with huge conversation history work correctly. This simulates the exact scenario where conversation history builds up and exceeds MCP_PROMPT_SIZE_LIMIT but should still work since history is internal processing. """ tool = ChatTool() # Small user input for continuation small_continuation_prompt = "Continue the discussion" # Mock huge conversation history (simulates many turns of conversation) # Calculate repetitions needed to exceed MCP_PROMPT_SIZE_LIMIT base_text = "=== CONVERSATION HISTORY ===\n" repeat_text = "Previous message content\n" # Add buffer to ensure we exceed the limit target_size = MCP_PROMPT_SIZE_LIMIT + 1000 available_space = target_size - len(base_text) repetitions_needed = (available_space // len(repeat_text)) + 1 huge_conversation_history = base_text + (repeat_text * repetitions_needed) # Ensure the history exceeds MCP limits assert len(huge_conversation_history) > MCP_PROMPT_SIZE_LIMIT temp_dir = tempfile.mkdtemp() with ( patch.object(tool, "get_model_provider") as mock_get_provider, patch("utils.model_context.ModelContext") as mock_model_context_class, ): from tests.mock_helpers import create_mock_provider mock_provider = create_mock_provider(model_name="flash") mock_provider.generate_content.return_value.content = "Continuing our conversation..." mock_get_provider.return_value = mock_provider # Mock ModelContext to avoid the comparison issue from utils.model_context import TokenAllocation mock_model_context = MagicMock() mock_model_context.model_name = "flash" mock_model_context.provider = mock_provider mock_model_context.calculate_token_allocation.return_value = TokenAllocation( total_tokens=1_048_576, content_tokens=838_861, response_tokens=209_715, file_tokens=335_544, history_tokens=335_544, ) mock_model_context_class.return_value = mock_model_context # Simulate continuation by having the request contain embedded conversation history # This mimics what server.py does when it embeds conversation history request_with_history = { "prompt": f"{huge_conversation_history}\n\n=== CURRENT REQUEST ===\n{small_continuation_prompt}", "model": "flash", "continuation_id": "test_thread_123", "working_directory_absolute_path": temp_dir, } # Mock the conversation history embedding to simulate server.py behavior original_execute = tool.__class__.execute async def mock_execute_with_history(self, arguments): # Check if this has continuation_id (simulating server.py logic) if arguments.get("continuation_id"): # Simulate the case where conversation history is already embedded in prompt # by server.py before calling the tool field_value = arguments.get("prompt", "") if "=== CONVERSATION HISTORY ===" in field_value: # Set the flag that history is embedded self._has_embedded_history = True # The prompt field contains both history AND user input # But we should only check the user input part for MCP boundary # (This is what our fix ensures happens in prepare_prompt) # Call original execute return await original_execute(self, arguments) tool.__class__.execute = mock_execute_with_history try: # This should succeed because: # 1. The actual user input is small (passes MCP boundary check) # 2. The huge conversation history is internal processing (not subject to MCP limits) result = await tool.execute(request_with_history) output = json.loads(result[0].text) # Should succeed even though total prompt with history is huge assert output["status"] != "resend_prompt" assert "Continuing our conversation" in output["content"] # Verify the model was called with the complete prompt (including huge history) mock_provider.generate_content.assert_called_once() call_kwargs = mock_provider.generate_content.call_args[1] final_prompt = call_kwargs.get("prompt") # The final prompt should contain both history and user input assert huge_conversation_history in final_prompt assert small_continuation_prompt in final_prompt # And it should be huge (proving we don't limit internal processing) assert len(final_prompt) > MCP_PROMPT_SIZE_LIMIT finally: # Restore original execute method tool.__class__.execute = original_execute shutil.rmtree(temp_dir, ignore_errors=True) if __name__ == "__main__": pytest.main([__file__, "-v"]) ================================================ FILE: tests/test_line_numbers_integration.py ================================================ """ Integration test demonstrating that all tools get line numbers by default. """ from tools.analyze import AnalyzeTool from tools.chat import ChatTool from tools.codereview import CodeReviewTool from tools.debug import DebugIssueTool from tools.precommit import PrecommitTool from tools.refactor import RefactorTool from tools.testgen import TestGenTool class TestLineNumbersIntegration: """Test that all tools inherit line number behavior correctly.""" def test_all_tools_want_line_numbers(self): """Verify that all tools want line numbers by default.""" tools = [ ChatTool(), AnalyzeTool(), CodeReviewTool(), DebugIssueTool(), RefactorTool(), TestGenTool(), PrecommitTool(), ] for tool in tools: assert tool.wants_line_numbers_by_default(), f"{tool.get_name()} should want line numbers by default" def test_no_tools_override_line_numbers(self): """Verify that no tools override the base class line number behavior.""" # Check that tools don't have their own wants_line_numbers_by_default method tools_classes = [ ChatTool, AnalyzeTool, CodeReviewTool, DebugIssueTool, RefactorTool, TestGenTool, PrecommitTool, ] for tool_class in tools_classes: # Check if the method is defined in the tool class itself # (not inherited from base) has_override = "wants_line_numbers_by_default" in tool_class.__dict__ assert not has_override, f"{tool_class.__name__} should not override wants_line_numbers_by_default" ================================================ FILE: tests/test_listmodels.py ================================================ """Tests for the ListModels tool""" import json import os from unittest.mock import patch import pytest from mcp.types import TextContent from tools.listmodels import ListModelsTool class TestListModelsTool: """Test the ListModels tool functionality""" @pytest.fixture def tool(self): """Create a ListModelsTool instance""" return ListModelsTool() def test_tool_metadata(self, tool): """Test tool has correct metadata""" assert tool.name == "listmodels" assert "model providers" in tool.description assert tool.get_request_model().__name__ == "ToolRequest" @pytest.mark.asyncio async def test_execute_with_no_providers(self, tool): """Test listing models with no providers configured""" with patch.dict(os.environ, {}, clear=True): # Set auto mode os.environ["DEFAULT_MODEL"] = "auto" result = await tool.execute({}) assert len(result) == 1 assert isinstance(result[0], TextContent) # Parse JSON response response = json.loads(result[0].text) assert response["status"] == "success" content = response["content"] # Check that providers show as not configured assert "Google Gemini ❌" in content assert "OpenAI ❌" in content assert "X.AI (Grok) ❌" in content assert "OpenRouter ❌" in content assert "Custom/Local API ❌" in content # Check summary shows 0 configured assert "**Configured Providers**: 0" in content @pytest.mark.asyncio async def test_execute_with_gemini_configured(self, tool): """Test listing models with Gemini configured""" env_vars = {"GEMINI_API_KEY": "test-key", "DEFAULT_MODEL": "auto"} with patch.dict(os.environ, env_vars, clear=True): result = await tool.execute({}) response = json.loads(result[0].text) content = response["content"] # Check Gemini shows as configured assert "Google Gemini ✅" in content assert "`flash` → `gemini-2.5-flash`" in content assert "`pro` → `gemini-3-pro-preview`" in content assert "1M context" in content assert "Supports structured code generation" in content # Check summary assert "**Configured Providers**: 1" in content @pytest.mark.asyncio async def test_execute_with_multiple_providers(self, tool): """Test listing models with multiple providers configured""" env_vars = { "GEMINI_API_KEY": "test-key", "OPENAI_API_KEY": "test-key", "XAI_API_KEY": "test-key", "DEFAULT_MODEL": "auto", } with patch.dict(os.environ, env_vars, clear=True): result = await tool.execute({}) response = json.loads(result[0].text) content = response["content"] # Check all show as configured assert "Google Gemini ✅" in content assert "OpenAI ✅" in content assert "X.AI (Grok) ✅" in content # Check models are listed assert "`o3`" in content assert "`grok`" in content # Check summary assert "**Configured Providers**: 3" in content @pytest.mark.asyncio async def test_execute_with_openrouter(self, tool): """Test listing models with OpenRouter configured""" env_vars = {"OPENROUTER_API_KEY": "test-key", "DEFAULT_MODEL": "auto"} with patch.dict(os.environ, env_vars, clear=True): result = await tool.execute({}) response = json.loads(result[0].text) content = response["content"] # Check OpenRouter shows as configured assert "OpenRouter ✅" in content assert "Access to multiple cloud AI providers" in content # Should show some models (mocked registry will have some) assert "Available Models" in content @pytest.mark.asyncio async def test_execute_with_custom_api(self, tool): """Test listing models with custom API configured""" env_vars = {"CUSTOM_API_URL": "http://localhost:11434", "DEFAULT_MODEL": "auto"} with patch.dict(os.environ, env_vars, clear=True): result = await tool.execute({}) response = json.loads(result[0].text) content = response["content"] # Check Custom API shows as configured assert "Custom/Local API ✅" in content assert "http://localhost:11434" in content assert "Local models via Ollama" in content @pytest.mark.asyncio async def test_output_includes_usage_tips(self, tool): """Test that output includes helpful usage tips""" result = await tool.execute({}) response = json.loads(result[0].text) content = response["content"] # Check for usage tips assert "**Usage Tips**:" in content assert "Use model aliases" in content assert "auto mode" in content def test_model_category(self, tool): """Test that tool uses FAST_RESPONSE category""" from tools.models import ToolModelCategory assert tool.get_model_category() == ToolModelCategory.FAST_RESPONSE ================================================ FILE: tests/test_listmodels_restrictions.py ================================================ """Test listmodels tool respects model restrictions.""" import asyncio import os import unittest from unittest.mock import MagicMock, patch from providers.base import ModelProvider from providers.registry import ModelProviderRegistry from providers.shared import ModelCapabilities, ProviderType from tools.listmodels import ListModelsTool class TestListModelsRestrictions(unittest.TestCase): """Test that listmodels respects OPENROUTER_ALLOWED_MODELS.""" def setUp(self): """Set up test environment.""" # Clear any existing registry state ModelProviderRegistry.clear_cache() # Create mock OpenRouter provider self.mock_openrouter = MagicMock(spec=ModelProvider) self.mock_openrouter.provider_type = ProviderType.OPENROUTER def make_capabilities( canonical: str, friendly: str, *, aliases=None, context: int = 200_000 ) -> ModelCapabilities: return ModelCapabilities( provider=ProviderType.OPENROUTER, model_name=canonical, friendly_name=friendly, intelligence_score=20, description=friendly, aliases=aliases or [], context_window=context, max_output_tokens=context, supports_extended_thinking=True, ) opus_caps = make_capabilities( "anthropic/claude-opus-4-20240229", "Claude Opus", aliases=["opus"], ) sonnet_caps = make_capabilities( "anthropic/claude-sonnet-4-20240229", "Claude Sonnet", aliases=["sonnet"], ) deepseek_caps = make_capabilities( "deepseek/deepseek-r1-0528:free", "DeepSeek R1", aliases=[], ) qwen_caps = make_capabilities( "qwen/qwen3-235b-a22b-04-28:free", "Qwen3", aliases=[], ) self._openrouter_caps_map = { "anthropic/claude-opus-4": opus_caps, "opus": opus_caps, "anthropic/claude-opus-4-20240229": opus_caps, "anthropic/claude-sonnet-4": sonnet_caps, "sonnet": sonnet_caps, "anthropic/claude-sonnet-4-20240229": sonnet_caps, "deepseek/deepseek-r1-0528:free": deepseek_caps, "qwen/qwen3-235b-a22b-04-28:free": qwen_caps, } self.mock_openrouter.get_capabilities.side_effect = self._openrouter_caps_map.__getitem__ self.mock_openrouter.get_capabilities_by_rank.return_value = [] self.mock_openrouter.list_models.return_value = [] # Create mock Gemini provider for comparison self.mock_gemini = MagicMock(spec=ModelProvider) self.mock_gemini.provider_type = ProviderType.GOOGLE self.mock_gemini.list_models.return_value = ["gemini-2.5-flash", "gemini-2.5-pro"] self.mock_gemini.get_capabilities_by_rank.return_value = [] self.mock_gemini.get_capabilities_by_rank.return_value = [] def tearDown(self): """Clean up after tests.""" ModelProviderRegistry.clear_cache() # Clean up environment variables for key in ["OPENROUTER_ALLOWED_MODELS", "OPENROUTER_API_KEY", "GEMINI_API_KEY"]: os.environ.pop(key, None) @patch.dict( os.environ, { "OPENROUTER_API_KEY": "test-key", "OPENROUTER_ALLOWED_MODELS": "opus,sonnet,deepseek/deepseek-r1-0528:free,qwen/qwen3-235b-a22b-04-28:free", "GEMINI_API_KEY": "gemini-test-key", }, ) @patch("utils.model_restrictions.get_restriction_service") @patch("providers.registries.openrouter.OpenRouterModelRegistry") @patch.object(ModelProviderRegistry, "get_available_models") @patch.object(ModelProviderRegistry, "get_provider") def test_listmodels_respects_openrouter_restrictions( self, mock_get_provider, mock_get_models, mock_registry_class, mock_get_restriction ): """Test that listmodels only shows allowed OpenRouter models.""" # Set up mock to return only allowed models when restrictions are respected # Include both aliased models and full model names without aliases self.mock_openrouter.list_models.return_value = [ "anthropic/claude-opus-4", # Has alias "opus" "anthropic/claude-sonnet-4", # Has alias "sonnet" "deepseek/deepseek-r1-0528:free", # No alias, full name "qwen/qwen3-235b-a22b-04-28:free", # No alias, full name ] # Mock registry instance mock_registry = MagicMock() mock_registry_class.return_value = mock_registry # Mock resolve method - return config for aliased models, None for others def resolve_side_effect(model_name): if "opus" in model_name.lower(): config = MagicMock() config.model_name = "anthropic/claude-opus-4-20240229" config.context_window = 200000 config.get_effective_capability_rank.return_value = 90 # High rank for Opus return config elif "sonnet" in model_name.lower(): config = MagicMock() config.model_name = "anthropic/claude-sonnet-4-20240229" config.context_window = 200000 config.get_effective_capability_rank.return_value = 80 # Lower rank for Sonnet return config elif "deepseek" in model_name.lower(): config = MagicMock() config.model_name = "deepseek/deepseek-r1-0528:free" config.context_window = 100000 config.get_effective_capability_rank.return_value = 70 return config elif "qwen" in model_name.lower(): config = MagicMock() config.model_name = "qwen/qwen3-235b-a22b-04-28:free" config.context_window = 100000 config.get_effective_capability_rank.return_value = 60 return config return None # No config for models without aliases mock_registry.resolve.side_effect = resolve_side_effect # Mock provider registry def get_provider_side_effect(provider_type, force_new=False): if provider_type == ProviderType.OPENROUTER: return self.mock_openrouter elif provider_type == ProviderType.GOOGLE: return self.mock_gemini return None mock_get_provider.side_effect = get_provider_side_effect # Ensure registry is cleared before test ModelProviderRegistry._registry = {} # Mock available models mock_get_models.return_value = { "gemini-2.5-flash": ProviderType.GOOGLE, "gemini-2.5-pro": ProviderType.GOOGLE, "anthropic/claude-opus-4-20240229": ProviderType.OPENROUTER, "anthropic/claude-sonnet-4-20240229": ProviderType.OPENROUTER, "deepseek/deepseek-r1-0528:free": ProviderType.OPENROUTER, "qwen/qwen3-235b-a22b-04-28:free": ProviderType.OPENROUTER, } # Mock restriction service mock_restriction_service = MagicMock() mock_restriction_service.has_restrictions.return_value = True mock_restriction_service.get_allowed_models.return_value = { "opus", "sonnet", "deepseek/deepseek-r1-0528:free", "qwen/qwen3-235b-a22b-04-28:free", } mock_get_restriction.return_value = mock_restriction_service # Create tool and execute tool = ListModelsTool() # Execute asynchronously loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) result_contents = loop.run_until_complete(tool.execute({})) loop.close() # Extract text content from result result_text = result_contents[0].text # Parse JSON response import json result_json = json.loads(result_text) result = result_json["content"] # Parse the output lines = result.split("\n") # Debug: print the actual result for troubleshooting # print(f"DEBUG: Full result:\n{result}") # Check that OpenRouter section exists openrouter_section_found = False openrouter_models = [] in_openrouter_section = False for line in lines: if "OpenRouter" in line and "✅" in line: openrouter_section_found = True elif ("Models (policy restricted)" in line or "Available Models" in line) and openrouter_section_found: in_openrouter_section = True elif in_openrouter_section: # Check for lines with model names in backticks # Format: - `model-name` (score X) if line.strip().startswith("- ") and "`" in line: # Extract model name between backticks parts = line.split("`") if len(parts) >= 2: model_name = parts[1] openrouter_models.append(model_name) # Stop parsing when we hit the next section elif "##" in line and in_openrouter_section: break self.assertTrue(openrouter_section_found, "OpenRouter section not found") self.assertEqual( len(openrouter_models), 4, f"Expected 4 models, got {len(openrouter_models)}: {openrouter_models}" ) # Verify we did not fall back to unrestricted listing self.mock_openrouter.list_models.assert_not_called() # Check for restriction note self.assertIn("OpenRouter models restricted by", result) @patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key", "GEMINI_API_KEY": "gemini-test-key"}, clear=True) @patch("providers.registries.openrouter.OpenRouterModelRegistry") @patch.object(ModelProviderRegistry, "get_provider") def test_listmodels_shows_all_models_without_restrictions(self, mock_get_provider, mock_registry_class): """Test that listmodels shows all models when no restrictions are set.""" # Clear any cached restriction service to ensure it reads from patched environment import utils.model_restrictions utils.model_restrictions._restriction_service = None # Set up mock to return many models when no restrictions all_models = [f"provider{i // 10}/model-{i}" for i in range(50)] # Simulate 50 models from different providers self.mock_openrouter.list_models.return_value = all_models # Mock registry instance mock_registry = MagicMock() mock_registry_class.return_value = mock_registry mock_registry.resolve.return_value = None # No configs for simplicity # Mock provider registry def get_provider_side_effect(provider_type, force_new=False): if provider_type == ProviderType.OPENROUTER: return self.mock_openrouter elif provider_type == ProviderType.GOOGLE: return self.mock_gemini return None mock_get_provider.side_effect = get_provider_side_effect # Create tool and execute tool = ListModelsTool() # Execute asynchronously loop = asyncio.new_event_loop() asyncio.set_event_loop(loop) result_contents = loop.run_until_complete(tool.execute({})) loop.close() # Extract text content from result result_text = result_contents[0].text # Parse JSON response import json result_json = json.loads(result_text) result = result_json["content"] # Count OpenRouter models specifically lines = result.split("\n") openrouter_section_found = False openrouter_model_count = 0 for line in lines: if "OpenRouter" in line and "✅" in line: openrouter_section_found = True elif "Custom/Local API" in line: # End of OpenRouter section break elif openrouter_section_found and line.strip().startswith("- ") and "`" in line: openrouter_model_count += 1 # After removing limits, the tool shows ALL available models (no truncation) # With 50 models from providers, we expect to see ALL of them self.assertGreaterEqual( openrouter_model_count, 30, f"Expected to see many OpenRouter models (no limits), found {openrouter_model_count}", ) # Should NOT show "and X more models available" message since we show all models now self.assertNotIn("more models available", result) # Verify list_models was called with respect_restrictions=True # (even without restrictions, we always pass True) self.mock_openrouter.list_models.assert_called_with(respect_restrictions=True) # Should NOT have restriction note when no restrictions are set self.assertNotIn("Restricted to models matching:", result) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_mcp_error_handling.py ================================================ import json from types import SimpleNamespace import pytest from mcp.types import CallToolRequest, CallToolRequestParams from providers.registry import ModelProviderRegistry from server import server as mcp_server def _install_dummy_provider(monkeypatch): """Ensure preflight model checks succeed without real provider configuration.""" class DummyProvider: def get_provider_type(self): return SimpleNamespace(value="dummy") def get_capabilities(self, model_name): return SimpleNamespace( supports_extended_thinking=False, allow_code_generation=False, supports_images=False, context_window=1_000_000, max_image_size_mb=10, ) monkeypatch.setattr( ModelProviderRegistry, "get_provider_for_model", classmethod(lambda cls, model_name: DummyProvider()), ) monkeypatch.setattr( ModelProviderRegistry, "get_available_models", classmethod(lambda cls, respect_restrictions=False: {"gemini-2.5-flash": None}), ) @pytest.mark.asyncio async def test_tool_execution_error_sets_is_error_flag_for_mcp_response(monkeypatch): """Ensure ToolExecutionError surfaces as CallToolResult with isError=True.""" _install_dummy_provider(monkeypatch) handler = mcp_server.request_handlers[CallToolRequest] arguments = { "prompt": "Trigger working_directory_absolute_path validation failure", "working_directory_absolute_path": "relative/path", # Not absolute -> ToolExecutionError from ChatTool "absolute_file_paths": [], "model": "gemini-2.5-flash", } request = CallToolRequest(params=CallToolRequestParams(name="chat", arguments=arguments)) server_result = await handler(request) assert server_result.root.isError is True assert server_result.root.content, "Expected error response content" payload = server_result.root.content[0].text data = json.loads(payload) assert data["status"] == "error" assert "absolute" in data["content"].lower() ================================================ FILE: tests/test_model_enumeration.py ================================================ """ Integration tests for model enumeration across all provider combinations. These tests ensure that the _get_available_models() method correctly returns all expected models based on which providers are configured via environment variables. """ import importlib import json import os import pytest from providers.registry import ModelProviderRegistry from tools.analyze import AnalyzeTool @pytest.mark.no_mock_provider class TestModelEnumeration: """Test model enumeration with various provider configurations""" def setup_method(self): """Set up clean state before each test.""" # Save original environment state self._original_env = { "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL", ""), "GEMINI_API_KEY": os.environ.get("GEMINI_API_KEY", ""), "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY", ""), "XAI_API_KEY": os.environ.get("XAI_API_KEY", ""), "OPENROUTER_API_KEY": os.environ.get("OPENROUTER_API_KEY", ""), "CUSTOM_API_URL": os.environ.get("CUSTOM_API_URL", ""), } # Clear provider registry ModelProviderRegistry._instance = None def teardown_method(self): """Clean up after each test.""" # Restore original environment for key, value in self._original_env.items(): if value: os.environ[key] = value elif key in os.environ: del os.environ[key] # Reload config import config importlib.reload(config) # Clear provider registry ModelProviderRegistry._instance = None def _setup_environment(self, provider_config): """Helper to set up environment variables for testing.""" # Clear all provider-related env vars first for key in ["GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "CUSTOM_API_URL"]: if key in os.environ: del os.environ[key] # Set new values for key, value in provider_config.items(): if value is not None: os.environ[key] = value # Set auto mode only if not explicitly set in provider_config if "DEFAULT_MODEL" not in provider_config: os.environ["DEFAULT_MODEL"] = "auto" # Reload config to pick up changes import config importlib.reload(config) # Note: tools.base has been refactored to tools.shared.base_tool and tools.simple.base # No longer need to reload as configuration is handled at provider level def test_no_models_when_no_providers_configured(self): """Test that no native models are included when no providers are configured.""" self._setup_environment({}) # No providers configured tool = AnalyzeTool() models = tool._get_available_models() # After the fix, models should only be shown from enabled providers # With no API keys configured, no providers should be enabled # Only OpenRouter aliases might still appear if they're in the registry # Filter out OpenRouter aliases that might still appear non_openrouter_models = [ m for m in models if "/" not in m and m not in ["gemini", "pro", "flash", "opus", "sonnet", "haiku"] ] # No native provider models should be present without API keys assert ( len(non_openrouter_models) == 0 ), f"No native models should be available without API keys, but found: {non_openrouter_models}" def test_openrouter_models_without_api_key(self): """Test that OpenRouter models are NOT included when API key is not configured.""" self._setup_environment({}) # No OpenRouter key tool = AnalyzeTool() models = tool._get_available_models() # OpenRouter-specific models should NOT be present openrouter_only_models = ["opus", "sonnet", "haiku"] found_count = sum(1 for m in openrouter_only_models if m in models) assert found_count == 0, "OpenRouter models should not be included without API key" def test_custom_models_without_custom_url(self): """Test that custom models are NOT included when CUSTOM_API_URL is not configured.""" self._setup_environment({}) # No custom URL tool = AnalyzeTool() models = tool._get_available_models() # Custom-only models should NOT be present custom_only_models = ["local-llama", "llama3.2"] found_count = sum(1 for m in custom_only_models if m in models) assert found_count == 0, "Custom models should not be included without CUSTOM_API_URL" def test_custom_models_not_exposed_with_openrouter_only(self): """Ensure OpenRouter access alone does not surface custom-only endpoints.""" self._setup_environment({"OPENROUTER_API_KEY": "test-openrouter-key"}) tool = AnalyzeTool() models = tool._get_available_models() for alias in ("local-llama", "llama3.2"): assert alias not in models, f"Custom model alias '{alias}' should remain hidden without CUSTOM_API_URL" def test_no_duplicates_with_overlapping_providers(self): """Test that models aren't duplicated when multiple providers offer the same model.""" self._setup_environment( { "OPENAI_API_KEY": "test", "OPENROUTER_API_KEY": "test", # OpenRouter also offers OpenAI models } ) tool = AnalyzeTool() models = tool._get_available_models() # Count occurrences of each model model_counts = {} for model in models: model_counts[model] = model_counts.get(model, 0) + 1 # Check no duplicates duplicates = {m: count for m, count in model_counts.items() if count > 1} assert len(duplicates) == 0, f"Found duplicate models: {duplicates}" @pytest.mark.parametrize( "model_name,should_exist", [ ("flash", False), # Gemini - not available without API key ("o3", False), # OpenAI - not available without API key ("grok", False), # X.AI - not available without API key ("gemini-2.5-flash", False), # Full Gemini name - not available without API key ("o4-mini", False), # OpenAI variant - not available without API key ("grok-4.1-fast", False), # X.AI variant - not available without API key ], ) def test_specific_native_models_only_with_api_keys(self, model_name, should_exist): """Test that native models are only present when their provider has API keys configured.""" self._setup_environment({}) # No providers tool = AnalyzeTool() models = tool._get_available_models() if should_exist: assert model_name in models, f"Model {model_name} should be present" else: assert model_name not in models, f"Native model {model_name} should not be present without API key" def test_openrouter_free_model_aliases_available(self, tmp_path, monkeypatch): """Free OpenRouter variants should expose both canonical names and aliases.""" # Configure environment with OpenRouter access only self._setup_environment({"OPENROUTER_API_KEY": "test-openrouter-key"}) # Create a temporary OpenRouter model config with a free variant custom_config = { "models": [ { "model_name": "deepseek/deepseek-r1:free", "aliases": ["deepseek-free", "r1-free"], "context_window": 163840, "max_output_tokens": 8192, "supports_extended_thinking": False, "supports_json_mode": True, "supports_function_calling": False, "supports_images": False, "max_image_size_mb": 0.0, "description": "DeepSeek R1 free tier variant", } ] } config_path = tmp_path / "openrouter_models.json" config_path.write_text(json.dumps(custom_config), encoding="utf-8") monkeypatch.setenv("OPENROUTER_MODELS_CONFIG_PATH", str(config_path)) # Reset cached registries so the temporary config is loaded from tools.shared.base_tool import BaseTool monkeypatch.setattr(BaseTool, "_openrouter_registry_cache", None, raising=False) from providers.openrouter import OpenRouterProvider monkeypatch.setattr(OpenRouterProvider, "_registry", None, raising=False) # Rebuild the provider registry with OpenRouter registered ModelProviderRegistry._instance = None from providers.shared import ProviderType ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) tool = AnalyzeTool() models = tool._get_available_models() assert "deepseek/deepseek-r1:free" in models, "Canonical free model name should be available" assert "deepseek-free" in models, "Free model alias should be included for MCP validation" # DELETED: test_auto_mode_behavior_with_environment_variables # This test was fundamentally broken due to registry corruption. # It cleared ModelProviderRegistry._instance without re-registering providers, # causing impossible test conditions (expecting models when no providers exist). # Functionality is already covered by test_auto_mode_comprehensive.py # DELETED: test_auto_mode_model_selection_validation # DELETED: test_environment_variable_precedence # Both tests suffered from the same registry corruption issue as the deleted test above. # They cleared ModelProviderRegistry._instance without re-registering providers, # causing empty model lists and impossible test conditions. # Auto mode functionality is already comprehensively tested in test_auto_mode_comprehensive.py ================================================ FILE: tests/test_model_metadata_continuation.py ================================================ """ Test model metadata preservation during conversation continuation. This test verifies that when using continuation_id without specifying a model, the system correctly retrieves and uses the model from the previous conversation turn instead of defaulting to DEFAULT_MODEL or the custom provider's default. Bug: https://github.com/BeehiveInnovations/pal-mcp-server/issues/111 """ from unittest.mock import MagicMock, patch import pytest from server import reconstruct_thread_context from utils.conversation_memory import add_turn, create_thread, get_thread from utils.model_context import ModelContext class TestModelMetadataContinuation: """Test model metadata preservation during conversation continuation.""" @pytest.mark.asyncio async def test_model_preserved_from_previous_turn(self): """Test that model is correctly retrieved from previous conversation turn.""" # Create a thread with a turn that has a specific model thread_id = create_thread("chat", {"prompt": "test"}) # Add an assistant turn with a specific model success = add_turn( thread_id, "assistant", "Here's my response", model_name="deepseek-r1-8b", model_provider="custom" ) assert success # Test continuation without model should use previous turn's model arguments = {"continuation_id": thread_id} # No model specified # Mock dependencies to avoid side effects with patch("utils.model_context.ModelContext.calculate_token_allocation") as mock_calc: mock_calc.return_value = MagicMock( total_tokens=200000, content_tokens=160000, response_tokens=40000, file_tokens=64000, history_tokens=64000, ) with patch("utils.conversation_memory.build_conversation_history") as mock_build: mock_build.return_value = ("=== CONVERSATION HISTORY ===\n", 1000) # Call the actual function enhanced_args = await reconstruct_thread_context(arguments) # Verify model was retrieved from thread assert enhanced_args.get("model") == "deepseek-r1-8b" # Verify ModelContext would use the correct model model_context = ModelContext.from_arguments(enhanced_args) assert model_context.model_name == "deepseek-r1-8b" @pytest.mark.asyncio async def test_reconstruct_thread_context_preserves_model(self): """Test that reconstruct_thread_context preserves model from previous turn.""" # Create thread with assistant turn thread_id = create_thread("chat", {"prompt": "initial"}) add_turn(thread_id, "assistant", "Initial response", model_name="o3-mini", model_provider="openai") # Test reconstruction without specifying model arguments = {"continuation_id": thread_id, "prompt": "follow-up question"} # Mock the model context to avoid initialization issues in tests with patch("utils.model_context.ModelContext.calculate_token_allocation") as mock_calc: mock_calc.return_value = MagicMock( total_tokens=200000, content_tokens=160000, response_tokens=40000, file_tokens=64000, history_tokens=64000, ) with patch("utils.conversation_memory.build_conversation_history") as mock_build: mock_build.return_value = ("=== CONVERSATION HISTORY ===\n", 1000) enhanced_args = await reconstruct_thread_context(arguments) # Verify model was retrieved from thread assert enhanced_args.get("model") == "o3-mini" @pytest.mark.asyncio async def test_multiple_turns_uses_last_assistant_model(self): """Test that with multiple turns, the last assistant turn's model is used.""" thread_id = create_thread("chat", {"prompt": "analyze this"}) # Add multiple turns with different models add_turn(thread_id, "assistant", "First response", model_name="gemini-2.5-flash", model_provider="google") add_turn(thread_id, "user", "Another question") add_turn(thread_id, "assistant", "Second response", model_name="o3", model_provider="openai") add_turn(thread_id, "user", "Final question") arguments = {"continuation_id": thread_id} # Mock dependencies with patch("utils.model_context.ModelContext.calculate_token_allocation") as mock_calc: mock_calc.return_value = MagicMock( total_tokens=200000, content_tokens=160000, response_tokens=40000, file_tokens=64000, history_tokens=64000, ) with patch("utils.conversation_memory.build_conversation_history") as mock_build: mock_build.return_value = ("=== CONVERSATION HISTORY ===\n", 1000) # Call the actual function enhanced_args = await reconstruct_thread_context(arguments) # Should use the most recent assistant model assert enhanced_args.get("model") == "o3" @pytest.mark.asyncio async def test_no_previous_assistant_turn_defaults(self): """Test behavior when there's no previous assistant turn.""" # Save and set DEFAULT_MODEL for test import importlib import os original_default = os.environ.get("DEFAULT_MODEL", "") os.environ["DEFAULT_MODEL"] = "auto" import config import utils.model_context importlib.reload(config) importlib.reload(utils.model_context) try: thread_id = create_thread("chat", {"prompt": "test"}) # Only add user turns add_turn(thread_id, "user", "First question") add_turn(thread_id, "user", "Second question") arguments = {"continuation_id": thread_id} # Mock dependencies with patch("utils.model_context.ModelContext.calculate_token_allocation") as mock_calc: mock_calc.return_value = MagicMock( total_tokens=200000, content_tokens=160000, response_tokens=40000, file_tokens=64000, history_tokens=64000, ) with patch("utils.conversation_memory.build_conversation_history") as mock_build: mock_build.return_value = ("=== CONVERSATION HISTORY ===\n", 1000) # Call the actual function enhanced_args = await reconstruct_thread_context(arguments) # Should not have set a model assert enhanced_args.get("model") is None # ModelContext should use DEFAULT_MODEL model_context = ModelContext.from_arguments(enhanced_args) from config import DEFAULT_MODEL assert model_context.model_name == DEFAULT_MODEL finally: # Restore original value if original_default: os.environ["DEFAULT_MODEL"] = original_default else: os.environ.pop("DEFAULT_MODEL", None) importlib.reload(config) importlib.reload(utils.model_context) @pytest.mark.asyncio async def test_explicit_model_overrides_previous_turn(self): """Test that explicitly specifying a model overrides the previous turn's model.""" thread_id = create_thread("chat", {"prompt": "test"}) add_turn(thread_id, "assistant", "Response", model_name="gemini-2.5-flash", model_provider="google") arguments = {"continuation_id": thread_id, "model": "o3"} # Explicitly specified # Mock dependencies with patch("utils.model_context.ModelContext.calculate_token_allocation") as mock_calc: mock_calc.return_value = MagicMock( total_tokens=200000, content_tokens=160000, response_tokens=40000, file_tokens=64000, history_tokens=64000, ) with patch("utils.conversation_memory.build_conversation_history") as mock_build: mock_build.return_value = ("=== CONVERSATION HISTORY ===\n", 1000) # Call the actual function enhanced_args = await reconstruct_thread_context(arguments) # Should keep the explicit model assert enhanced_args.get("model") == "o3" @pytest.mark.asyncio async def test_thread_chain_model_preservation(self): """Test model preservation across thread chains (parent-child relationships).""" # Create parent thread parent_id = create_thread("chat", {"prompt": "analyze"}) add_turn(parent_id, "assistant", "Analysis", model_name="gemini-2.5-pro", model_provider="google") # Create child thread using a simple tool instead of workflow tool child_id = create_thread("chat", {"prompt": "review"}, parent_thread_id=parent_id) # Child thread should be able to access parent's model through chain traversal # NOTE: Current implementation only checks current thread (not parent threads) context = get_thread(child_id) assert context.parent_thread_id == parent_id arguments = {"continuation_id": child_id} # Mock dependencies with patch("utils.model_context.ModelContext.calculate_token_allocation") as mock_calc: mock_calc.return_value = MagicMock( total_tokens=200000, content_tokens=160000, response_tokens=40000, file_tokens=64000, history_tokens=64000, ) with patch("utils.conversation_memory.build_conversation_history") as mock_build: mock_build.return_value = ("=== CONVERSATION HISTORY ===\n", 1000) # Call the actual function enhanced_args = await reconstruct_thread_context(arguments) # No turns in child thread yet, so model should not be set assert enhanced_args.get("model") is None ================================================ FILE: tests/test_model_resolution_bug.py ================================================ """ Test to reproduce and fix the OpenRouter model name resolution bug. This test specifically targets the bug where: 1. User specifies "gemini" in consensus tool 2. System incorrectly resolves to "gemini-2.5-pro" instead of "google/gemini-2.5-pro" 3. OpenRouter API returns "gemini-2.5-pro is not a valid model ID" """ from unittest.mock import Mock, patch from providers.openrouter import OpenRouterProvider from providers.shared import ProviderType from tools.consensus import ConsensusTool class TestModelResolutionBug: """Test cases for the OpenRouter model name resolution bug.""" def setup_method(self): """Setup test environment.""" self.consensus_tool = ConsensusTool() def test_openrouter_registry_resolves_gemini_alias(self): """Test that OpenRouter registry properly resolves 'gemini' to 'google/gemini-3-pro-preview'.""" # Test the registry directly provider = OpenRouterProvider("test_key") # Test alias resolution resolved_model_name = provider._resolve_model_name("gemini") assert ( resolved_model_name == "google/gemini-3-pro-preview" ), f"Expected 'google/gemini-3-pro-preview', got '{resolved_model_name}'" # Test that it also works with 'pro' alias resolved_pro = provider._resolve_model_name("pro") assert ( resolved_pro == "google/gemini-3-pro-preview" ), f"Expected 'google/gemini-3-pro-preview', got '{resolved_pro}'" # DELETED: test_provider_registry_returns_openrouter_for_gemini # This test had a flawed mock setup - it mocked get_provider() but called get_provider_for_model(). # The test was trying to verify OpenRouter model resolution functionality that is already # comprehensively tested in working OpenRouter provider tests. @patch.dict("os.environ", {"OPENROUTER_API_KEY": "test_key"}, clear=False) def test_consensus_tool_model_resolution_bug_reproduction(self): """Test that the new consensus workflow tool properly handles OpenRouter model resolution.""" import asyncio # Create a mock OpenRouter provider that tracks what model names it receives mock_provider = Mock(spec=OpenRouterProvider) mock_provider.get_provider_type.return_value = ProviderType.OPENROUTER # Mock response for successful generation mock_response = Mock() mock_response.content = "Test response" mock_response.usage = None mock_provider.generate_content.return_value = mock_response # Track the model name passed to generate_content received_model_names = [] def track_generate_content(*args, **kwargs): received_model_names.append(kwargs.get("model_name", args[1] if len(args) > 1 else "unknown")) return mock_response mock_provider.generate_content.side_effect = track_generate_content # Mock the get_model_provider to return our mock with patch.object(self.consensus_tool, "get_model_provider", return_value=mock_provider): # Set initial prompt self.consensus_tool.initial_prompt = "Test prompt" # Create a mock request request = Mock() request.relevant_files = [] request.continuation_id = None request.images = None # Test model consultation directly result = asyncio.run(self.consensus_tool._consult_model({"model": "gemini", "stance": "neutral"}, request)) # Verify that generate_content was called assert len(received_model_names) == 1 # The consensus tool should pass the original alias "gemini" # The OpenRouter provider should resolve it internally received_model = received_model_names[0] print(f"Model name passed to provider: {received_model}") assert received_model == "gemini", f"Expected 'gemini' to be passed to provider, got '{received_model}'" # Verify the result structure assert result["model"] == "gemini" assert result["status"] == "success" def test_bug_reproduction_with_malformed_model_name(self): """Test what happens when 'gemini-2.5-pro' (malformed) is passed to OpenRouter.""" provider = OpenRouterProvider("test_key") # This should NOT resolve because 'gemini-2.5-pro' is not in the OpenRouter registry resolved = provider._resolve_model_name("gemini-2.5-pro") # The bug: this returns "gemini-2.5-pro" as-is instead of resolving to proper name # This is what causes the OpenRouter API to fail assert resolved == "gemini-2.5-pro", f"Expected fallback to 'gemini-2.5-pro', got '{resolved}'" # Verify the registry doesn't have this malformed name config = provider._registry.resolve("gemini-2.5-pro") assert config is None, "Registry should not contain 'gemini-2.5-pro' - only 'google/gemini-2.5-pro'" if __name__ == "__main__": # Run the tests test = TestModelResolutionBug() test.setup_method() print("Testing OpenRouter registry resolution...") test.test_openrouter_registry_resolves_gemini_alias() print("✅ Registry resolves aliases correctly") print("\nTesting malformed model name handling...") test.test_bug_reproduction_with_malformed_model_name() print("✅ Confirmed: malformed names fall through as-is") print("\nConsensus tool test completed successfully.") print("\nAll tests completed. The bug is fixed.") ================================================ FILE: tests/test_model_restrictions.py ================================================ """Tests for model restriction functionality.""" import os from unittest.mock import MagicMock, patch import pytest from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.shared import ProviderType from utils.model_restrictions import ModelRestrictionService class TestModelRestrictionService: """Test cases for ModelRestrictionService.""" def test_no_restrictions_by_default(self): """Test that no restrictions exist when env vars are not set.""" with patch.dict(os.environ, {}, clear=True): service = ModelRestrictionService() # Should allow all models assert service.is_allowed(ProviderType.OPENAI, "o3") assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-pro") assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-flash") assert service.is_allowed(ProviderType.OPENROUTER, "anthropic/claude-opus-4") assert service.is_allowed(ProviderType.OPENROUTER, "openai/o3") # Should have no restrictions assert not service.has_restrictions(ProviderType.OPENAI) assert not service.has_restrictions(ProviderType.GOOGLE) assert not service.has_restrictions(ProviderType.OPENROUTER) def test_load_single_model_restriction(self): """Test loading a single allowed model.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini"}): service = ModelRestrictionService() # Should only allow o3-mini assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert not service.is_allowed(ProviderType.OPENAI, "o3") assert not service.is_allowed(ProviderType.OPENAI, "o4-mini") # Google and OpenRouter should have no restrictions assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-pro") assert service.is_allowed(ProviderType.OPENROUTER, "anthropic/claude-opus-4") def test_load_multiple_models_restriction(self): """Test loading multiple allowed models.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini", "GOOGLE_ALLOWED_MODELS": "flash,pro"}): # Instantiate providers so alias resolution for allow-lists is available openai_provider = OpenAIModelProvider(api_key="test-key") gemini_provider = GeminiModelProvider(api_key="test-key") from providers.registry import ModelProviderRegistry def fake_get_provider(provider_type, force_new=False): mapping = { ProviderType.OPENAI: openai_provider, ProviderType.GOOGLE: gemini_provider, } return mapping.get(provider_type) with patch.object(ModelProviderRegistry, "get_provider", side_effect=fake_get_provider): service = ModelRestrictionService() # Check OpenAI models assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert service.is_allowed(ProviderType.OPENAI, "o4-mini") assert not service.is_allowed(ProviderType.OPENAI, "o3") # Check Google models assert service.is_allowed(ProviderType.GOOGLE, "flash") assert service.is_allowed(ProviderType.GOOGLE, "pro") assert service.is_allowed(ProviderType.GOOGLE, "gemini-3-pro-preview") def test_case_insensitive_and_whitespace_handling(self): """Test that model names are case-insensitive and whitespace is trimmed.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": " O3-MINI , o4-Mini "}): service = ModelRestrictionService() # Should work with any case assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert service.is_allowed(ProviderType.OPENAI, "O3-MINI") assert service.is_allowed(ProviderType.OPENAI, "o4-mini") assert service.is_allowed(ProviderType.OPENAI, "O4-Mini") def test_empty_string_allows_all(self): """Test that empty string allows all models (same as unset).""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "", "GOOGLE_ALLOWED_MODELS": "flash"}): service = ModelRestrictionService() # OpenAI should allow all models (empty string = no restrictions) assert service.is_allowed(ProviderType.OPENAI, "o3") assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert service.is_allowed(ProviderType.OPENAI, "o4-mini") # Google should only allow flash (and its resolved name) assert service.is_allowed(ProviderType.GOOGLE, "flash") assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-flash", "flash") assert not service.is_allowed(ProviderType.GOOGLE, "pro") assert not service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-pro", "pro") def test_filter_models(self): """Test filtering a list of models based on restrictions.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini"}): service = ModelRestrictionService() models = ["o3", "o3-mini", "o4-mini", "o3-pro"] filtered = service.filter_models(ProviderType.OPENAI, models) assert filtered == ["o3-mini", "o4-mini"] def test_get_allowed_models(self): """Test getting the set of allowed models.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mini"}): service = ModelRestrictionService() allowed = service.get_allowed_models(ProviderType.OPENAI) assert allowed == {"o3-mini", "o4-mini"} # No restrictions for Google assert service.get_allowed_models(ProviderType.GOOGLE) is None def test_shorthand_names_in_restrictions(self): """Test that shorthand names work in restrictions.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4mini,o3mini", "GOOGLE_ALLOWED_MODELS": "flash,pro"}): # Instantiate providers so the registry can resolve aliases OpenAIModelProvider(api_key="test-key") GeminiModelProvider(api_key="test-key") service = ModelRestrictionService() # When providers check models, they pass both resolved and original names # OpenAI: 'o4mini' shorthand allows o4-mini assert service.is_allowed(ProviderType.OPENAI, "o4-mini", "o4mini") # How providers actually call it assert service.is_allowed(ProviderType.OPENAI, "o4-mini") # Canonical should also be allowed # OpenAI: o3-mini allowed directly assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert not service.is_allowed(ProviderType.OPENAI, "o3") # Google should allow both models via shorthands assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-flash", "flash") assert service.is_allowed(ProviderType.GOOGLE, "gemini-2.5-pro", "pro") # Also test that full names work when specified in restrictions assert service.is_allowed(ProviderType.OPENAI, "o3-mini", "o3mini") # Even with shorthand def test_validation_against_known_models(self, caplog): """Test validation warnings for unknown models.""" with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini,o4-mimi"}): # Note the typo: o4-mimi service = ModelRestrictionService() # Create mock provider with known models mock_provider = MagicMock() mock_provider.MODEL_CAPABILITIES = { "o3": {"context_window": 200000}, "o3-mini": {"context_window": 200000}, "o4-mini": {"context_window": 200000}, } mock_provider.list_models.return_value = ["o3", "o3-mini", "o4-mini"] provider_instances = {ProviderType.OPENAI: mock_provider} service.validate_against_known_models(provider_instances) # Should have logged a warning about the typo assert "o4-mimi" in caplog.text assert "not a recognized" in caplog.text def test_openrouter_model_restrictions(self): """Test OpenRouter model restrictions functionality.""" with patch.dict(os.environ, {"OPENROUTER_ALLOWED_MODELS": "opus,sonnet"}): service = ModelRestrictionService() # Should only allow specified OpenRouter models assert service.is_allowed(ProviderType.OPENROUTER, "opus") assert service.is_allowed(ProviderType.OPENROUTER, "sonnet") assert service.is_allowed(ProviderType.OPENROUTER, "anthropic/claude-opus-4", "opus") # With original name assert not service.is_allowed(ProviderType.OPENROUTER, "haiku") assert not service.is_allowed(ProviderType.OPENROUTER, "anthropic/claude-3-haiku") assert not service.is_allowed(ProviderType.OPENROUTER, "mistral-large") # Other providers should have no restrictions assert service.is_allowed(ProviderType.OPENAI, "o3") assert service.is_allowed(ProviderType.GOOGLE, "pro") # Should have restrictions for OpenRouter assert service.has_restrictions(ProviderType.OPENROUTER) assert not service.has_restrictions(ProviderType.OPENAI) assert not service.has_restrictions(ProviderType.GOOGLE) def test_openrouter_filter_models(self): """Test filtering OpenRouter models based on restrictions.""" with patch.dict(os.environ, {"OPENROUTER_ALLOWED_MODELS": "opus,mistral"}): service = ModelRestrictionService() models = ["opus", "sonnet", "haiku", "mistral", "llama"] filtered = service.filter_models(ProviderType.OPENROUTER, models) assert filtered == ["opus", "mistral"] def test_combined_provider_restrictions(self): """Test that restrictions work correctly when set for multiple providers.""" with patch.dict( os.environ, { "OPENAI_ALLOWED_MODELS": "o3-mini", "GOOGLE_ALLOWED_MODELS": "flash", "OPENROUTER_ALLOWED_MODELS": "opus,sonnet", }, ): service = ModelRestrictionService() # OpenAI restrictions assert service.is_allowed(ProviderType.OPENAI, "o3-mini") assert not service.is_allowed(ProviderType.OPENAI, "o3") # Google restrictions assert service.is_allowed(ProviderType.GOOGLE, "flash") assert not service.is_allowed(ProviderType.GOOGLE, "pro") # OpenRouter restrictions assert service.is_allowed(ProviderType.OPENROUTER, "opus") assert service.is_allowed(ProviderType.OPENROUTER, "sonnet") assert not service.is_allowed(ProviderType.OPENROUTER, "haiku") # All providers should have restrictions assert service.has_restrictions(ProviderType.OPENAI) assert service.has_restrictions(ProviderType.GOOGLE) assert service.has_restrictions(ProviderType.OPENROUTER) class TestProviderIntegration: """Test integration with actual providers.""" @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini"}) def test_openai_provider_respects_restrictions(self): """Test that OpenAI provider respects restrictions.""" # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = OpenAIModelProvider(api_key="test-key") # Should validate allowed model assert provider.validate_model_name("o3-mini") # Should not validate disallowed model assert not provider.validate_model_name("o3") # get_capabilities should raise for disallowed model with pytest.raises(ValueError) as exc_info: provider.get_capabilities("o3") assert "not allowed by restriction policy" in str(exc_info.value) @patch.dict(os.environ, {"GOOGLE_ALLOWED_MODELS": "gemini-2.5-flash,flash"}) def test_gemini_provider_respects_restrictions(self): """Test that Gemini provider respects restrictions.""" # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = GeminiModelProvider(api_key="test-key") # Should validate allowed models (both shorthand and full name allowed) assert provider.validate_model_name("flash") assert provider.validate_model_name("gemini-2.5-flash") # Should not validate disallowed model assert not provider.validate_model_name("pro") assert not provider.validate_model_name("gemini-2.5-pro") # get_capabilities should raise for disallowed model with pytest.raises(ValueError) as exc_info: provider.get_capabilities("pro") assert "not allowed by restriction policy" in str(exc_info.value) @patch.dict(os.environ, {"GOOGLE_ALLOWED_MODELS": "flash"}) def test_gemini_parameter_order_regression_protection(self): """Test that prevents regression of parameter order bug in is_allowed calls. This test specifically catches the bug where parameters were incorrectly passed as (provider, user_input, resolved_name) instead of (provider, resolved_name, user_input). The bug was subtle because the is_allowed method uses OR logic, so it worked in most cases by accident. This test creates a scenario where the parameter order matters. """ # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = GeminiModelProvider(api_key="test-key") from providers.registry import ModelProviderRegistry with patch.object(ModelProviderRegistry, "get_provider", return_value=provider): # Test case: Only alias "flash" is allowed, not the full name # If parameters are in wrong order, this test will catch it # Should allow "flash" alias assert provider.validate_model_name("flash") # Should allow getting capabilities for "flash" capabilities = provider.get_capabilities("flash") assert capabilities.model_name == "gemini-2.5-flash" # Canonical form should also be allowed now that alias is on the allowlist assert provider.validate_model_name("gemini-2.5-flash") # Unrelated models remain blocked assert not provider.validate_model_name("pro") assert not provider.validate_model_name("gemini-2.5-pro") @patch.dict(os.environ, {"GOOGLE_ALLOWED_MODELS": "gemini-2.5-flash"}) def test_gemini_parameter_order_edge_case_full_name_only(self): """Test parameter order with only full name allowed, not alias. This is the reverse scenario - only the full canonical name is allowed, not the shorthand alias. This tests that the parameter order is correct when resolving aliases. """ # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = GeminiModelProvider(api_key="test-key") # Should allow full name assert provider.validate_model_name("gemini-2.5-flash") # Should also allow alias that resolves to allowed full name # This works because is_allowed checks both resolved_name and original_name assert provider.validate_model_name("flash") # Should not allow "pro" alias assert not provider.validate_model_name("pro") assert not provider.validate_model_name("gemini-2.5-pro") class TestCustomProviderOpenRouterRestrictions: """Test custom provider integration with OpenRouter restrictions.""" @patch.dict(os.environ, {"OPENROUTER_ALLOWED_MODELS": "opus,sonnet", "OPENROUTER_API_KEY": "test-key"}) def test_custom_provider_respects_openrouter_restrictions(self): """Test that custom provider correctly defers OpenRouter models to OpenRouter provider.""" # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None from providers.custom import CustomProvider provider = CustomProvider(base_url="http://test.com/v1") # CustomProvider should NOT validate OpenRouter models - they should be deferred to OpenRouter assert not provider.validate_model_name("opus") assert not provider.validate_model_name("sonnet") assert not provider.validate_model_name("haiku") # Should still validate custom models defined in conf/custom_models.json assert provider.validate_model_name("local-llama") @patch.dict(os.environ, {"OPENROUTER_ALLOWED_MODELS": "opus", "OPENROUTER_API_KEY": "test-key"}) def test_custom_provider_openrouter_capabilities_restrictions(self): """Test that custom provider's get_capabilities correctly handles OpenRouter models.""" # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None from providers.custom import CustomProvider provider = CustomProvider(base_url="http://test.com/v1") # For OpenRouter models, CustomProvider should defer by raising with pytest.raises(ValueError): provider.get_capabilities("opus") # Should raise for disallowed OpenRouter model (still defers) with pytest.raises(ValueError): provider.get_capabilities("haiku") # Should still work for custom models capabilities = provider.get_capabilities("local-llama") assert capabilities.provider == ProviderType.CUSTOM @patch.dict(os.environ, {"OPENROUTER_ALLOWED_MODELS": "opus"}, clear=False) def test_custom_provider_no_openrouter_key_ignores_restrictions(self): """Test that when OpenRouter key is not set, cloud models are rejected regardless of restrictions.""" # Make sure OPENROUTER_API_KEY is not set if "OPENROUTER_API_KEY" in os.environ: del os.environ["OPENROUTER_API_KEY"] # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None from providers.custom import CustomProvider provider = CustomProvider(base_url="http://test.com/v1") # Should not validate OpenRouter models when key is not available assert not provider.validate_model_name("opus") # Even though it's in allowed list assert not provider.validate_model_name("haiku") # Should still validate custom models assert provider.validate_model_name("local-llama") @patch.dict(os.environ, {"OPENROUTER_ALLOWED_MODELS": "", "OPENROUTER_API_KEY": "test-key"}) def test_custom_provider_empty_restrictions_allows_all_openrouter(self): """Test that custom provider correctly defers OpenRouter models regardless of restrictions.""" # Clear any cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None from providers.custom import CustomProvider provider = CustomProvider(base_url="http://test.com/v1") # CustomProvider should NOT validate OpenRouter models - they should be deferred to OpenRouter assert not provider.validate_model_name("opus") assert not provider.validate_model_name("sonnet") assert not provider.validate_model_name("haiku") class TestRegistryIntegration: """Test integration with ModelProviderRegistry.""" @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "mini", "GOOGLE_ALLOWED_MODELS": "flash"}) def test_registry_with_shorthand_restrictions(self): """Test that registry handles shorthand restrictions correctly.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None from providers.registry import ModelProviderRegistry # Clear registry cache ModelProviderRegistry.clear_cache() # Get available models with restrictions # This test documents current behavior - get_available_models doesn't handle aliases ModelProviderRegistry.get_available_models(respect_restrictions=True) # Currently, this will be empty because get_available_models doesn't # recognize that "mini" allows "o4-mini" # This is a known limitation that should be documented @patch("providers.registry.ModelProviderRegistry.get_provider") def test_get_available_models_respects_restrictions(self, mock_get_provider): """Test that registry filters models based on restrictions.""" from providers.registry import ModelProviderRegistry # Mock providers mock_openai = MagicMock() mock_openai.MODEL_CAPABILITIES = { "o3": {"context_window": 200000}, "o3-mini": {"context_window": 200000}, } mock_openai.get_provider_type.return_value = ProviderType.OPENAI def openai_list_models( *, respect_restrictions: bool = True, include_aliases: bool = True, lowercase: bool = False, unique: bool = False, ): from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if respect_restrictions else None models = [] for model_name, config in mock_openai.MODEL_CAPABILITIES.items(): if isinstance(config, str): target_model = config if restriction_service and not restriction_service.is_allowed(ProviderType.OPENAI, target_model): continue if include_aliases: models.append(model_name) else: if restriction_service and not restriction_service.is_allowed(ProviderType.OPENAI, model_name): continue models.append(model_name) if lowercase: models = [m.lower() for m in models] if unique: seen = set() ordered = [] for name in models: if name in seen: continue seen.add(name) ordered.append(name) models = ordered return models mock_openai.list_models = MagicMock(side_effect=openai_list_models) mock_gemini = MagicMock() mock_gemini.MODEL_CAPABILITIES = { "gemini-2.5-pro": {"context_window": 1048576}, "gemini-2.5-flash": {"context_window": 1048576}, } mock_gemini.get_provider_type.return_value = ProviderType.GOOGLE def gemini_list_models( *, respect_restrictions: bool = True, include_aliases: bool = True, lowercase: bool = False, unique: bool = False, ): from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if respect_restrictions else None models = [] for model_name, config in mock_gemini.MODEL_CAPABILITIES.items(): if isinstance(config, str): target_model = config if restriction_service and not restriction_service.is_allowed(ProviderType.GOOGLE, target_model): continue if include_aliases: models.append(model_name) else: if restriction_service and not restriction_service.is_allowed(ProviderType.GOOGLE, model_name): continue models.append(model_name) if lowercase: models = [m.lower() for m in models] if unique: seen = set() ordered = [] for name in models: if name in seen: continue seen.add(name) ordered.append(name) models = ordered return models mock_gemini.list_models = MagicMock(side_effect=gemini_list_models) def get_provider_side_effect(provider_type): if provider_type == ProviderType.OPENAI: return mock_openai elif provider_type == ProviderType.GOOGLE: return mock_gemini return None mock_get_provider.side_effect = get_provider_side_effect # Set up registry with providers registry = ModelProviderRegistry() registry._providers = { ProviderType.OPENAI: type(mock_openai), ProviderType.GOOGLE: type(mock_gemini), } with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-mini", "GOOGLE_ALLOWED_MODELS": "gemini-2.5-flash"}): # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None available = ModelProviderRegistry.get_available_models(respect_restrictions=True) # Should only include allowed models assert "o3-mini" in available assert "o3" not in available assert "gemini-2.5-flash" in available assert "gemini-2.5-pro" not in available class TestShorthandRestrictions: """Test that shorthand model names work correctly in restrictions.""" @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "mini", "GOOGLE_ALLOWED_MODELS": "flash"}) def test_providers_validate_shorthands_correctly(self): """Test that providers correctly validate shorthand names.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None # Test OpenAI provider openai_provider = OpenAIModelProvider(api_key="test-key") gemini_provider = GeminiModelProvider(api_key="test-key") from providers.registry import ModelProviderRegistry def registry_side_effect(provider_type, force_new=False): mapping = { ProviderType.OPENAI: openai_provider, ProviderType.GOOGLE: gemini_provider, } return mapping.get(provider_type) with patch.object(ModelProviderRegistry, "get_provider", side_effect=registry_side_effect): assert openai_provider.validate_model_name("mini") # Should work with shorthand assert openai_provider.validate_model_name("gpt-5-mini") # Canonical resolved from shorthand assert not openai_provider.validate_model_name("o4-mini") # Unrelated model still blocked assert not openai_provider.validate_model_name("o3-mini") # Test Gemini provider assert gemini_provider.validate_model_name("flash") # Should work with shorthand assert gemini_provider.validate_model_name("gemini-2.5-flash") # Canonical allowed assert not gemini_provider.validate_model_name("pro") # Not allowed @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3mini,mini,o4-mini"}) def test_multiple_shorthands_for_same_model(self): """Test that multiple shorthands work correctly.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None openai_provider = OpenAIModelProvider(api_key="test-key") # Both shorthands should work assert openai_provider.validate_model_name("mini") # mini -> o4-mini assert openai_provider.validate_model_name("o3mini") # o3mini -> o3-mini # Resolved names should be allowed when their shorthands are present assert openai_provider.validate_model_name("o4-mini") # Explicitly allowed assert openai_provider.validate_model_name("o3-mini") # Allowed via shorthand # Other models should not work assert not openai_provider.validate_model_name("o3") assert not openai_provider.validate_model_name("o3-pro") @patch.dict( os.environ, {"OPENAI_ALLOWED_MODELS": "mini,o4-mini", "GOOGLE_ALLOWED_MODELS": "flash,gemini-2.5-flash"}, ) def test_both_shorthand_and_full_name_allowed(self): """Test that we can allow both shorthand and full names.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None # OpenAI - both mini and o4-mini are allowed openai_provider = OpenAIModelProvider(api_key="test-key") assert openai_provider.validate_model_name("mini") assert openai_provider.validate_model_name("o4-mini") # Gemini - both flash and full name are allowed gemini_provider = GeminiModelProvider(api_key="test-key") assert gemini_provider.validate_model_name("flash") assert gemini_provider.validate_model_name("gemini-2.5-flash") class TestAutoModeWithRestrictions: """Test auto mode behavior with restrictions.""" @patch("providers.registry.ModelProviderRegistry.get_provider") def test_fallback_model_respects_restrictions(self, mock_get_provider): """Test that fallback model selection respects restrictions.""" from providers.registry import ModelProviderRegistry from tools.models import ToolModelCategory # Mock providers mock_openai = MagicMock() mock_openai.MODEL_CAPABILITIES = { "o3": {"context_window": 200000}, "o3-mini": {"context_window": 200000}, "o4-mini": {"context_window": 200000}, } mock_openai.get_provider_type.return_value = ProviderType.OPENAI def openai_list_models( *, respect_restrictions: bool = True, include_aliases: bool = True, lowercase: bool = False, unique: bool = False, ): from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if respect_restrictions else None models = [] for model_name, config in mock_openai.MODEL_CAPABILITIES.items(): if isinstance(config, str): target_model = config if restriction_service and not restriction_service.is_allowed(ProviderType.OPENAI, target_model): continue if include_aliases: models.append(model_name) else: if restriction_service and not restriction_service.is_allowed(ProviderType.OPENAI, model_name): continue models.append(model_name) if lowercase: models = [m.lower() for m in models] if unique: seen = set() ordered = [] for name in models: if name in seen: continue seen.add(name) ordered.append(name) models = ordered return models mock_openai.list_models = MagicMock(side_effect=openai_list_models) # Add get_preferred_model method to mock to match new implementation def get_preferred_model(category, allowed_models): # Simple preference logic for testing - just return first allowed model return allowed_models[0] if allowed_models else None mock_openai.get_preferred_model = get_preferred_model def get_provider_side_effect(provider_type): if provider_type == ProviderType.OPENAI: return mock_openai return None mock_get_provider.side_effect = get_provider_side_effect # Set up registry registry = ModelProviderRegistry() registry._providers = {ProviderType.OPENAI: type(mock_openai)} with patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o4-mini"}): # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None # Should pick o4-mini instead of o3-mini for fast response model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) assert model == "o4-mini" def test_fallback_with_shorthand_restrictions(self, monkeypatch): """Test fallback model selection with shorthand restrictions.""" # Use monkeypatch to set environment variables with automatic cleanup monkeypatch.setenv("OPENAI_ALLOWED_MODELS", "mini") monkeypatch.setenv("GEMINI_API_KEY", "") monkeypatch.setenv("OPENAI_API_KEY", "test-key") # Clear caches and reset registry import utils.model_restrictions from providers.registry import ModelProviderRegistry from tools.models import ToolModelCategory utils.model_restrictions._restriction_service = None # Store original providers for restoration registry = ModelProviderRegistry() original_providers = registry._providers.copy() original_initialized = registry._initialized_providers.copy() try: # Clear registry and register only OpenAI and Gemini providers ModelProviderRegistry._instance = None from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) # Even with "mini" restriction, fallback should work if provider handles it correctly # This tests the real-world scenario model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) # The fallback will depend on how get_available_models handles aliases # When "mini" is allowed, it's returned as the allowed model # "mini" is now an alias for gpt-5-mini, but the list shows "mini" itself assert model in ["mini", "gpt-5-mini", "o4-mini", "gemini-2.5-flash"] finally: # Restore original registry state registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() registry._providers.update(original_providers) registry._initialized_providers.update(original_initialized) ================================================ FILE: tests/test_o3_pro_output_text_fix.py ================================================ """ Tests for o3-pro output_text parsing fix using HTTP transport recording. This test validates the fix that uses `response.output_text` convenience field instead of manually parsing `response.output.content[].text`. Uses HTTP transport recorder to record real o3-pro API responses at the HTTP level while allowing the OpenAI SDK to create real response objects that we can test. RECORDING: To record new responses, delete the cassette file and run with real API keys. """ import logging import os import tempfile from pathlib import Path from unittest.mock import patch import pytest from dotenv import load_dotenv from providers import ModelProviderRegistry from tests.transport_helpers import inject_transport from tools.chat import ChatTool logger = logging.getLogger(__name__) # Load environment variables from .env file load_dotenv() # Use absolute path for cassette directory cassette_dir = Path(__file__).parent / "openai_cassettes" cassette_dir.mkdir(exist_ok=True) @pytest.mark.asyncio class TestO3ProOutputTextFix: """Test o3-pro response parsing fix using respx for HTTP recording/replay.""" def setup_method(self): """Set up the test by ensuring clean registry state.""" # Use the new public API for registry cleanup ModelProviderRegistry.reset_for_testing() # Provider registration is now handled by inject_transport helper # Clear restriction service to ensure it re-reads environment # This is necessary because previous tests may have set restrictions # that are cached in the singleton import utils.model_restrictions utils.model_restrictions._restriction_service = None def teardown_method(self): """Clean up after test to ensure no state pollution.""" # Use the new public API for registry cleanup ModelProviderRegistry.reset_for_testing() @pytest.mark.no_mock_provider # Disable provider mocking for this test @patch.dict(os.environ, {"OPENAI_ALLOWED_MODELS": "o3-pro", "LOCALE": ""}) async def test_o3_pro_uses_output_text_field(self, monkeypatch): """Test that o3-pro parsing uses the output_text convenience field via ChatTool.""" cassette_path = cassette_dir / "o3_pro_basic_math.json" # Check if we need to record or replay if not cassette_path.exists(): # Recording mode - check for real API key real_api_key = os.getenv("OPENAI_API_KEY", "").strip() if not real_api_key or real_api_key.startswith("dummy"): pytest.fail( f"Cassette file not found at {cassette_path}. " "To record: Set OPENAI_API_KEY environment variable to a valid key and run this test. " "Note: Recording will make a real API call to OpenAI." ) # Real API key is available, we'll record the cassette logger.debug("🎬 Recording mode: Using real API key to record cassette") else: # Replay mode - use dummy key monkeypatch.setenv("OPENAI_API_KEY", "dummy-key-for-replay") logger.debug("📼 Replay mode: Using recorded cassette") # Simplified transport injection - just one line! inject_transport(monkeypatch, cassette_path) # Execute ChatTool test with custom transport result = await self._execute_chat_tool_test() # Verify the response works correctly self._verify_chat_tool_response(result) # Verify cassette exists assert cassette_path.exists() async def _execute_chat_tool_test(self): """Execute the ChatTool with o3-pro and return the result.""" chat_tool = ChatTool() with tempfile.TemporaryDirectory() as workdir: arguments = { "prompt": "What is 2 + 2?", "model": "o3-pro", "temperature": 1.0, "working_directory_absolute_path": workdir, } return await chat_tool.execute(arguments) def _verify_chat_tool_response(self, result): """Verify the ChatTool response contains expected data.""" # Basic response validation assert result is not None assert isinstance(result, list) assert len(result) > 0 assert result[0].type == "text" # Parse JSON response import json response_data = json.loads(result[0].text) # Debug log the response logger.debug(f"Response data: {json.dumps(response_data, indent=2)}") # Verify response structure - no cargo culting if response_data["status"] == "error": pytest.fail(f"Chat tool returned error: {response_data.get('error', 'Unknown error')}") assert response_data["status"] in ["success", "continuation_available"] assert "4" in response_data["content"] # Verify o3-pro was actually used metadata = response_data["metadata"] assert metadata["model_used"] == "o3-pro" assert metadata["provider_used"] == "openai" ================================================ FILE: tests/test_o3_temperature_fix_simple.py ================================================ """ Simple integration test for the O3 model temperature parameter fix. This test confirms that the fix properly excludes temperature parameters for O3 models while maintaining them for regular models. """ from unittest.mock import Mock, patch from providers.openai import OpenAIModelProvider class TestO3TemperatureParameterFixSimple: """Simple test for O3 model parameter filtering.""" @patch("utils.model_restrictions.get_restriction_service") @patch("providers.openai_compatible.OpenAI") def test_o3_models_exclude_temperature_from_api_call(self, mock_openai_class, mock_restriction_service): """Test that O3 models don't send temperature to the API.""" # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service # Setup mock client mock_client = Mock() mock_openai_class.return_value = mock_client # Setup mock response mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "o3-mini" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = Mock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response # Create provider provider = OpenAIModelProvider(api_key="test-key") # Override _resolve_model_name to return the resolved model name provider._resolve_model_name = lambda name: name # Override model validation to bypass restrictions provider.validate_model_name = lambda name: True # Call generate_content with O3 model provider.generate_content(prompt="Test prompt", model_name="o3-mini", temperature=0.5, max_output_tokens=100) # Verify the API call was made without temperature or max_tokens mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] assert "temperature" not in call_kwargs, "O3 models should not include temperature parameter" assert "max_tokens" not in call_kwargs, "O3 models should not include max_tokens parameter" assert call_kwargs["model"] == "o3-mini" assert "messages" in call_kwargs @patch("utils.model_restrictions.get_restriction_service") @patch("providers.openai_compatible.OpenAI") def test_regular_models_include_temperature_in_api_call(self, mock_openai_class, mock_restriction_service): """Test that regular models still send temperature to the API.""" # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service # Setup mock client mock_client = Mock() mock_openai_class.return_value = mock_client # Setup mock response mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "gpt-4.1-2025-04-14" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = Mock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response # Create provider provider = OpenAIModelProvider(api_key="test-key") # Override _resolve_model_name to return the resolved model name provider._resolve_model_name = lambda name: name # Override model validation to bypass restrictions provider.validate_model_name = lambda name: True # Call generate_content with regular model (use supported model) provider.generate_content( prompt="Test prompt", model_name="gpt-4.1-2025-04-14", temperature=0.5, max_output_tokens=100 ) # Verify the API call was made WITH temperature and max_tokens mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["temperature"] == 0.5, "Regular models should include temperature parameter" assert call_kwargs["max_tokens"] == 100, "Regular models should include max_tokens parameter" assert call_kwargs["model"] == "gpt-4.1-2025-04-14" @patch("utils.model_restrictions.get_restriction_service") @patch("providers.openai_compatible.OpenAI") def test_o3_models_filter_unsupported_parameters(self, mock_openai_class, mock_restriction_service): """Test that O3 models filter out top_p, frequency_penalty, etc.""" # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service # Setup mock client mock_client = Mock() mock_openai_class.return_value = mock_client # Setup mock response mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "o3" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = Mock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response # Create provider provider = OpenAIModelProvider(api_key="test-key") # Override _resolve_model_name to return the resolved model name provider._resolve_model_name = lambda name: name # Override model validation to bypass restrictions provider.validate_model_name = lambda name: True # Call generate_content with O3 model and unsupported parameters provider.generate_content( prompt="Test prompt", model_name="o3", temperature=0.5, top_p=0.9, frequency_penalty=0.1, presence_penalty=0.1, seed=42, stop=["END"], ) # Verify the API call filters out unsupported parameters mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] # Should be excluded for O3 models assert "temperature" not in call_kwargs, "O3 should not include temperature" assert "top_p" not in call_kwargs, "O3 should not include top_p" assert "frequency_penalty" not in call_kwargs, "O3 should not include frequency_penalty" assert "presence_penalty" not in call_kwargs, "O3 should not include presence_penalty" # Should be included (supported parameters) assert call_kwargs["seed"] == 42, "O3 should include seed parameter" assert call_kwargs["stop"] == ["END"], "O3 should include stop parameter" @patch("utils.model_restrictions.get_restriction_service") def test_all_o3_models_have_correct_temperature_capability(self, mock_restriction_service): """Test that all O3/O4 models have supports_temperature=False in their capabilities.""" from providers.openai import OpenAIModelProvider # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service provider = OpenAIModelProvider(api_key="test-key") # Test O3/O4 models that should NOT support temperature parameter o3_o4_models = ["o3", "o3-mini", "o3-pro", "o4-mini"] for model in o3_o4_models: capabilities = provider.get_capabilities(model) assert hasattr( capabilities, "supports_temperature" ), f"Model {model} capabilities should have supports_temperature field" assert capabilities.supports_temperature is False, f"Model {model} should have supports_temperature=False" # Test that regular models DO support temperature parameter regular_models = ["gpt-4.1-2025-04-14"] for model in regular_models: try: capabilities = provider.get_capabilities(model) assert hasattr( capabilities, "supports_temperature" ), f"Model {model} capabilities should have supports_temperature field" assert capabilities.supports_temperature is True, f"Model {model} should have supports_temperature=True" except ValueError: # Skip if model not in MODEL_CAPABILITIES (that's okay for this test) pass @patch("utils.model_restrictions.get_restriction_service") def test_openai_provider_temperature_constraints(self, mock_restriction_service): """Test that OpenAI provider has correct temperature constraints for O3 models.""" from providers.openai import OpenAIModelProvider # Mock restriction service to allow all models mock_service = Mock() mock_service.is_allowed.return_value = True mock_restriction_service.return_value = mock_service provider = OpenAIModelProvider(api_key="test-key") # Test O3 model constraints o3_capabilities = provider.get_capabilities("o3-mini") assert o3_capabilities.temperature_constraint is not None # O3 models should have fixed temperature constraint temp_constraint = o3_capabilities.temperature_constraint assert temp_constraint.validate(1.0) is True assert temp_constraint.validate(0.5) is False # Test regular model constraints - use gpt-4.1 which is supported gpt41_capabilities = provider.get_capabilities("gpt-4.1") assert gpt41_capabilities.temperature_constraint is not None # Regular models should allow a range temp_constraint = gpt41_capabilities.temperature_constraint assert temp_constraint.validate(0.5) is True assert temp_constraint.validate(1.0) is True ================================================ FILE: tests/test_openai_compatible_token_usage.py ================================================ """Tests for OpenAI-compatible provider token usage extraction.""" import unittest from unittest.mock import Mock from providers.openai_compatible import OpenAICompatibleProvider class TestOpenAICompatibleTokenUsage(unittest.TestCase): """Test OpenAI-compatible provider token usage handling.""" def setUp(self): """Set up test fixtures.""" # Create a concrete implementation for testing class TestProvider(OpenAICompatibleProvider): FRIENDLY_NAME = "Test" MODEL_CAPABILITIES = {"test-model": {"context_window": 4096}} def get_capabilities(self, model_name): return Mock() def get_provider_type(self): return Mock() def validate_model_name(self, model_name): return True def list_models(self, **kwargs): return ["test-model"] self.provider = TestProvider("test-key") def test_extract_usage_with_valid_tokens(self): """Test token extraction with valid token counts.""" response = Mock() response.usage = Mock() response.usage.prompt_tokens = 100 response.usage.completion_tokens = 50 response.usage.total_tokens = 150 usage = self.provider._extract_usage(response) self.assertEqual(usage["input_tokens"], 100) self.assertEqual(usage["output_tokens"], 50) self.assertEqual(usage["total_tokens"], 150) def test_extract_usage_with_none_prompt_tokens(self): """Test token extraction when prompt_tokens is None (regression test for bug).""" response = Mock() response.usage = Mock() response.usage.prompt_tokens = None # This was causing crashes response.usage.completion_tokens = 50 response.usage.total_tokens = None usage = self.provider._extract_usage(response) # Should default to 0 when None self.assertEqual(usage["input_tokens"], 0) self.assertEqual(usage["output_tokens"], 50) self.assertEqual(usage["total_tokens"], 0) def test_extract_usage_with_none_completion_tokens(self): """Test token extraction when completion_tokens is None (regression test for bug).""" response = Mock() response.usage = Mock() response.usage.prompt_tokens = 100 response.usage.completion_tokens = None # This was causing crashes response.usage.total_tokens = None usage = self.provider._extract_usage(response) self.assertEqual(usage["input_tokens"], 100) # Should default to 0 when None self.assertEqual(usage["output_tokens"], 0) self.assertEqual(usage["total_tokens"], 0) def test_extract_usage_with_all_none_tokens(self): """Test token extraction when all token counts are None.""" response = Mock() response.usage = Mock() response.usage.prompt_tokens = None response.usage.completion_tokens = None response.usage.total_tokens = None usage = self.provider._extract_usage(response) # Should default to 0 for all when None self.assertEqual(usage["input_tokens"], 0) self.assertEqual(usage["output_tokens"], 0) self.assertEqual(usage["total_tokens"], 0) def test_extract_usage_without_usage(self): """Test token extraction when response has no usage.""" response = Mock(spec=[]) # No usage attribute usage = self.provider._extract_usage(response) # Should return empty dict self.assertEqual(usage, {}) def test_extract_usage_with_zero_tokens(self): """Test token extraction with zero token counts.""" response = Mock() response.usage = Mock() response.usage.prompt_tokens = 0 response.usage.completion_tokens = 0 response.usage.total_tokens = 0 usage = self.provider._extract_usage(response) self.assertEqual(usage["input_tokens"], 0) self.assertEqual(usage["output_tokens"], 0) self.assertEqual(usage["total_tokens"], 0) def test_alternative_token_format_with_none(self): """Test alternative token format (input_tokens/output_tokens) with None values.""" # This tests the other code path in generate_content_openai_responses # Simulate a response with input_tokens/output_tokens attributes that could be None response = Mock() response.input_tokens = None # This was causing crashes response.output_tokens = 50 # Test the pattern: getattr(response, "input_tokens", 0) or 0 input_tokens = getattr(response, "input_tokens", 0) or 0 output_tokens = getattr(response, "output_tokens", 0) or 0 # Should not crash and should handle None gracefully self.assertEqual(input_tokens, 0) self.assertEqual(output_tokens, 50) # Test that addition works total = input_tokens + output_tokens self.assertEqual(total, 50) if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_openai_provider.py ================================================ """Tests for OpenAI provider implementation.""" import os from unittest.mock import MagicMock, patch from providers.openai import OpenAIModelProvider from providers.shared import ProviderType class TestOpenAIProvider: """Test OpenAI provider functionality.""" def setup_method(self): """Set up clean state before each test.""" # Clear restriction service cache before each test import utils.model_restrictions utils.model_restrictions._restriction_service = None def teardown_method(self): """Clean up after each test to avoid singleton issues.""" # Clear restriction service cache after each test import utils.model_restrictions utils.model_restrictions._restriction_service = None @patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}) def test_initialization(self): """Test provider initialization.""" provider = OpenAIModelProvider("test-key") assert provider.api_key == "test-key" assert provider.get_provider_type() == ProviderType.OPENAI assert provider.base_url == "https://api.openai.com/v1" def test_initialization_with_custom_url(self): """Test provider initialization with custom base URL.""" provider = OpenAIModelProvider("test-key", base_url="https://custom.openai.com/v1") assert provider.api_key == "test-key" assert provider.base_url == "https://custom.openai.com/v1" def test_model_validation(self): """Test model name validation.""" provider = OpenAIModelProvider("test-key") # Test valid models assert provider.validate_model_name("o3") is True assert provider.validate_model_name("o3-mini") is True assert provider.validate_model_name("o3-pro") is True assert provider.validate_model_name("o4-mini") is True assert provider.validate_model_name("o4-mini") is True assert provider.validate_model_name("gpt-5") is True assert provider.validate_model_name("gpt-5-mini") is True assert provider.validate_model_name("gpt-5.2") is True assert provider.validate_model_name("gpt-5.1-codex") is True assert provider.validate_model_name("gpt-5.1-codex-mini") is True # Test valid aliases assert provider.validate_model_name("mini") is True assert provider.validate_model_name("o3mini") is True assert provider.validate_model_name("o4mini") is True assert provider.validate_model_name("o4mini") is True assert provider.validate_model_name("gpt5") is True assert provider.validate_model_name("gpt5-mini") is True assert provider.validate_model_name("gpt5mini") is True assert provider.validate_model_name("gpt5.2") is True assert provider.validate_model_name("gpt5.1") is True assert provider.validate_model_name("gpt5.1-codex") is True assert provider.validate_model_name("codex-mini") is True # Test invalid model assert provider.validate_model_name("invalid-model") is False assert provider.validate_model_name("gpt-4") is False assert provider.validate_model_name("gemini-pro") is False def test_resolve_model_name(self): """Test model name resolution.""" provider = OpenAIModelProvider("test-key") # Test shorthand resolution assert provider._resolve_model_name("mini") == "gpt-5-mini" # "mini" now resolves to gpt-5-mini assert provider._resolve_model_name("o3mini") == "o3-mini" assert provider._resolve_model_name("o4mini") == "o4-mini" assert provider._resolve_model_name("o4mini") == "o4-mini" assert provider._resolve_model_name("gpt5") == "gpt-5" assert provider._resolve_model_name("gpt5-mini") == "gpt-5-mini" assert provider._resolve_model_name("gpt5mini") == "gpt-5-mini" assert provider._resolve_model_name("gpt5.2") == "gpt-5.2" assert provider._resolve_model_name("gpt5.1") == "gpt-5.2" assert provider._resolve_model_name("gpt5.1-codex") == "gpt-5.1-codex" assert provider._resolve_model_name("codex-mini") == "gpt-5.1-codex-mini" # Test full name passthrough assert provider._resolve_model_name("o3") == "o3" assert provider._resolve_model_name("o3-mini") == "o3-mini" assert provider._resolve_model_name("o3-pro") == "o3-pro" assert provider._resolve_model_name("o4-mini") == "o4-mini" assert provider._resolve_model_name("o4-mini") == "o4-mini" assert provider._resolve_model_name("gpt-5") == "gpt-5" assert provider._resolve_model_name("gpt-5-mini") == "gpt-5-mini" assert provider._resolve_model_name("gpt-5.2") == "gpt-5.2" assert provider._resolve_model_name("gpt-5.1") == "gpt-5.2" assert provider._resolve_model_name("gpt-5.1-codex") == "gpt-5.1-codex" assert provider._resolve_model_name("gpt-5.1-codex-mini") == "gpt-5.1-codex-mini" def test_get_capabilities_o3(self): """Test getting model capabilities for O3.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("o3") assert capabilities.model_name == "o3" # Should NOT be resolved in capabilities assert capabilities.friendly_name == "OpenAI (O3)" assert capabilities.context_window == 200_000 assert capabilities.provider == ProviderType.OPENAI assert not capabilities.supports_extended_thinking assert capabilities.supports_system_prompts is True assert capabilities.supports_streaming is True assert capabilities.supports_function_calling is True # Test temperature constraint (O3 has fixed temperature) assert capabilities.temperature_constraint.value == 1.0 def test_get_capabilities_with_alias(self): """Test getting model capabilities with alias resolves correctly.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("mini") assert capabilities.model_name == "gpt-5-mini" # "mini" now resolves to gpt-5-mini assert capabilities.friendly_name == "OpenAI (GPT-5-mini)" assert capabilities.context_window == 400_000 assert capabilities.provider == ProviderType.OPENAI def test_get_capabilities_gpt5(self): """Test getting model capabilities for GPT-5.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("gpt-5") assert capabilities.model_name == "gpt-5" assert capabilities.friendly_name == "OpenAI (GPT-5)" assert capabilities.context_window == 400_000 assert capabilities.max_output_tokens == 128_000 assert capabilities.provider == ProviderType.OPENAI assert capabilities.supports_extended_thinking is True assert capabilities.supports_system_prompts is True assert capabilities.supports_streaming is False assert capabilities.supports_function_calling is True assert capabilities.supports_temperature is True def test_get_capabilities_gpt5_mini(self): """Test getting model capabilities for GPT-5-mini.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("gpt-5-mini") assert capabilities.model_name == "gpt-5-mini" assert capabilities.friendly_name == "OpenAI (GPT-5-mini)" assert capabilities.context_window == 400_000 assert capabilities.max_output_tokens == 128_000 assert capabilities.provider == ProviderType.OPENAI assert capabilities.supports_extended_thinking is True assert capabilities.supports_system_prompts is True assert capabilities.supports_streaming is False assert capabilities.supports_function_calling is True assert capabilities.supports_temperature is True def test_get_capabilities_gpt52(self): """Test GPT-5.2 capabilities reflect new metadata.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("gpt-5.2") assert capabilities.model_name == "gpt-5.2" assert capabilities.supports_streaming is True assert capabilities.supports_function_calling is True assert capabilities.supports_json_mode is True assert capabilities.allow_code_generation is True def test_get_capabilities_gpt51_codex(self): """Test GPT-5.1 Codex is responses-only and non-streaming.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("gpt-5.1-codex") assert capabilities.model_name == "gpt-5.1-codex" assert capabilities.supports_streaming is False assert capabilities.use_openai_response_api is True assert capabilities.allow_code_generation is True def test_get_capabilities_gpt51_codex_mini(self): """Test GPT-5.1 Codex mini exposes streaming and code generation.""" provider = OpenAIModelProvider("test-key") capabilities = provider.get_capabilities("gpt-5.1-codex-mini") assert capabilities.model_name == "gpt-5.1-codex-mini" assert capabilities.supports_streaming is True assert capabilities.allow_code_generation is True @patch("providers.openai_compatible.OpenAI") def test_generate_content_resolves_alias_before_api_call(self, mock_openai_class): """Test that generate_content resolves aliases before making API calls. This is the CRITICAL test that was missing - verifying that aliases like 'mini' get resolved to 'o4-mini' before being sent to OpenAI API. """ # Set up mock OpenAI client mock_client = MagicMock() mock_openai_class.return_value = mock_client # Mock the completion response mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "gpt-4.1-2025-04-14" # API returns the resolved model name mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response provider = OpenAIModelProvider("test-key") # Call generate_content with alias 'gpt4.1' (resolves to gpt-4.1, supports temperature) result = provider.generate_content( prompt="Test prompt", model_name="gpt4.1", temperature=1.0, # This should be resolved to "gpt-4.1" ) # Verify the API was called with the RESOLVED model name mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] # CRITICAL ASSERTION: The API should receive "gpt-4.1", not "gpt4.1" assert call_kwargs["model"] == "gpt-4.1", f"Expected 'gpt-4.1' but API received '{call_kwargs['model']}'" # Verify other parameters (gpt-4.1 supports temperature unlike O3/O4 models) assert call_kwargs["temperature"] == 1.0 assert len(call_kwargs["messages"]) == 1 assert call_kwargs["messages"][0]["role"] == "user" assert call_kwargs["messages"][0]["content"] == "Test prompt" # Verify response assert result.content == "Test response" assert result.model_name == "gpt-4.1" # Should be the resolved name @patch("providers.openai_compatible.OpenAI") def test_generate_content_other_aliases(self, mock_openai_class): """Test other alias resolutions in generate_content.""" # Set up mock mock_client = MagicMock() mock_openai_class.return_value = mock_client mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response provider = OpenAIModelProvider("test-key") # Test o3mini -> o3-mini mock_response.model = "o3-mini" provider.generate_content(prompt="Test", model_name="o3mini", temperature=1.0) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "o3-mini" # Test o4mini -> o4-mini mock_response.model = "o4-mini" provider.generate_content(prompt="Test", model_name="o4mini", temperature=1.0) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "o4-mini" @patch("providers.openai_compatible.OpenAI") def test_generate_content_no_alias_passthrough(self, mock_openai_class): """Test that full model names pass through unchanged.""" # Set up mock mock_client = MagicMock() mock_openai_class.return_value = mock_client mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "o3-mini" mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response provider = OpenAIModelProvider("test-key") # Test full model name passes through unchanged (use o3-mini since o3-pro has special handling) provider.generate_content(prompt="Test", model_name="o3-mini", temperature=1.0) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "o3-mini" # Should be unchanged def test_extended_thinking_capabilities(self): """Thinking-mode support should be reflected via ModelCapabilities.""" provider = OpenAIModelProvider("test-key") supported_aliases = [ "gpt-5", "gpt-5-mini", "gpt-5-nano", "gpt5", "gpt5-mini", "gpt5mini", "gpt5-nano", "gpt5nano", "nano", "mini", # resolves to gpt-5-mini ] for alias in supported_aliases: assert provider.get_capabilities(alias).supports_extended_thinking is True unsupported_aliases = ["o3", "o3-mini", "o4-mini"] for alias in unsupported_aliases: assert provider.get_capabilities(alias).supports_extended_thinking is False # Invalid models should not validate, treat as unsupported assert not provider.validate_model_name("invalid-model") @patch("providers.openai_compatible.OpenAI") def test_o3_pro_routes_to_responses_endpoint(self, mock_openai_class): """Test that o3-pro model routes to the /v1/responses endpoint (mock test).""" # Set up mock for OpenAI client responses endpoint mock_client = MagicMock() mock_openai_class.return_value = mock_client mock_response = MagicMock() # New o3-pro format: direct output_text field mock_response.output_text = "4" mock_response.model = "o3-pro" mock_response.id = "test-id" mock_response.created_at = 1234567890 mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.responses.create.return_value = mock_response provider = OpenAIModelProvider("test-key") # Generate content with o3-pro result = provider.generate_content(prompt="What is 2 + 2?", model_name="o3-pro", temperature=1.0) # Verify responses.create was called mock_client.responses.create.assert_called_once() call_args = mock_client.responses.create.call_args[1] assert call_args["model"] == "o3-pro" assert call_args["input"][0]["role"] == "user" assert "What is 2 + 2?" in call_args["input"][0]["content"][0]["text"] # Verify the response assert result.content == "4" assert result.model_name == "o3-pro" assert result.metadata["endpoint"] == "responses" @patch("providers.openai_compatible.OpenAI") def test_non_o3_pro_uses_chat_completions(self, mock_openai_class): """Test that non-o3-pro models use the standard chat completions endpoint.""" # Set up mock mock_client = MagicMock() mock_openai_class.return_value = mock_client mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "o3-mini" mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response provider = OpenAIModelProvider("test-key") # Generate content with o3-mini (not o3-pro) result = provider.generate_content(prompt="Test prompt", model_name="o3-mini", temperature=1.0) # Verify chat.completions.create was called mock_client.chat.completions.create.assert_called_once() # Verify the response assert result.content == "Test response" assert result.model_name == "o3-mini" ================================================ FILE: tests/test_openrouter_provider.py ================================================ """Tests for OpenRouter provider.""" import os from unittest.mock import Mock, patch import pytest from providers.openrouter import OpenRouterProvider from providers.registry import ModelProviderRegistry from providers.shared import ProviderType class TestOpenRouterProvider: """Test cases for OpenRouter provider.""" def test_provider_initialization(self): """Test OpenRouter provider initialization.""" provider = OpenRouterProvider(api_key="test-key") assert provider.api_key == "test-key" assert provider.base_url == "https://openrouter.ai/api/v1" assert provider.FRIENDLY_NAME == "OpenRouter" def test_custom_headers(self): """Test OpenRouter custom headers.""" # Test default headers assert "HTTP-Referer" in OpenRouterProvider.DEFAULT_HEADERS assert "X-Title" in OpenRouterProvider.DEFAULT_HEADERS # Test with environment variables with patch.dict(os.environ, {"OPENROUTER_REFERER": "https://myapp.com", "OPENROUTER_TITLE": "My App"}): from importlib import reload import providers.openrouter reload(providers.openrouter) provider = providers.openrouter.OpenRouterProvider(api_key="test-key") assert provider.DEFAULT_HEADERS["HTTP-Referer"] == "https://myapp.com" assert provider.DEFAULT_HEADERS["X-Title"] == "My App" def test_model_validation(self): """Test model validation.""" provider = OpenRouterProvider(api_key="test-key") # OpenRouter accepts models with provider prefixes or known models assert provider.validate_model_name("openai/gpt-4") is True assert provider.validate_model_name("anthropic/claude-3-opus") is True assert provider.validate_model_name("google/any-model-name") is True assert provider.validate_model_name("groq/llama-3.1-8b") is True assert provider.validate_model_name("grok-4") is True # Unknown models without provider prefix are rejected assert provider.validate_model_name("gpt-4") is False assert provider.validate_model_name("unknown-model") is False def test_get_capabilities(self): """Test capability generation.""" provider = OpenRouterProvider(api_key="test-key") # Test with a model in the registry (using alias) caps = provider.get_capabilities("o3") assert caps.provider == ProviderType.OPENROUTER assert caps.model_name == "openai/o3" # Resolved name assert caps.friendly_name == "OpenRouter (openai/o3)" # Test with a model not in registry - should raise error with pytest.raises(ValueError, match="Unsupported model 'unknown-model' for provider openrouter"): provider.get_capabilities("unknown-model") # Test with model that has provider prefix - should get generic capabilities caps = provider.get_capabilities("provider/unknown-model") assert caps.provider == ProviderType.OPENROUTER assert caps.model_name == "provider/unknown-model" assert caps.context_window == 32_768 # Safe default assert hasattr(caps, "_is_generic") and caps._is_generic is True def test_model_alias_resolution(self): """Test model alias resolution.""" provider = OpenRouterProvider(api_key="test-key") # Test alias resolution assert provider._resolve_model_name("opus") == "anthropic/claude-opus-4.5" assert provider._resolve_model_name("opus4.5") == "anthropic/claude-opus-4.5" assert provider._resolve_model_name("opus4.1") == "anthropic/claude-opus-4.1" assert provider._resolve_model_name("sonnet") == "anthropic/claude-sonnet-4.5" assert provider._resolve_model_name("sonnet4.1") == "anthropic/claude-sonnet-4.1" assert provider._resolve_model_name("o3") == "openai/o3" assert provider._resolve_model_name("o3-mini") == "openai/o3-mini" assert provider._resolve_model_name("o3mini") == "openai/o3-mini" assert provider._resolve_model_name("o4-mini") == "openai/o4-mini" assert provider._resolve_model_name("o4-mini") == "openai/o4-mini" assert provider._resolve_model_name("haiku") == "anthropic/claude-3.5-haiku" assert provider._resolve_model_name("mistral") == "mistralai/mistral-large-2411" assert provider._resolve_model_name("grok-4") == "x-ai/grok-4" assert provider._resolve_model_name("grok4") == "x-ai/grok-4" assert provider._resolve_model_name("grok") == "x-ai/grok-4" assert provider._resolve_model_name("deepseek") == "deepseek/deepseek-r1-0528" assert provider._resolve_model_name("r1") == "deepseek/deepseek-r1-0528" # Test case-insensitive assert provider._resolve_model_name("OPUS") == "anthropic/claude-opus-4.5" assert provider._resolve_model_name("SONNET") == "anthropic/claude-sonnet-4.5" assert provider._resolve_model_name("O3") == "openai/o3" assert provider._resolve_model_name("Mistral") == "mistralai/mistral-large-2411" # Test direct model names (should pass through unchanged) assert provider._resolve_model_name("anthropic/claude-opus-4.1") == "anthropic/claude-opus-4.1" assert provider._resolve_model_name("openai/o3") == "openai/o3" # Test unknown models pass through assert provider._resolve_model_name("unknown-model") == "unknown-model" assert provider._resolve_model_name("custom/model-v2") == "custom/model-v2" def test_openrouter_registration(self): """Test OpenRouter can be registered and retrieved.""" with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}): # Clean up any existing registration ModelProviderRegistry.unregister_provider(ProviderType.OPENROUTER) # Register the provider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Retrieve and verify provider = ModelProviderRegistry.get_provider(ProviderType.OPENROUTER) assert provider is not None assert isinstance(provider, OpenRouterProvider) class TestOpenRouterAutoMode: """Test auto mode functionality when only OpenRouter is configured.""" def setup_method(self): """Store original state before each test.""" self.registry = ModelProviderRegistry() self._original_providers = self.registry._providers.copy() self._original_initialized = self.registry._initialized_providers.copy() self.registry._providers.clear() self.registry._initialized_providers.clear() self._original_env = {} for key in ["OPENROUTER_API_KEY", "GEMINI_API_KEY", "OPENAI_API_KEY", "DEFAULT_MODEL"]: self._original_env[key] = os.environ.get(key) def teardown_method(self): """Restore original state after each test.""" self.registry._providers.clear() self.registry._initialized_providers.clear() self.registry._providers.update(self._original_providers) self.registry._initialized_providers.update(self._original_initialized) for key, value in self._original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_openrouter_only_auto_mode(self): """Test that auto mode works when only OpenRouter is configured.""" os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" os.environ["DEFAULT_MODEL"] = "auto" mock_registry = Mock() model_names = [ "google/gemini-2.5-flash", "google/gemini-2.5-pro", "openai/o3", "openai/o3-mini", "anthropic/claude-opus-4.1", "anthropic/claude-sonnet-4.1", ] mock_registry.list_models.return_value = model_names # Mock resolve to return a ModelCapabilities-like object for each model def mock_resolve(model_name): if model_name in model_names: mock_config = Mock() mock_config.provider = ProviderType.OPENROUTER mock_config.aliases = [] # Empty list of aliases mock_config.get_effective_capability_rank = Mock(return_value=50) # Add ranking method return mock_config return None mock_registry.resolve.side_effect = mock_resolve ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) provider = ModelProviderRegistry.get_provider(ProviderType.OPENROUTER) assert provider is not None, "OpenRouter provider should be available with API key" provider._registry = mock_registry available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) assert len(available_models) > 0, "Should find OpenRouter models in auto mode" assert all(provider_type == ProviderType.OPENROUTER for provider_type in available_models.values()) for model in model_names: assert model in available_models, f"Model {model} should be available" @pytest.mark.no_mock_provider def test_openrouter_with_restrictions(self): """Test that OpenRouter respects model restrictions.""" os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) os.environ["OPENROUTER_ALLOWED_MODELS"] = "anthropic/claude-opus-4.1,google/gemini-2.5-flash" os.environ["DEFAULT_MODEL"] = "auto" # Force reload to pick up new environment variable import utils.model_restrictions utils.model_restrictions._restriction_service = None mock_registry = Mock() mock_models = [ "google/gemini-2.5-flash", "google/gemini-2.5-pro", "anthropic/claude-opus-4.1", "anthropic/claude-sonnet-4.1", ] mock_registry.list_models.return_value = mock_models # Mock the resolve method to return model configs with aliases mock_model_config = Mock() mock_model_config.aliases = [] # Empty aliases for simplicity mock_model_config.get_effective_capability_rank = Mock(return_value=50) # Add ranking method mock_registry.resolve.return_value = mock_model_config ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) provider = ModelProviderRegistry.get_provider(ProviderType.OPENROUTER) provider._registry = mock_registry available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) assert len(available_models) > 0, "Should have some allowed models" expected_allowed = {"google/gemini-2.5-flash", "anthropic/claude-opus-4.1"} assert ( set(available_models.keys()) == expected_allowed ), f"Expected {expected_allowed}, but got {set(available_models.keys())}" @pytest.mark.no_mock_provider def test_no_providers_fails_auto_mode(self): """Test that auto mode fails gracefully when no providers are available.""" os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("OPENROUTER_API_KEY", None) os.environ["DEFAULT_MODEL"] = "auto" available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) assert len(available_models) == 0, "Should have no models when no providers are configured" @pytest.mark.no_mock_provider def test_openrouter_without_registry(self): """Test that OpenRouter without _registry attribute doesn't crash.""" os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" os.environ["DEFAULT_MODEL"] = "auto" mock_provider_class = Mock() mock_provider_instance = Mock(spec=["get_provider_type", "list_models", "get_all_model_capabilities"]) mock_provider_instance.get_provider_type.return_value = ProviderType.OPENROUTER mock_provider_instance.list_models.return_value = [] mock_provider_instance.get_all_model_capabilities.return_value = {} mock_provider_class.return_value = mock_provider_instance ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, mock_provider_class) available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) assert len(available_models) == 0, "Should have no models when OpenRouter has no registry" class TestOpenRouterRegistry: """Test cases for OpenRouter model registry.""" def test_registry_loading(self): """Test registry loads models from config.""" from providers.registries.openrouter import OpenRouterModelRegistry registry = OpenRouterModelRegistry() # Should have loaded models models = registry.list_models() assert len(models) > 0 assert "anthropic/claude-opus-4.1" in models assert "openai/o3" in models # Should have loaded aliases aliases = registry.list_aliases() assert len(aliases) > 0 assert "opus" in aliases assert "o3" in aliases assert "sonnet" in aliases def test_registry_capabilities(self): """Test registry provides correct capabilities.""" from providers.registries.openrouter import OpenRouterModelRegistry registry = OpenRouterModelRegistry() # Test known model (opus alias now points to 4.5) caps = registry.get_capabilities("opus") assert caps is not None assert caps.model_name == "anthropic/claude-opus-4.5" assert caps.context_window == 200000 # Claude's context window # Test using full model name for 4.5 caps = registry.get_capabilities("anthropic/claude-opus-4.5") assert caps is not None assert caps.model_name == "anthropic/claude-opus-4.5" # Test opus4.5 alias caps = registry.get_capabilities("opus4.5") assert caps is not None assert caps.model_name == "anthropic/claude-opus-4.5" # Test using full model name for 4.1 caps = registry.get_capabilities("anthropic/claude-opus-4.1") assert caps is not None assert caps.model_name == "anthropic/claude-opus-4.1" # Test opus4.1 alias still works caps = registry.get_capabilities("opus4.1") assert caps is not None assert caps.model_name == "anthropic/claude-opus-4.1" # Test unknown model caps = registry.get_capabilities("non-existent-model") assert caps is None def test_multiple_aliases_same_model(self): """Test multiple aliases pointing to same model.""" from providers.registries.openrouter import OpenRouterModelRegistry registry = OpenRouterModelRegistry() # All these should resolve to Claude Sonnet 4.5 sonnet_45_aliases = ["sonnet", "sonnet4.5"] for alias in sonnet_45_aliases: config = registry.resolve(alias) assert config is not None assert config.model_name == "anthropic/claude-sonnet-4.5" # Test Sonnet 4.1 alias config = registry.resolve("sonnet4.1") assert config is not None assert config.model_name == "anthropic/claude-sonnet-4.1" class TestOpenRouterFunctionality: """Test OpenRouter-specific functionality.""" def test_openrouter_always_uses_correct_url(self): """Test that OpenRouter always uses the correct base URL.""" provider = OpenRouterProvider(api_key="test-key") assert provider.base_url == "https://openrouter.ai/api/v1" # Even if we try to change it, it should remain the OpenRouter URL # (This is a characteristic of the OpenRouter provider) provider.base_url = "http://example.com" # Try to change it # But new instances should always use the correct URL provider2 = OpenRouterProvider(api_key="test-key") assert provider2.base_url == "https://openrouter.ai/api/v1" def test_openrouter_headers_set_correctly(self): """Test that OpenRouter specific headers are set.""" provider = OpenRouterProvider(api_key="test-key") # Check default headers assert "HTTP-Referer" in provider.DEFAULT_HEADERS assert "X-Title" in provider.DEFAULT_HEADERS assert provider.DEFAULT_HEADERS["X-Title"] == "PAL MCP Server" def test_openrouter_model_registry_initialized(self): """Test that model registry is properly initialized.""" provider = OpenRouterProvider(api_key="test-key") # Registry should be initialized assert hasattr(provider, "_registry") assert provider._registry is not None ================================================ FILE: tests/test_openrouter_registry.py ================================================ """Tests for OpenRouter model registry functionality.""" import json import os import tempfile from unittest.mock import patch import pytest from providers.registries.openrouter import OpenRouterModelRegistry from providers.shared import ModelCapabilities, ProviderType class TestOpenRouterModelRegistry: """Test cases for OpenRouter model registry.""" def test_registry_initialization(self): """Test registry initializes with default config.""" registry = OpenRouterModelRegistry() # Should load models from default location assert len(registry.list_models()) > 0 assert len(registry.list_aliases()) > 0 def test_custom_config_path(self): """Test registry with custom config path.""" # Create temporary config config_data = { "models": [ { "model_name": "test/model-1", "aliases": ["test1", "t1"], "context_window": 4096, "max_output_tokens": 2048, } ] } with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: json.dump(config_data, f) temp_path = f.name try: registry = OpenRouterModelRegistry(config_path=temp_path) assert len(registry.list_models()) == 1 assert "test/model-1" in registry.list_models() assert "test1" in registry.list_aliases() assert "t1" in registry.list_aliases() finally: os.unlink(temp_path) def test_environment_variable_override(self): """Test OPENROUTER_MODELS_CONFIG_PATH environment variable.""" # Create custom config config_data = { "models": [ {"model_name": "env/model", "aliases": ["envtest"], "context_window": 8192, "max_output_tokens": 4096} ] } with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: json.dump(config_data, f) temp_path = f.name try: # Set environment variable original_env = os.environ.get("OPENROUTER_MODELS_CONFIG_PATH") os.environ["OPENROUTER_MODELS_CONFIG_PATH"] = temp_path # Create registry without explicit path registry = OpenRouterModelRegistry() # Should load from environment path assert "env/model" in registry.list_models() assert "envtest" in registry.list_aliases() finally: # Restore environment if original_env is not None: os.environ["OPENROUTER_MODELS_CONFIG_PATH"] = original_env else: del os.environ["OPENROUTER_MODELS_CONFIG_PATH"] os.unlink(temp_path) def test_alias_resolution(self): """Test alias resolution functionality.""" registry = OpenRouterModelRegistry() # Test various aliases test_cases = [ ("opus", "anthropic/claude-opus-4.5"), # opus now points to 4.5 ("OPUS", "anthropic/claude-opus-4.5"), # Case insensitive ("claude-opus", "anthropic/claude-opus-4.5"), ("opus4.5", "anthropic/claude-opus-4.5"), ("opus4.1", "anthropic/claude-opus-4.1"), # 4.1 still accessible ("sonnet", "anthropic/claude-sonnet-4.5"), ("o3", "openai/o3"), ("deepseek", "deepseek/deepseek-r1-0528"), ("mistral", "mistralai/mistral-large-2411"), ] for alias, expected_model in test_cases: config = registry.resolve(alias) assert config is not None, f"Failed to resolve alias '{alias}'" assert config.model_name == expected_model def test_direct_model_name_lookup(self): """Test looking up models by their full name.""" registry = OpenRouterModelRegistry() # Should be able to look up by full model name config = registry.resolve("anthropic/claude-opus-4.1") assert config is not None assert config.model_name == "anthropic/claude-opus-4.1" config = registry.resolve("openai/o3") assert config is not None assert config.model_name == "openai/o3" def test_unknown_model_resolution(self): """Test resolution of unknown models.""" registry = OpenRouterModelRegistry() # Unknown aliases should return None assert registry.resolve("unknown-alias") is None assert registry.resolve("") is None assert registry.resolve("non-existent") is None def test_model_capabilities_conversion(self): """Test that registry returns ModelCapabilities directly.""" registry = OpenRouterModelRegistry() config = registry.resolve("opus") assert config is not None # Registry now returns ModelCapabilities objects directly # opus alias now points to 4.5 assert config.provider == ProviderType.OPENROUTER assert config.model_name == "anthropic/claude-opus-4.5" assert config.friendly_name == "OpenRouter (anthropic/claude-opus-4.5)" assert config.context_window == 200000 assert not config.supports_extended_thinking def test_duplicate_alias_detection(self): """Test that duplicate aliases are detected.""" config_data = { "models": [ {"model_name": "test/model-1", "aliases": ["dupe"], "context_window": 4096, "max_output_tokens": 2048}, { "model_name": "test/model-2", "aliases": ["DUPE"], # Same alias, different case "context_window": 8192, "max_output_tokens": 2048, }, ] } with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: json.dump(config_data, f) temp_path = f.name try: with pytest.raises(ValueError, match="Duplicate alias"): OpenRouterModelRegistry(config_path=temp_path) finally: os.unlink(temp_path) def test_backwards_compatibility_max_tokens(self): """Test that legacy max_tokens field maps to max_output_tokens.""" config_data = { "models": [ { "model_name": "test/old-model", "aliases": ["old"], "max_tokens": 16384, # Old field name should cause error "supports_extended_thinking": False, } ] } with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: json.dump(config_data, f) temp_path = f.name try: with patch.dict("os.environ", {}, clear=True): with pytest.raises(ValueError, match="max_output_tokens"): OpenRouterModelRegistry(config_path=temp_path) finally: os.unlink(temp_path) def test_missing_config_file(self): """Test behavior with missing config file.""" # Use a non-existent path with patch.dict("os.environ", {}, clear=True): registry = OpenRouterModelRegistry(config_path="/non/existent/path.json") # Should initialize with empty maps assert len(registry.list_models()) == 0 assert len(registry.list_aliases()) == 0 assert registry.resolve("anything") is None def test_invalid_json_config(self): """Test handling of invalid JSON.""" with tempfile.NamedTemporaryFile(mode="w", suffix=".json", delete=False) as f: f.write("{ invalid json }") temp_path = f.name try: registry = OpenRouterModelRegistry(config_path=temp_path) # Should handle gracefully and initialize empty assert len(registry.list_models()) == 0 assert len(registry.list_aliases()) == 0 finally: os.unlink(temp_path) def test_model_with_all_capabilities(self): """Test model with all capability flags.""" from providers.shared import TemperatureConstraint caps = ModelCapabilities( provider=ProviderType.OPENROUTER, model_name="test/full-featured", friendly_name="OpenRouter (test/full-featured)", aliases=["full"], context_window=128000, max_output_tokens=8192, supports_extended_thinking=True, supports_system_prompts=True, supports_streaming=True, supports_function_calling=True, supports_json_mode=True, description="Fully featured test model", temperature_constraint=TemperatureConstraint.create("range"), ) assert caps.context_window == 128000 assert caps.supports_extended_thinking assert caps.supports_system_prompts assert caps.supports_streaming assert caps.supports_function_calling # Note: supports_json_mode is not in ModelCapabilities yet ================================================ FILE: tests/test_openrouter_store_parameter.py ================================================ """Tests for OpenRouter store parameter handling in responses endpoint. Regression tests for GitHub Issue #348: OpenAI "store" parameter validation error for certain models via OpenRouter. OpenRouter's /responses endpoint rejects store:true via Zod validation. This is an endpoint-level limitation, not model-specific. These tests verify that: - OpenRouter provider omits the store parameter - Direct OpenAI provider includes store: true """ import unittest from unittest.mock import Mock, patch from providers.openai_compatible import OpenAICompatibleProvider from providers.shared import ProviderType class MockOpenRouterProvider(OpenAICompatibleProvider): """Mock provider that simulates OpenRouter behavior.""" FRIENDLY_NAME = "OpenRouter Test" def get_provider_type(self): return ProviderType.OPENROUTER def get_capabilities(self, model_name): mock_caps = Mock() mock_caps.default_reasoning_effort = "high" return mock_caps def validate_model_name(self, model_name): return True def list_models(self, **kwargs): return ["openai/gpt-5-pro", "openai/gpt-5.1-codex"] class MockOpenAIProvider(OpenAICompatibleProvider): """Mock provider that simulates direct OpenAI behavior.""" FRIENDLY_NAME = "OpenAI Test" def get_provider_type(self): return ProviderType.OPENAI def get_capabilities(self, model_name): mock_caps = Mock() mock_caps.default_reasoning_effort = "high" return mock_caps def validate_model_name(self, model_name): return True def list_models(self, **kwargs): return ["gpt-5-pro", "gpt-5.1-codex"] class TestStoreParameterHandling(unittest.TestCase): """Test store parameter is conditionally included based on provider type. **Feature: openrouter-store-parameter-fix, Property 1: OpenRouter requests omit store parameter** **Feature: openrouter-store-parameter-fix, Property 2: Direct OpenAI requests include store parameter** """ def test_openrouter_responses_omits_store_parameter(self): """Test that OpenRouter provider omits store parameter from responses endpoint. **Feature: openrouter-store-parameter-fix, Property 1: OpenRouter requests omit store parameter** **Validates: Requirements 1.1, 2.1** OpenRouter's /responses endpoint rejects store:true via Zod validation (Issue #348). The store parameter should be omitted entirely for OpenRouter requests. """ # Capture the completion_params passed to the API captured_params = {} def capture_create(**kwargs): captured_params.update(kwargs) # Return a mock response mock_response = Mock() mock_response.output_text = "Test response" mock_response.usage = None return mock_response mock_client_instance = Mock() mock_client_instance.responses.create = capture_create with patch.object( MockOpenRouterProvider, "client", new_callable=lambda: property(lambda self: mock_client_instance) ): provider = MockOpenRouterProvider("test-key") # Call the method that builds completion_params provider._generate_with_responses_endpoint( model_name="openai/gpt-5-pro", messages=[{"role": "user", "content": "test"}], temperature=0.7, ) # Verify store parameter is NOT in the request self.assertNotIn("store", captured_params, "OpenRouter requests should NOT include 'store' parameter") def test_openai_responses_includes_store_parameter(self): """Test that direct OpenAI provider includes store parameter in responses endpoint. **Feature: openrouter-store-parameter-fix, Property 2: Direct OpenAI requests include store parameter** **Validates: Requirements 1.2, 2.2** Direct OpenAI API supports the store parameter for stored completions. The store parameter should be included with value True for OpenAI requests. """ # Capture the completion_params passed to the API captured_params = {} def capture_create(**kwargs): captured_params.update(kwargs) # Return a mock response mock_response = Mock() mock_response.output_text = "Test response" mock_response.usage = None return mock_response mock_client_instance = Mock() mock_client_instance.responses.create = capture_create with patch.object( MockOpenAIProvider, "client", new_callable=lambda: property(lambda self: mock_client_instance) ): provider = MockOpenAIProvider("test-key") # Call the method that builds completion_params provider._generate_with_responses_endpoint( model_name="gpt-5-pro", messages=[{"role": "user", "content": "test"}], temperature=0.7, ) # Verify store parameter IS in the request with value True self.assertIn("store", captured_params, "OpenAI requests should include 'store' parameter") self.assertTrue(captured_params["store"], "OpenAI requests should have store=True") if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_parse_model_option.py ================================================ """Tests for parse_model_option function.""" from server import parse_model_option class TestParseModelOption: """Test cases for model option parsing.""" def test_openrouter_free_suffix_preserved(self): """Test that OpenRouter :free suffix is preserved as part of model name.""" model, option = parse_model_option("openai/gpt-3.5-turbo:free") assert model == "openai/gpt-3.5-turbo:free" assert option is None def test_openrouter_beta_suffix_preserved(self): """Test that OpenRouter :beta suffix is preserved as part of model name.""" model, option = parse_model_option("anthropic/claude-opus-4.1:beta") assert model == "anthropic/claude-opus-4.1:beta" assert option is None def test_openrouter_preview_suffix_preserved(self): """Test that OpenRouter :preview suffix is preserved as part of model name.""" model, option = parse_model_option("google/gemini-pro:preview") assert model == "google/gemini-pro:preview" assert option is None def test_ollama_tag_parsed_as_option(self): """Test that Ollama tags are parsed as options.""" model, option = parse_model_option("llama3.2:latest") assert model == "llama3.2" assert option == "latest" def test_consensus_stance_parsed_as_option(self): """Test that consensus stances are parsed as options.""" model, option = parse_model_option("o3:for") assert model == "o3" assert option == "for" model, option = parse_model_option("gemini-2.5-pro:against") assert model == "gemini-2.5-pro" assert option == "against" def test_openrouter_unknown_suffix_parsed_as_option(self): """Test that unknown suffixes on OpenRouter models are parsed as options.""" model, option = parse_model_option("openai/gpt-4:custom-tag") assert model == "openai/gpt-4" assert option == "custom-tag" def test_plain_model_name(self): """Test plain model names without colons.""" model, option = parse_model_option("gpt-4") assert model == "gpt-4" assert option is None def test_url_not_parsed(self): """Test that URLs are not parsed for options.""" model, option = parse_model_option("http://localhost:8080") assert model == "http://localhost:8080" assert option is None def test_whitespace_handling(self): """Test that whitespace is properly stripped.""" model, option = parse_model_option(" openai/gpt-3.5-turbo:free ") assert model == "openai/gpt-3.5-turbo:free" assert option is None model, option = parse_model_option(" llama3.2 : latest ") assert model == "llama3.2" assert option == "latest" def test_case_insensitive_suffix_matching(self): """Test that OpenRouter suffix matching is case-insensitive.""" model, option = parse_model_option("openai/gpt-3.5-turbo:FREE") assert model == "openai/gpt-3.5-turbo:FREE" # Original case preserved assert option is None model, option = parse_model_option("openai/gpt-3.5-turbo:Free") assert model == "openai/gpt-3.5-turbo:Free" # Original case preserved assert option is None ================================================ FILE: tests/test_path_traversal_security.py ================================================ """ Test path traversal security fix. Fixes vulnerability reported in: - https://github.com/BeehiveInnovations/zen-mcp-server/issues/293 - https://github.com/BeehiveInnovations/zen-mcp-server/issues/312 The vulnerability: is_dangerous_path() only did exact string matching, so /etc was blocked but /etc/passwd was allowed. Additionally, this fix properly handles home directory containers: - /home and C:\\Users are blocked (exact match only) - /home/user/project paths are allowed through is_dangerous_path() and handled by is_home_directory_root() in resolve_and_validate_path() """ from pathlib import Path from utils.security_config import is_dangerous_path class TestPathTraversalFix: """Test that subdirectories of dangerous system paths are blocked.""" def test_exact_match_still_works(self): """Test that exact dangerous paths are still blocked.""" assert is_dangerous_path(Path("/etc")) is True assert is_dangerous_path(Path("/usr")) is True assert is_dangerous_path(Path("/var")) is True def test_subdirectory_now_blocked(self): """Test that subdirectories of system paths are blocked (the fix).""" # These were allowed before the fix assert is_dangerous_path(Path("/etc/passwd")) is True assert is_dangerous_path(Path("/etc/shadow")) is True assert is_dangerous_path(Path("/etc/hosts")) is True assert is_dangerous_path(Path("/var/log/auth.log")) is True def test_deeply_nested_blocked(self): """Test that deeply nested system paths are blocked.""" assert is_dangerous_path(Path("/etc/ssh/sshd_config")) is True assert is_dangerous_path(Path("/usr/local/bin/python")) is True def test_root_blocked(self): """Test that root directory is blocked.""" assert is_dangerous_path(Path("/")) is True def test_safe_paths_allowed(self): """Test that safe paths are still allowed.""" # User project directories should be allowed assert is_dangerous_path(Path("/tmp/test")) is False assert is_dangerous_path(Path("/tmp/myproject/src")) is False def test_similar_names_not_blocked(self): """Test that paths with similar names are not blocked.""" # /etcbackup should NOT be blocked (it's not under /etc) assert is_dangerous_path(Path("/tmp/etcbackup")) is False assert is_dangerous_path(Path("/tmp/my_etc_files")) is False class TestHomeDirectoryHandling: """Test that home directory containers are handled correctly. Home containers (/home, C:\\Users) should only block the exact path, not subdirectories. Subdirectory access control is delegated to is_home_directory_root() in resolve_and_validate_path(). """ def test_home_container_blocked(self): """Test that /home itself is blocked.""" assert is_dangerous_path(Path("/home")) is True def test_home_subdirectories_allowed(self): """Test that /home subdirectories pass through is_dangerous_path(). These paths should NOT be blocked by is_dangerous_path() because: 1. /home/user/project is a valid user workspace 2. Access control for /home/username is handled by is_home_directory_root() """ # User home directories should pass is_dangerous_path() # (they are handled by is_home_directory_root() separately) assert is_dangerous_path(Path("/home/user")) is False assert is_dangerous_path(Path("/home/user/project")) is False assert is_dangerous_path(Path("/home/user/project/src/main.py")) is False def test_home_deeply_nested_allowed(self): """Test that deeply nested home paths are allowed.""" assert is_dangerous_path(Path("/home/user/documents/work/project/src")) is False class TestRegressionPrevention: """Regression tests for the specific vulnerability.""" def test_etc_passwd_blocked(self): """Test /etc/passwd is blocked (common attack target).""" assert is_dangerous_path(Path("/etc/passwd")) is True def test_etc_shadow_blocked(self): """Test /etc/shadow is blocked (password hashes).""" assert is_dangerous_path(Path("/etc/shadow")) is True class TestWindowsPathHandling: """Test Windows path handling with trailing backslash. Fixes issue reported in PR #353: Windows paths like C:\\ have trailing backslash which caused double separator issues with string prefix matching. Using Path.is_relative_to() resolves this correctly. """ def test_windows_root_drive_blocked(self): """Test that Windows root drive C:\\ is blocked.""" from pathlib import PureWindowsPath # Simulate Windows path behavior using PureWindowsPath # On Linux, we test the logic with PureWindowsPath to verify cross-platform correctness c_root = PureWindowsPath("C:\\") assert c_root.parent == c_root # Root check works def test_windows_dangerous_subdirectory_detection(self): """Test that Windows subdirectories are correctly detected as dangerous. This verifies the fix for the double backslash issue: - Before fix: "C:\\" + "\\" = "C:\\\\" which doesn't match "C:\\Users" - After fix: Path.is_relative_to() handles this correctly """ from pathlib import PureWindowsPath # Verify is_relative_to works correctly for Windows paths c_users = PureWindowsPath("C:\\Users") c_root = PureWindowsPath("C:\\") # This is the key test - subdirectory detection must work assert c_users.is_relative_to(c_root) is True # Deeper paths should also work c_users_admin = PureWindowsPath("C:\\Users\\Admin") assert c_users_admin.is_relative_to(c_root) is True assert c_users_admin.is_relative_to(c_users) is True def test_windows_path_not_relative_to_different_drive(self): """Test that paths on different drives are not related.""" from pathlib import PureWindowsPath d_path = PureWindowsPath("D:\\Data") c_root = PureWindowsPath("C:\\") # D: drive paths should not be relative to C: assert d_path.is_relative_to(c_root) is False ================================================ FILE: tests/test_per_tool_model_defaults.py ================================================ """ Test per-tool model default selection functionality """ import json import os import shutil import tempfile from unittest.mock import MagicMock, patch import pytest from providers.registry import ModelProviderRegistry, ProviderType from tools.analyze import AnalyzeTool from tools.chat import ChatTool from tools.codereview import CodeReviewTool from tools.debug import DebugIssueTool from tools.models import ToolModelCategory from tools.precommit import PrecommitTool from tools.shared.base_tool import BaseTool from tools.shared.exceptions import ToolExecutionError from tools.thinkdeep import ThinkDeepTool class TestToolModelCategories: """Test that each tool returns the correct model category.""" def test_thinkdeep_category(self): tool = ThinkDeepTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_debug_category(self): tool = DebugIssueTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_analyze_category(self): tool = AnalyzeTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_precommit_category(self): tool = PrecommitTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_chat_category(self): tool = ChatTool() assert tool.get_model_category() == ToolModelCategory.FAST_RESPONSE def test_codereview_category(self): tool = CodeReviewTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_base_tool_default_category(self): # Test that BaseTool defaults to BALANCED class TestTool(BaseTool): def get_name(self): return "test" def get_description(self): return "test" def get_input_schema(self): return {} def get_system_prompt(self): return "test" def get_request_model(self): return MagicMock async def prepare_prompt(self, request): return "test" tool = TestTool() assert tool.get_model_category() == ToolModelCategory.BALANCED class TestModelSelection: """Test model selection based on tool categories.""" def teardown_method(self): """Clean up after each test to prevent state pollution.""" ModelProviderRegistry.clear_cache() # Unregister all providers for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) def test_extended_reasoning_with_openai(self): """Test EXTENDED_REASONING with OpenAI provider.""" # Setup with only OpenAI provider ModelProviderRegistry.clear_cache() # First unregister all providers to ensure isolation for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) # OpenAI prefers GPT-5.1-Codex for extended reasoning (coding tasks) assert model == "gpt-5.1-codex" def test_extended_reasoning_with_gemini_only(self): """Test EXTENDED_REASONING prefers pro when only Gemini is available.""" # Clear cache and unregister all providers first ModelProviderRegistry.clear_cache() for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) # Register only Gemini provider with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-key"}, clear=False): from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) # Gemini should return one of its models for extended reasoning # The default behavior may return flash when pro is not explicitly preferred assert model in ["gemini-3-pro-preview", "gemini-2.5-flash", "gemini-2.0-flash"] def test_fast_response_with_openai(self): """Test FAST_RESPONSE with OpenAI provider.""" # Setup with only OpenAI provider ModelProviderRegistry.clear_cache() # First unregister all providers to ensure isolation for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) # OpenAI now prefers gpt-5.2 for fast response (based on our new preference order) assert model == "gpt-5.2" def test_fast_response_with_gemini_only(self): """Test FAST_RESPONSE prefers flash when only Gemini is available.""" # Clear cache and unregister all providers first ModelProviderRegistry.clear_cache() for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) # Register only Gemini provider with patch.dict(os.environ, {"GOOGLE_API_KEY": "test-key"}, clear=False): from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.FAST_RESPONSE) # Gemini should return one of its models for fast response assert model in ["gemini-2.5-flash", "gemini-2.0-flash", "gemini-2.5-pro"] def test_balanced_category_fallback(self): """Test BALANCED category uses existing logic.""" # Setup with only OpenAI provider ModelProviderRegistry.clear_cache() # First unregister all providers to ensure isolation for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) with patch.dict(os.environ, {"OPENAI_API_KEY": "test-key"}, clear=False): from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.BALANCED) # OpenAI prefers gpt-5.2 for balanced (based on our new preference order) assert model == "gpt-5.2" def test_no_category_uses_balanced_logic(self): """Test that no category specified uses balanced logic.""" # Setup with only Gemini provider with patch.dict(os.environ, {"GEMINI_API_KEY": "test-key"}, clear=False): from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model() # Should pick flash for balanced use assert model == "gemini-2.5-flash" class TestFlexibleModelSelection: """Test that model selection handles various naming scenarios.""" def test_fallback_handles_mixed_model_names(self): """Test that fallback selection works with different providers.""" # Test with different provider configurations test_cases = [ # Case 1: OpenAI provider for extended reasoning { "env": {"OPENAI_API_KEY": "test-key"}, "provider_type": ProviderType.OPENAI, "category": ToolModelCategory.EXTENDED_REASONING, "expected": "gpt-5.1-codex", # GPT-5.1-Codex prioritized for coding tasks }, # Case 2: Gemini provider for fast response { "env": {"GEMINI_API_KEY": "test-key"}, "provider_type": ProviderType.GOOGLE, "category": ToolModelCategory.FAST_RESPONSE, "expected": "gemini-2.5-flash", }, # Case 3: OpenAI provider for fast response { "env": {"OPENAI_API_KEY": "test-key"}, "provider_type": ProviderType.OPENAI, "category": ToolModelCategory.FAST_RESPONSE, "expected": "gpt-5.2", # Based on new preference order }, ] for case in test_cases: # Clear registry for clean test ModelProviderRegistry.clear_cache() # First unregister all providers to ensure isolation for provider_type in list(ProviderType): ModelProviderRegistry.unregister_provider(provider_type) with patch.dict(os.environ, case["env"], clear=False): # Register the appropriate provider if case["provider_type"] == ProviderType.OPENAI: from providers.openai import OpenAIModelProvider ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) elif case["provider_type"] == ProviderType.GOOGLE: from providers.gemini import GeminiModelProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) model = ModelProviderRegistry.get_preferred_fallback_model(case["category"]) assert model == case["expected"], f"Failed for case: {case}, got {model}" class TestCustomProviderFallback: """Test fallback to custom/openrouter providers.""" def test_extended_reasoning_custom_fallback(self): """Test EXTENDED_REASONING with custom provider.""" # Setup with custom provider ModelProviderRegistry.clear_cache() with patch.dict(os.environ, {"CUSTOM_API_URL": "http://localhost:11434", "CUSTOM_API_KEY": ""}, clear=False): from providers.custom import CustomProvider ModelProviderRegistry.register_provider(ProviderType.CUSTOM, CustomProvider) provider = ModelProviderRegistry.get_provider(ProviderType.CUSTOM) if provider: model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) # Should get a model from custom provider assert model is not None def test_extended_reasoning_final_fallback(self): """Test EXTENDED_REASONING falls back to default when no providers.""" # Clear all providers ModelProviderRegistry.clear_cache() for provider_type in list( ModelProviderRegistry._instance._providers.keys() if ModelProviderRegistry._instance else [] ): ModelProviderRegistry.unregister_provider(provider_type) model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) # Should fall back to hardcoded default assert model == "gemini-2.5-flash" class TestAutoModeErrorMessages: """Test that auto mode error messages include suggested models.""" def teardown_method(self): """Clean up after each test to prevent state pollution.""" # Clear provider registry singleton ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_chat_auto_error_message(self): """Test Chat tool suggests appropriate model in auto mode.""" with patch("config.IS_AUTO_MODE", True): with patch("config.DEFAULT_MODEL", "auto"): with patch.object(ModelProviderRegistry, "get_available_models") as mock_get_available: # Mock OpenAI models available mock_get_available.return_value = { "o3": ProviderType.OPENAI, "o3-mini": ProviderType.OPENAI, "o4-mini": ProviderType.OPENAI, } # Mock the provider lookup to return None for auto model with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider_for: mock_get_provider_for.return_value = None tool = ChatTool() temp_dir = tempfile.mkdtemp() try: with pytest.raises(ToolExecutionError) as exc_info: await tool.execute( {"prompt": "test", "model": "auto", "working_directory_absolute_path": temp_dir} ) finally: shutil.rmtree(temp_dir, ignore_errors=True) error_output = json.loads(exc_info.value.payload) assert error_output["status"] == "error" assert "Model 'auto' is not available" in error_output["content"] # Removed TestFileContentPreparation class # The original test was using MagicMock which caused TypeErrors when comparing with integers # The test has been removed to avoid mocking issues and encourage real integration testing class TestProviderHelperMethods: """Test the helper methods for finding models from custom/openrouter.""" def test_extended_reasoning_with_custom_provider(self): """Test extended reasoning model selection with custom provider.""" # Setup with custom provider with patch.dict(os.environ, {"CUSTOM_API_URL": "http://localhost:11434", "CUSTOM_API_KEY": ""}, clear=False): from providers.custom import CustomProvider ModelProviderRegistry.register_provider(ProviderType.CUSTOM, CustomProvider) provider = ModelProviderRegistry.get_provider(ProviderType.CUSTOM) if provider: # Custom provider should return a model for extended reasoning model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) assert model is not None def test_extended_reasoning_with_openrouter(self): """Test extended reasoning model selection with OpenRouter.""" # Setup with OpenRouter provider with patch.dict(os.environ, {"OPENROUTER_API_KEY": "test-key"}, clear=False): from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # OpenRouter should provide a model for extended reasoning model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) # Should return first available OpenRouter model assert model is not None def test_fallback_when_no_providers_available(self): """Test fallback when no providers are available.""" # Clear all providers ModelProviderRegistry.clear_cache() for provider_type in list( ModelProviderRegistry._instance._providers.keys() if ModelProviderRegistry._instance else [] ): ModelProviderRegistry.unregister_provider(provider_type) # Should return hardcoded fallback model = ModelProviderRegistry.get_preferred_fallback_model(ToolModelCategory.EXTENDED_REASONING) assert model == "gemini-2.5-flash" class TestEffectiveAutoMode: """Test the is_effective_auto_mode method.""" def test_explicit_auto_mode(self): """Test when DEFAULT_MODEL is explicitly 'auto'.""" with patch("config.DEFAULT_MODEL", "auto"): with patch("config.IS_AUTO_MODE", True): tool = ChatTool() assert tool.is_effective_auto_mode() is True def test_unavailable_model_triggers_auto_mode(self): """Test when DEFAULT_MODEL is set but not available.""" with patch("config.DEFAULT_MODEL", "o3"): with patch("config.IS_AUTO_MODE", False): with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: mock_get_provider.return_value = None # Model not available tool = ChatTool() assert tool.is_effective_auto_mode() is True def test_available_model_no_auto_mode(self): """Test when DEFAULT_MODEL is set and available.""" with patch("config.DEFAULT_MODEL", "pro"): with patch("config.IS_AUTO_MODE", False): with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: mock_get_provider.return_value = MagicMock() # Model is available tool = ChatTool() assert tool.is_effective_auto_mode() is False class TestRuntimeModelSelection: """Test runtime model selection behavior.""" def teardown_method(self): """Clean up after each test to prevent state pollution.""" # Clear provider registry singleton ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_explicit_auto_in_request(self): """Test when Claude explicitly passes model='auto'.""" with patch("config.DEFAULT_MODEL", "pro"): # DEFAULT_MODEL is a real model with patch("config.IS_AUTO_MODE", False): # Not in auto mode tool = ThinkDeepTool() result = await tool.execute( { "step": "test", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "test", "model": "auto", } ) assert len(result) == 1 assert "Model 'auto' is not available" in result[0].text @pytest.mark.asyncio async def test_unavailable_model_in_request(self): """Test when Claude passes an unavailable model.""" with patch("config.DEFAULT_MODEL", "pro"): with patch("config.IS_AUTO_MODE", False): with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: # Model is not available mock_get_provider.return_value = None tool = ChatTool() temp_dir = tempfile.mkdtemp() try: with pytest.raises(ToolExecutionError) as exc_info: await tool.execute( {"prompt": "test", "model": "gpt-5-turbo", "working_directory_absolute_path": temp_dir} ) finally: shutil.rmtree(temp_dir, ignore_errors=True) # Should require model selection error_output = json.loads(exc_info.value.payload) assert error_output["status"] == "error" assert "gpt-5-turbo" in error_output["content"] assert "is not available" in error_output["content"] class TestSchemaGeneration: """Test schema generation with different configurations.""" def test_schema_with_explicit_auto_mode(self): """Test schema when DEFAULT_MODEL='auto'.""" with patch("config.DEFAULT_MODEL", "auto"): with patch("config.IS_AUTO_MODE", True): tool = ChatTool() schema = tool.get_input_schema() # Model should be required assert "model" in schema["required"] def test_schema_with_unavailable_default_model(self): """Test schema when DEFAULT_MODEL is set but unavailable.""" with patch("config.DEFAULT_MODEL", "o3"): with patch("config.IS_AUTO_MODE", False): with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: mock_get_provider.return_value = None # Model not available tool = AnalyzeTool() schema = tool.get_input_schema() # Model should be required due to unavailable DEFAULT_MODEL assert "model" in schema["required"] def test_schema_with_available_default_model(self): """Test schema when DEFAULT_MODEL is available.""" with patch("config.DEFAULT_MODEL", "pro"): with patch("config.IS_AUTO_MODE", False): with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: mock_get_provider.return_value = MagicMock() # Model is available tool = ThinkDeepTool() schema = tool.get_input_schema() # Model should remain optional when DEFAULT_MODEL is available assert "model" not in schema["required"] class TestUnavailableModelFallback: """Test fallback behavior when DEFAULT_MODEL is not available.""" @pytest.mark.asyncio async def test_unavailable_default_model_fallback(self): """Test that unavailable DEFAULT_MODEL triggers auto mode behavior.""" with patch("config.DEFAULT_MODEL", "o3"): # Set DEFAULT_MODEL to a specific model with patch("config.IS_AUTO_MODE", False): # Not in auto mode with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: # Model is not available (no provider) mock_get_provider.return_value = None tool = ThinkDeepTool() result = await tool.execute( { "step": "test", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "test", } ) # No model specified # Should get model error since fallback model is also unavailable assert len(result) == 1 # Workflow tools try fallbacks and report when the fallback model is not available assert "is not available" in result[0].text # Should list available models in the error assert "Available models:" in result[0].text @pytest.mark.asyncio async def test_available_default_model_no_fallback(self): """Test that available DEFAULT_MODEL works normally.""" with patch("config.DEFAULT_MODEL", "pro"): with patch("config.IS_AUTO_MODE", False): with patch.object(ModelProviderRegistry, "get_provider_for_model") as mock_get_provider: # Model is available mock_provider = MagicMock() mock_provider.generate_content.return_value = MagicMock(content="Test response", metadata={}) mock_get_provider.return_value = mock_provider # Mock the provider lookup in BaseTool.get_model_provider with patch.object(BaseTool, "get_model_provider") as mock_get_model_provider: mock_get_model_provider.return_value = mock_provider tool = ChatTool() temp_dir = tempfile.mkdtemp() try: result = await tool.execute({"prompt": "test", "working_directory_absolute_path": temp_dir}) finally: shutil.rmtree(temp_dir, ignore_errors=True) # Should work normally, not require model parameter assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "Test response" in output["content"] ================================================ FILE: tests/test_pii_sanitizer.py ================================================ #!/usr/bin/env python3 """Test cases for PII sanitizer.""" import unittest from .pii_sanitizer import PIIPattern, PIISanitizer class TestPIISanitizer(unittest.TestCase): """Test PII sanitization functionality.""" def setUp(self): """Set up test sanitizer.""" self.sanitizer = PIISanitizer() def test_api_key_sanitization(self): """Test various API key formats are sanitized.""" test_cases = [ # OpenAI keys ("sk-proj-abcd1234567890ABCD1234567890abcd1234567890ABCD12", "sk-proj-SANITIZED"), ("sk-1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN", "sk-SANITIZED"), # Anthropic keys ("sk-ant-abcd1234567890ABCD1234567890abcd1234567890ABCD12", "sk-ant-SANITIZED"), # Google keys ("AIzaSyD-1234567890abcdefghijklmnopqrstuv", "AIza-SANITIZED"), # GitHub tokens ("ghp_1234567890abcdefghijklmnopqrstuvwxyz", "gh_SANITIZED"), ("ghs_1234567890abcdefghijklmnopqrstuvwxyz", "gh_SANITIZED"), ] for original, expected in test_cases: with self.subTest(original=original): result = self.sanitizer.sanitize_string(original) self.assertEqual(result, expected) def test_personal_info_sanitization(self): """Test personal information is sanitized.""" test_cases = [ # Email addresses ("john.doe@example.com", "user@example.com"), ("test123@company.org", "user@example.com"), # Phone numbers (all now use the same pattern) ("(555) 123-4567", "(XXX) XXX-XXXX"), ("555-123-4567", "(XXX) XXX-XXXX"), ("+1-555-123-4567", "(XXX) XXX-XXXX"), # SSN ("123-45-6789", "XXX-XX-XXXX"), # Credit card ("1234 5678 9012 3456", "XXXX-XXXX-XXXX-XXXX"), ("1234-5678-9012-3456", "XXXX-XXXX-XXXX-XXXX"), ] for original, expected in test_cases: with self.subTest(original=original): result = self.sanitizer.sanitize_string(original) self.assertEqual(result, expected) def test_header_sanitization(self): """Test HTTP header sanitization.""" headers = { "Authorization": "Bearer sk-proj-abcd1234567890ABCD1234567890abcd1234567890ABCD12", "API-Key": "sk-1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN", "Content-Type": "application/json", "User-Agent": "MyApp/1.0", "Cookie": "session=abc123; user=john.doe@example.com", } sanitized = self.sanitizer.sanitize_headers(headers) self.assertEqual(sanitized["Authorization"], "Bearer SANITIZED") self.assertEqual(sanitized["API-Key"], "sk-SANITIZED") self.assertEqual(sanitized["Content-Type"], "application/json") self.assertEqual(sanitized["User-Agent"], "MyApp/1.0") self.assertIn("user@example.com", sanitized["Cookie"]) def test_nested_structure_sanitization(self): """Test sanitization of nested data structures.""" data = { "user": { "email": "john.doe@example.com", "api_key": "sk-proj-abcd1234567890ABCD1234567890abcd1234567890ABCD12", }, "tokens": [ "ghp_1234567890abcdefghijklmnopqrstuvwxyz", "Bearer sk-ant-abcd1234567890ABCD1234567890abcd1234567890ABCD12", ], "metadata": {"ip": "192.168.1.100", "phone": "(555) 123-4567"}, } sanitized = self.sanitizer.sanitize_value(data) self.assertEqual(sanitized["user"]["email"], "user@example.com") self.assertEqual(sanitized["user"]["api_key"], "sk-proj-SANITIZED") self.assertEqual(sanitized["tokens"][0], "gh_SANITIZED") self.assertEqual(sanitized["tokens"][1], "Bearer sk-ant-SANITIZED") self.assertEqual(sanitized["metadata"]["ip"], "0.0.0.0") self.assertEqual(sanitized["metadata"]["phone"], "(XXX) XXX-XXXX") def test_url_sanitization(self): """Test URL parameter sanitization.""" urls = [ ( "https://api.example.com/v1/users?api_key=sk-1234567890abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMN", "https://api.example.com/v1/users?api_key=SANITIZED", ), ( "https://example.com/login?token=ghp_1234567890abcdefghijklmnopqrstuvwxyz&user=test", "https://example.com/login?token=SANITIZED&user=test", ), ] for original, expected in urls: with self.subTest(url=original): result = self.sanitizer.sanitize_url(original) self.assertEqual(result, expected) def test_disable_sanitization(self): """Test that sanitization can be disabled.""" self.sanitizer.sanitize_enabled = False sensitive_data = "sk-proj-abcd1234567890ABCD1234567890abcd1234567890ABCD12" result = self.sanitizer.sanitize_string(sensitive_data) # Should return original when disabled self.assertEqual(result, sensitive_data) def test_custom_pattern(self): """Test adding custom PII patterns.""" # Add custom pattern for internal employee IDs custom_pattern = PIIPattern.create( name="employee_id", pattern=r"EMP\d{6}", replacement="EMP-REDACTED", description="Internal employee IDs" ) self.sanitizer.add_pattern(custom_pattern) text = "Employee EMP123456 has access to the system" result = self.sanitizer.sanitize_string(text) self.assertEqual(result, "Employee EMP-REDACTED has access to the system") if __name__ == "__main__": unittest.main() ================================================ FILE: tests/test_pip_detection_fix.py ================================================ """Tests for pip detection fix in run-server.sh script. This test file ensures our pip detection improvements work correctly and don't break existing functionality. """ import os import subprocess import tempfile from pathlib import Path import pytest class TestPipDetectionFix: """Test cases for issue #188: PIP is available but not recognized.""" def test_run_server_script_syntax_valid(self): """Test that run-server.sh has valid bash syntax.""" result = subprocess.run(["bash", "-n", "./run-server.sh"], capture_output=True, text=True) assert result.returncode == 0, f"Syntax error in run-server.sh: {result.stderr}" def test_run_server_has_proper_shebang(self): """Test that run-server.sh starts with proper shebang.""" content = Path("./run-server.sh").read_text() assert content.startswith("#!/bin/bash"), "Script missing proper bash shebang" def test_critical_functions_exist(self): """Test that all critical functions are defined in the script.""" content = Path("./run-server.sh").read_text() critical_functions = ["find_python", "setup_environment", "setup_venv", "install_dependencies", "bootstrap_pip"] for func in critical_functions: assert f"{func}()" in content, f"Critical function {func}() not found in script" def test_pip_detection_consistency_issue(self): """Test the specific issue: pip works in setup_venv but fails in install_dependencies. This test verifies that our fix ensures consistent Python executable paths. """ # Test that the get_venv_python_path function now returns absolute paths content = Path("./run-server.sh").read_text() # Check that get_venv_python_path includes our absolute path conversion logic assert "abs_venv_path" in content, "get_venv_python_path should use absolute paths" assert 'cd "$(dirname' in content, "Should convert to absolute path" # Test successful completion - our fix should make the script more robust result = subprocess.run(["bash", "-n", "./run-server.sh"], capture_output=True, text=True) assert result.returncode == 0, "Script should have valid syntax after our fix" def test_pip_detection_with_non_interactive_shell(self): """Test pip detection works in non-interactive shell environments. This addresses the contributor's suggestion about non-interactive shells not sourcing ~/.bashrc where pip PATH might be defined. """ # Test case for Git Bash on Windows and non-interactive Linux shells with tempfile.TemporaryDirectory() as temp_dir: # Create mock virtual environment structure venv_path = Path(temp_dir) / ".pal_venv" bin_path = venv_path / "bin" bin_path.mkdir(parents=True) # Create mock python executable python_exe = bin_path / "python" python_exe.write_text("#!/bin/bash\necho 'Python 3.12.3'\n") python_exe.chmod(0o755) # Create mock pip executable pip_exe = bin_path / "pip" pip_exe.write_text("#!/bin/bash\necho 'pip 23.0.1'\n") pip_exe.chmod(0o755) # Test that we can detect pip using explicit paths (not PATH) assert python_exe.exists(), "Mock python executable should exist" assert pip_exe.exists(), "Mock pip executable should exist" assert python_exe.is_file(), "Python should be a file" assert pip_exe.is_file(), "Pip should be a file" def test_enhanced_diagnostic_messages_included(self): """Test that our enhanced diagnostic messages are included in the script. Verify that the script contains the enhanced error diagnostics we added. """ content = Path("./run-server.sh").read_text() # Check that enhanced diagnostic information is present in the script expected_diagnostic_patterns = [ "Enhanced diagnostic information for debugging", "Diagnostic information:", "Python executable:", "Python executable exists:", "Python executable permissions:", "Virtual environment path:", "Virtual environment exists:", "Final diagnostic information:", ] for pattern in expected_diagnostic_patterns: assert pattern in content, f"Enhanced diagnostic pattern '{pattern}' should be in script" def test_setup_env_file_does_not_create_bsd_backup(self, tmp_path): """Ensure setup_env_file avoids creating .env'' artifacts (BSD sed behavior).""" script_path = Path("./run-server.sh").resolve() # Prepare temp workspace with example env env_example = Path(".env.example").read_text() target_example = tmp_path / ".env.example" target_example.write_text(env_example) # Run setup_env_file inside isolated shell session command = f""" set -e cd "{tmp_path}" source "{script_path}" setup_env_file """ env = os.environ.copy() subprocess.run(["bash", "-lc", command], check=True, env=env, text=True) artifacts = {p.name for p in tmp_path.glob(".env*")} assert ".env''" not in artifacts, "setup_env_file should not create BSD sed backup artifacts" assert ".env" in artifacts, ".env should be created from .env.example" if __name__ == "__main__": pytest.main([__file__, "-v"]) ================================================ FILE: tests/test_planner.py ================================================ """ Tests for the planner tool. """ from unittest.mock import patch import pytest from tools.models import ToolModelCategory from tools.planner import PlannerRequest, PlannerTool from tools.shared.exceptions import ToolExecutionError class TestPlannerTool: """Test suite for PlannerTool.""" def test_tool_metadata(self): """Test basic tool metadata and configuration.""" tool = PlannerTool() assert tool.get_name() == "planner" assert "sequential planning" in tool.get_description() assert tool.get_default_temperature() == 1.0 # TEMPERATURE_BALANCED assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING assert tool.get_default_thinking_mode() == "medium" def test_request_validation(self): """Test Pydantic request model validation.""" # Valid interactive step request step_request = PlannerRequest( step="Create database migration scripts", step_number=3, total_steps=10, next_step_required=True ) assert step_request.step == "Create database migration scripts" assert step_request.step_number == 3 assert step_request.next_step_required is True assert step_request.is_step_revision is False # default # Missing required fields should fail with pytest.raises(ValueError): PlannerRequest() # Missing all required fields with pytest.raises(ValueError): PlannerRequest(step="test") # Missing other required fields def test_input_schema_generation(self): """Test JSON schema generation for MCP client.""" tool = PlannerTool() schema = tool.get_input_schema() assert schema["type"] == "object" # Interactive planning fields assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "is_step_revision" in schema["properties"] assert "is_branch_point" in schema["properties"] assert "branch_id" in schema["properties"] assert "continuation_id" in schema["properties"] # Check that workflow-based planner includes model field and excludes some fields assert "model" in schema["properties"] # Workflow tools include model field assert "images" not in schema["properties"] # Excluded for planning assert "absolute_file_paths" not in schema["properties"] # Excluded for planning assert "temperature" not in schema["properties"] assert "thinking_mode" not in schema["properties"] # Check required fields assert "step" in schema["required"] assert "step_number" in schema["required"] assert "total_steps" in schema["required"] assert "next_step_required" in schema["required"] def test_model_category_for_planning(self): """Test that planner uses extended reasoning category.""" tool = PlannerTool() category = tool.get_model_category() # Planning needs deep thinking assert category == ToolModelCategory.EXTENDED_REASONING @pytest.mark.asyncio async def test_execute_first_step(self): """Test execute method for first planning step.""" tool = PlannerTool() arguments = { "step": "Plan a microservices migration for our monolithic e-commerce platform", "step_number": 1, "total_steps": 10, "next_step_required": True, } # Mock conversation memory functions and UUID generation with patch("utils.conversation_memory.uuid.uuid4") as mock_uuid: mock_uuid.return_value.hex = "test-uuid-123" mock_uuid.return_value.__str__ = lambda x: "test-uuid-123" with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 assert result[0].type == "text" # Parse the JSON response import json parsed_response = json.loads(result[0].text) assert parsed_response["step_number"] == 1 assert parsed_response["total_steps"] == 10 assert parsed_response["next_step_required"] is True assert parsed_response["continuation_id"] == "test-uuid-123" # For complex plans (>=5 steps) on first step, expect deep thinking pause assert parsed_response["status"] == "pause_for_deep_thinking" assert parsed_response["thinking_required"] is True assert "required_thinking" in parsed_response assert "MANDATORY: DO NOT call the planner tool again immediately" in parsed_response["next_steps"] @pytest.mark.asyncio async def test_execute_subsequent_step(self): """Test execute method for subsequent planning step.""" tool = PlannerTool() arguments = { "step": "Set up deployment configuration for each microservice", "step_number": 2, "total_steps": 8, "next_step_required": True, "continuation_id": "existing-uuid-456", } # Mock conversation memory functions with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 assert result[0].type == "text" # Parse the JSON response import json parsed_response = json.loads(result[0].text) assert parsed_response["step_number"] == 2 assert parsed_response["total_steps"] == 8 assert parsed_response["next_step_required"] is True assert parsed_response["continuation_id"] == "existing-uuid-456" # For complex plans (>=5 steps) on step 2, expect deep thinking pause assert parsed_response["status"] == "pause_for_deep_thinking" assert parsed_response["thinking_required"] is True assert "required_thinking" in parsed_response assert "STOP! Complex planning requires reflection between steps" in parsed_response["next_steps"] @pytest.mark.asyncio async def test_execute_with_continuation_context(self): """Test execute method with continuation that loads previous context.""" tool = PlannerTool() arguments = { "step": "Continue planning the deployment phase", "step_number": 1, # Step 1 with continuation_id loads context "total_steps": 8, "next_step_required": True, "continuation_id": "test-continuation-id", } # Mock thread with completed plan from utils.conversation_memory import ConversationTurn, ThreadContext mock_turn = ConversationTurn( role="assistant", content='{"status": "planning_success", "planning_complete": true, "plan_summary": "COMPLETE PLAN: Authentication system with 3 steps completed"}', tool_name="planner", model_name="claude-planner", timestamp="2024-01-01T00:00:00Z", ) mock_thread = ThreadContext( thread_id="test-id", tool_name="planner", turns=[mock_turn], created_at="2024-01-01T00:00:00Z", last_updated_at="2024-01-01T00:00:00Z", initial_context={}, ) with patch("utils.conversation_memory.get_thread", return_value=mock_thread): with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 response_text = result[0].text # Should include previous plan context in JSON import json parsed_response = json.loads(response_text) # Check that the continuation works (workflow architecture handles context differently) assert parsed_response["step_number"] == 1 assert parsed_response["continuation_id"] == "test-continuation-id" assert parsed_response["next_step_required"] is True @pytest.mark.asyncio async def test_execute_final_step(self): """Test execute method for final planning step.""" tool = PlannerTool() arguments = { "step": "Deploy and monitor the new system", "step_number": 10, "total_steps": 10, "next_step_required": False, # Final step "continuation_id": "test-uuid-789", } # Mock conversation memory functions with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 response_text = result[0].text # Parse the structured JSON response import json parsed_response = json.loads(response_text) # Check final step structure assert parsed_response["status"] == "planning_complete" assert parsed_response["step_number"] == 10 assert parsed_response["planning_complete"] is True assert "plan_summary" in parsed_response assert "COMPLETE PLAN:" in parsed_response["plan_summary"] @pytest.mark.asyncio async def test_execute_with_branching(self): """Test execute method with branching.""" tool = PlannerTool() arguments = { "step": "Use Kubernetes for orchestration", "step_number": 4, "total_steps": 10, "next_step_required": True, "is_branch_point": True, "branch_from_step": 3, "branch_id": "cloud-native-path", "continuation_id": "test-uuid-branch", } # Mock conversation memory functions with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 response_text = result[0].text # Parse the JSON response import json parsed_response = json.loads(response_text) assert parsed_response["metadata"]["branches"] == ["cloud-native-path"] assert "cloud-native-path" in str(tool.branches) @pytest.mark.asyncio async def test_execute_with_revision(self): """Test execute method with step revision.""" tool = PlannerTool() arguments = { "step": "Revise API design to use GraphQL instead of REST", "step_number": 3, "total_steps": 8, "next_step_required": True, "is_step_revision": True, "revises_step_number": 2, "continuation_id": "test-uuid-revision", } # Mock conversation memory functions with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 response_text = result[0].text # Parse the JSON response import json parsed_response = json.loads(response_text) assert parsed_response["step_number"] == 3 assert parsed_response["next_step_required"] is True assert parsed_response["metadata"]["is_step_revision"] is True assert parsed_response["metadata"]["revises_step_number"] == 2 # Check that step data was stored in history assert len(tool.work_history) > 0 latest_step = tool.work_history[-1] assert latest_step["is_step_revision"] is True assert latest_step["revises_step_number"] == 2 @pytest.mark.asyncio async def test_execute_adjusts_total_steps(self): """Test execute method adjusts total steps when current step exceeds estimate.""" tool = PlannerTool() arguments = { "step": "Additional step discovered during planning", "step_number": 8, "total_steps": 5, # Current step exceeds total "next_step_required": True, "continuation_id": "test-uuid-adjust", } # Mock conversation memory functions with patch("utils.conversation_memory.add_turn"): result = await tool.execute(arguments) # Should return a list with TextContent assert len(result) == 1 response_text = result[0].text # Parse the JSON response import json parsed_response = json.loads(response_text) # Total steps should be adjusted to match current step assert parsed_response["total_steps"] == 8 assert parsed_response["step_number"] == 8 assert parsed_response["status"] == "pause_for_planning" @pytest.mark.asyncio async def test_execute_error_handling(self): """Test execute method error handling.""" tool = PlannerTool() # Invalid arguments - missing required fields arguments = { "step": "Invalid request" # Missing required fields: step_number, total_steps, next_step_required } with pytest.raises(ToolExecutionError) as exc_info: await tool.execute(arguments) import json parsed_response = json.loads(exc_info.value.payload) assert parsed_response["status"] == "planner_failed" assert "error" in parsed_response @pytest.mark.asyncio async def test_execute_step_history_tracking(self): """Test that execute method properly tracks step history.""" tool = PlannerTool() # Execute multiple steps step1_args = {"step": "First step", "step_number": 1, "total_steps": 3, "next_step_required": True} step2_args = { "step": "Second step", "step_number": 2, "total_steps": 3, "next_step_required": True, "continuation_id": "test-uuid-history", } # Mock conversation memory functions with patch("utils.conversation_memory.create_thread", return_value="test-uuid-history"): with patch("utils.conversation_memory.add_turn"): await tool.execute(step1_args) await tool.execute(step2_args) # Should have tracked both steps assert len(tool.work_history) == 2 assert tool.work_history[0]["step"] == "First step" assert tool.work_history[1]["step"] == "Second step" # Integration test class TestPlannerToolIntegration: """Integration tests for planner tool.""" def setup_method(self): """Set up model context for integration tests.""" from utils.model_context import ModelContext self.tool = PlannerTool() self.tool._model_context = ModelContext("flash") # Test model @pytest.mark.asyncio async def test_interactive_planning_flow(self): """Test complete interactive planning flow.""" arguments = { "step": "Plan a complete system redesign", "step_number": 1, "total_steps": 5, "next_step_required": True, } # Mock conversation memory functions and UUID generation with patch("utils.conversation_memory.uuid.uuid4") as mock_uuid: mock_uuid.return_value.hex = "test-flow-uuid" mock_uuid.return_value.__str__ = lambda x: "test-flow-uuid" with patch("utils.conversation_memory.add_turn"): result = await self.tool.execute(arguments) # Verify response structure assert len(result) == 1 response_text = result[0].text # Parse the JSON response import json parsed_response = json.loads(response_text) assert parsed_response["step_number"] == 1 assert parsed_response["total_steps"] == 5 assert parsed_response["continuation_id"] == "test-flow-uuid" # For complex plans (>=5 steps) on first step, expect deep thinking pause assert parsed_response["status"] == "pause_for_deep_thinking" assert parsed_response["thinking_required"] is True @pytest.mark.asyncio async def test_simple_planning_flow(self): """Test simple planning flow without deep thinking pauses.""" arguments = { "step": "Plan a simple feature update", "step_number": 1, "total_steps": 3, # Simple plan < 5 steps "next_step_required": True, } # Mock conversation memory functions and UUID generation with patch("utils.conversation_memory.uuid.uuid4") as mock_uuid: mock_uuid.return_value.hex = "test-simple-uuid" mock_uuid.return_value.__str__ = lambda x: "test-simple-uuid" with patch("utils.conversation_memory.add_turn"): result = await self.tool.execute(arguments) # Verify response structure assert len(result) == 1 response_text = result[0].text # Parse the JSON response import json parsed_response = json.loads(response_text) assert parsed_response["step_number"] == 1 assert parsed_response["total_steps"] == 3 assert parsed_response["continuation_id"] == "test-simple-uuid" # For simple plans (< 5 steps), expect normal flow without deep thinking pause assert parsed_response["status"] == "pause_for_planning" assert "thinking_required" not in parsed_response assert "Continue with step 2" in parsed_response["next_steps"] ================================================ FILE: tests/test_precommit_workflow.py ================================================ """ Unit tests for the workflow-based PrecommitTool Tests the core functionality of the precommit workflow tool including: - Tool metadata and configuration - Request model validation - Workflow step handling - Tool categorization """ import pytest from tools.models import ToolModelCategory from tools.precommit import PrecommitRequest, PrecommitTool class TestPrecommitWorkflowTool: """Test suite for the workflow-based PrecommitTool""" def test_tool_metadata(self): """Test basic tool metadata""" tool = PrecommitTool() assert tool.get_name() == "precommit" assert "git changes" in tool.get_description() assert "systematic analysis" in tool.get_description() def test_tool_model_category(self): """Test that precommit tool uses extended reasoning category""" tool = PrecommitTool() assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING def test_default_temperature(self): """Test analytical temperature setting""" tool = PrecommitTool() temp = tool.get_default_temperature() # Should be analytical temperature (now 1.0) assert temp == 1.0 def test_request_model_basic_validation(self): """Test basic request model validation""" # Valid minimal workflow request request = PrecommitRequest( step="Initial validation step", step_number=1, total_steps=3, next_step_required=True, findings="Initial findings", path="/test/repo", # Required for step 1 ) assert request.step == "Initial validation step" assert request.step_number == 1 assert request.total_steps == 3 assert request.next_step_required is True assert request.findings == "Initial findings" assert request.path == "/test/repo" def test_request_model_step_one_validation(self): """Test that step 1 requires path field""" # Step 1 without path should fail with pytest.raises(ValueError, match="Step 1 requires 'path' field"): PrecommitRequest( step="Initial validation step", step_number=1, total_steps=3, next_step_required=True, findings="Initial findings", # Missing path for step 1 ) def test_request_model_later_steps_no_path_required(self): """Test that later steps don't require path""" # Step 2+ without path should be fine request = PrecommitRequest( step="Continued validation", step_number=2, total_steps=3, next_step_required=True, findings="Detailed findings", # No path needed for step 2+ ) assert request.step_number == 2 assert request.path is None def test_request_model_optional_fields(self): """Test optional workflow fields""" request = PrecommitRequest( step="Validation with optional fields", step_number=1, total_steps=2, next_step_required=False, findings="Comprehensive findings", path="/test/repo", precommit_type="external", files_checked=["/file1.py", "/file2.py"], relevant_files=["/file1.py"], relevant_context=["function_name", "class_name"], issues_found=[{"severity": "medium", "description": "Test issue"}], images=["/screenshot.png"], ) assert request.precommit_type == "external" assert len(request.files_checked) == 2 assert len(request.relevant_files) == 1 assert len(request.relevant_context) == 2 assert len(request.issues_found) == 1 assert len(request.images) == 1 def test_precommit_specific_fields(self): """Test precommit-specific configuration fields""" request = PrecommitRequest( step="Validation with git config", step_number=1, total_steps=1, next_step_required=False, findings="Complete validation", path="/repo", compare_to="main", include_staged=True, include_unstaged=False, focus_on="security issues", severity_filter="high", ) assert request.compare_to == "main" assert request.include_staged is True assert request.include_unstaged is False assert request.focus_on == "security issues" assert request.severity_filter == "high" def test_precommit_type_validation(self): """Test precommit type validation""" valid_types = ["external", "internal"] for precommit_type in valid_types: request = PrecommitRequest( step="Test precommit type", step_number=1, total_steps=1, next_step_required=False, findings="Test findings", path="/repo", precommit_type=precommit_type, ) assert request.precommit_type == precommit_type # Test default is external request = PrecommitRequest( step="Test default type", step_number=1, total_steps=1, next_step_required=False, findings="Test findings", path="/repo", ) assert request.precommit_type == "external" def test_severity_filter_options(self): """Test severity filter validation""" valid_severities = ["critical", "high", "medium", "low", "all"] for severity in valid_severities: request = PrecommitRequest( step="Test severity filter", step_number=1, total_steps=1, next_step_required=False, findings="Test findings", path="/repo", severity_filter=severity, ) assert request.severity_filter == severity def test_input_schema_generation(self): """Test that input schema is generated correctly""" tool = PrecommitTool() schema = tool.get_input_schema() # Check basic schema structure assert schema["type"] == "object" assert "properties" in schema assert "required" in schema # Check required fields are present required_fields = {"step", "step_number", "total_steps", "next_step_required", "findings"} assert all(field in schema["properties"] for field in required_fields) # Check model field is present and configured correctly assert "model" in schema["properties"] assert schema["properties"]["model"]["type"] == "string" def test_workflow_request_model_method(self): """Test get_workflow_request_model returns correct model""" tool = PrecommitTool() assert tool.get_workflow_request_model() == PrecommitRequest assert tool.get_request_model() == PrecommitRequest def test_system_prompt_integration(self): """Test system prompt integration""" tool = PrecommitTool() system_prompt = tool.get_system_prompt() # Should get the precommit prompt assert isinstance(system_prompt, str) assert len(system_prompt) > 0 ================================================ FILE: tests/test_prompt_regression.py ================================================ """ Integration tests to ensure normal prompt handling works with real API calls. This test module verifies that all tools continue to work correctly with normal-sized prompts using real integration testing instead of mocks. INTEGRATION TESTS: These tests are marked with @pytest.mark.integration and make real API calls. They use the local-llama model which is FREE and runs locally via Ollama. Prerequisites: - Ollama installed and running locally - CUSTOM_API_URL environment variable set to your Ollama endpoint (e.g., http://localhost:11434) - local-llama model available through custom provider configuration - No API keys required - completely FREE to run unlimited times! Running Tests: - All tests (including integration): pytest tests/test_prompt_regression.py - Unit tests only: pytest tests/test_prompt_regression.py -m "not integration" - Integration tests only: pytest tests/test_prompt_regression.py -m "integration" Note: Integration tests skip gracefully if CUSTOM_API_URL is not set. They are excluded from CI/CD but run by default locally when Ollama is configured. """ import json import os import tempfile import pytest # Load environment variables from .env file from dotenv import load_dotenv from tools.analyze import AnalyzeTool from tools.chat import ChatTool from tools.codereview import CodeReviewTool from tools.thinkdeep import ThinkDeepTool load_dotenv() # Check if CUSTOM_API_URL is available for local-llama CUSTOM_API_AVAILABLE = os.getenv("CUSTOM_API_URL") is not None def skip_if_no_custom_api(): """Helper to skip integration tests if CUSTOM_API_URL is not available.""" if not CUSTOM_API_AVAILABLE: pytest.skip( "CUSTOM_API_URL not set. To run integration tests with local-llama, ensure CUSTOM_API_URL is set in .env file (e.g., http://localhost:11434/v1)" ) class TestPromptIntegration: """Integration test suite for normal prompt handling with real API calls.""" @pytest.mark.integration @pytest.mark.asyncio async def test_chat_normal_prompt(self): """Test chat tool with normal prompt using real API.""" skip_if_no_custom_api() tool = ChatTool() result = await tool.execute( { "prompt": "Explain Python decorators in one sentence", "model": "local-llama", # Use available model for integration tests "working_directory_absolute_path": tempfile.gettempdir(), } ) assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "content" in output assert len(output["content"]) > 0 @pytest.mark.integration @pytest.mark.asyncio async def test_chat_with_files(self): """Test chat tool with absolute_file_paths parameter using real API.""" skip_if_no_custom_api() tool = ChatTool() # Create a temporary Python file for testing with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: f.write( """ def hello_world(): \"\"\"A simple hello world function.\"\"\" return "Hello, World!" if __name__ == "__main__": print(hello_world()) """ ) temp_file = f.name try: result = await tool.execute( { "prompt": "What does this Python code do?", "absolute_file_paths": [temp_file], "model": "local-llama", "working_directory_absolute_path": tempfile.gettempdir(), } ) assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "content" in output # Should mention the hello world function assert "hello" in output["content"].lower() or "function" in output["content"].lower() finally: # Clean up temp file os.unlink(temp_file) @pytest.mark.integration @pytest.mark.asyncio async def test_thinkdeep_normal_analysis(self): """Test thinkdeep tool with normal analysis using real API.""" skip_if_no_custom_api() tool = ThinkDeepTool() result = await tool.execute( { "step": "I think we should use a cache for performance", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Building a high-traffic API - considering scalability and reliability", "problem_context": "Building a high-traffic API", "focus_areas": ["scalability", "reliability"], "model": "local-llama", } ) assert len(result) == 1 output = json.loads(result[0].text) # ThinkDeep workflow tool should process the analysis assert "status" in output assert output["status"] in ["calling_expert_analysis", "analysis_complete", "pause_for_investigation"] @pytest.mark.integration @pytest.mark.asyncio async def test_codereview_normal_review(self): """Test codereview tool with workflow inputs using real API.""" skip_if_no_custom_api() tool = CodeReviewTool() # Create a temporary Python file for testing with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: f.write( """ def process_user_input(user_input): # Potentially unsafe code for demonstration query = f"SELECT * FROM users WHERE name = '{user_input}'" return query def main(): user_name = input("Enter name: ") result = process_user_input(user_name) print(result) """ ) temp_file = f.name try: result = await tool.execute( { "step": "Initial code review investigation - examining security vulnerabilities", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": "Found security issues in code", "relevant_files": [temp_file], "review_type": "security", "focus_on": "Look for SQL injection vulnerabilities", "model": "local-llama", } ) assert len(result) == 1 output = json.loads(result[0].text) assert "status" in output assert output["status"] in ["pause_for_code_review", "calling_expert_analysis"] finally: # Clean up temp file os.unlink(temp_file) # NOTE: Precommit test has been removed because the precommit tool has been # refactored to use a workflow-based pattern instead of accepting simple prompt/path fields. # The new precommit tool requires workflow fields like: step, step_number, total_steps, # next_step_required, findings, etc. See simulator_tests/test_precommitworkflow_validation.py # for comprehensive workflow testing. # NOTE: Debug tool test has been commented out because the debug tool has been # refactored to use a self-investigation pattern instead of accepting prompt/error_context fields. # The new debug tool requires fields like: step, step_number, total_steps, next_step_required, findings # @pytest.mark.asyncio # async def test_debug_normal_error(self, mock_model_response): # """Test debug tool with normal error description.""" # tool = DebugIssueTool() # # with patch.object(tool, "get_model_provider") as mock_get_provider: # mock_provider = MagicMock() # mock_provider.get_provider_type.return_value = MagicMock(value="google") # mock_provider.supports_thinking_mode.return_value = False # mock_provider.generate_content.return_value = mock_model_response( # "Root cause: The variable is undefined. Fix: Initialize it..." # ) # mock_get_provider.return_value = mock_provider # # result = await tool.execute( # { # "prompt": "TypeError: Cannot read property 'name' of undefined", # "error_context": "at line 42 in user.js\n console.log(user.name)", # "runtime_info": "Node.js v16.14.0", # } # ) # # assert len(result) == 1 # output = json.loads(result[0].text) # assert output["status"] in ["success", "continuation_available"] # assert "Next Steps:" in output["content"] # assert "Root cause" in output["content"] @pytest.mark.integration @pytest.mark.asyncio async def test_analyze_normal_question(self): """Test analyze tool with normal question using real API.""" skip_if_no_custom_api() tool = AnalyzeTool() # Create a temporary Python file demonstrating MVC pattern with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: f.write( """ # Model class User: def __init__(self, name, email): self.name = name self.email = email # View class UserView: def display_user(self, user): return f"User: {user.name} ({user.email})" # Controller class UserController: def __init__(self, model, view): self.model = model self.view = view def get_user_display(self): return self.view.display_user(self.model) """ ) temp_file = f.name try: result = await tool.execute( { "step": "What design patterns are used in this codebase?", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial architectural analysis", "relevant_files": [temp_file], "analysis_type": "architecture", "model": "local-llama", } ) assert len(result) == 1 output = json.loads(result[0].text) assert "status" in output # Workflow analyze tool should process the analysis assert output["status"] in ["calling_expert_analysis", "pause_for_investigation"] finally: # Clean up temp file os.unlink(temp_file) @pytest.mark.integration @pytest.mark.asyncio async def test_empty_optional_fields(self): """Test tools work with empty optional fields using real API.""" skip_if_no_custom_api() tool = ChatTool() # Test with no absolute_file_paths parameter result = await tool.execute( { "prompt": "Hello", "model": "local-llama", "working_directory_absolute_path": tempfile.gettempdir(), } ) assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "content" in output @pytest.mark.integration @pytest.mark.asyncio async def test_thinking_modes_work(self): """Test that thinking modes are properly passed through using real API.""" skip_if_no_custom_api() tool = ChatTool() result = await tool.execute( { "prompt": "Explain quantum computing briefly", "thinking_mode": "low", "temperature": 0.8, "model": "local-llama", "working_directory_absolute_path": tempfile.gettempdir(), } ) assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "content" in output # Should contain some quantum-related content assert "quantum" in output["content"].lower() or "computing" in output["content"].lower() @pytest.mark.integration @pytest.mark.asyncio async def test_special_characters_in_prompts(self): """Test prompts with special characters work correctly using real API.""" skip_if_no_custom_api() tool = ChatTool() special_prompt = ( 'Test with "quotes" and\nnewlines\tand tabs. Please just respond with the number that is the answer to 1+1.' ) result = await tool.execute( { "prompt": special_prompt, "model": "local-llama", "working_directory_absolute_path": tempfile.gettempdir(), } ) assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "content" in output # Should handle the special characters without crashing - the exact content doesn't matter as much as not failing assert len(output["content"]) > 0 @pytest.mark.integration @pytest.mark.asyncio async def test_mixed_file_paths(self): """Test handling of various file path formats using real API.""" skip_if_no_custom_api() tool = AnalyzeTool() # Create multiple temporary files to test different path formats temp_files = [] try: # Create first file with tempfile.NamedTemporaryFile(mode="w", suffix=".py", delete=False) as f: f.write("def function_one(): pass") temp_files.append(f.name) # Create second file with tempfile.NamedTemporaryFile(mode="w", suffix=".js", delete=False) as f: f.write("function functionTwo() { return 'hello'; }") temp_files.append(f.name) result = await tool.execute( { "step": "Analyze these files", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial file analysis", "relevant_files": temp_files, "model": "local-llama", } ) assert len(result) == 1 output = json.loads(result[0].text) assert "status" in output # Should process the files assert output["status"] in [ "calling_expert_analysis", "pause_for_investigation", "files_required_to_continue", ] finally: # Clean up temp files for temp_file in temp_files: if os.path.exists(temp_file): os.unlink(temp_file) @pytest.mark.integration @pytest.mark.asyncio async def test_unicode_content(self): """Test handling of unicode content in prompts using real API.""" skip_if_no_custom_api() tool = ChatTool() unicode_prompt = "Explain what these mean: 你好世界 (Chinese) and مرحبا بالعالم (Arabic)" result = await tool.execute( { "prompt": unicode_prompt, "model": "local-llama", "working_directory_absolute_path": tempfile.gettempdir(), } ) assert len(result) == 1 output = json.loads(result[0].text) assert output["status"] in ["success", "continuation_available"] assert "content" in output # Should mention hello or world or greeting in some form (including French equivalents) content_lower = output["content"].lower() assert ( "hello" in content_lower or "world" in content_lower or "greeting" in content_lower or "bonjour" in content_lower # French: hello or "monde" in content_lower # French: world or "salut" in content_lower # French: greeting ) if __name__ == "__main__": # Run integration tests by default when called directly pytest.main([__file__, "-v", "-m", "integration"]) ================================================ FILE: tests/test_prompt_size_limit_bug_fix.py ================================================ """ Test for the prompt size limit bug fix. This test verifies that SimpleTool correctly validates only the original user prompt when conversation history is embedded, rather than validating the full enhanced prompt. """ from tools.chat import ChatTool from tools.shared.base_models import ToolRequest class TestPromptSizeLimitBugFix: """Test that the prompt size limit bug is fixed""" def test_prompt_size_validation_with_conversation_history(self): """Test that prompt size validation uses original prompt when conversation history is embedded""" # Create a ChatTool instance tool = ChatTool() # Simulate a short user prompt (should not trigger size limit) short_user_prompt = "Thanks for the help!" # Simulate conversation history (large content) conversation_history = "=== CONVERSATION HISTORY ===\n" + ("Previous conversation content. " * 5000) # Simulate enhanced prompt with conversation history (what server.py creates) enhanced_prompt = f"{conversation_history}\n\n=== NEW USER INPUT ===\n{short_user_prompt}" # Simulate server.py behavior: store original prompt in _current_arguments tool._current_arguments = { "prompt": enhanced_prompt, # Enhanced with history "_original_user_prompt": short_user_prompt, # Original user input (our fix) "model": "local-llama", } # Test the hook method directly validation_content = tool.get_prompt_content_for_size_validation(enhanced_prompt) # Should return the original short prompt, not the enhanced prompt assert validation_content == short_user_prompt assert len(validation_content) == len(short_user_prompt) assert len(validation_content) < 1000 # Much smaller than enhanced prompt # Verify the enhanced prompt would have triggered the bug assert len(enhanced_prompt) > 50000 # This would trigger size limit # Test that size check passes with the original prompt size_check = tool.check_prompt_size(validation_content) assert size_check is None # No size limit error # Test that size check would fail with enhanced prompt size_check_enhanced = tool.check_prompt_size(enhanced_prompt) assert size_check_enhanced is not None # Would trigger size limit assert size_check_enhanced["status"] == "resend_prompt" def test_prompt_size_validation_without_original_prompt(self): """Test fallback behavior when no original prompt is stored (new conversations)""" tool = ChatTool() user_content = "Regular prompt without conversation history" # No _current_arguments (new conversation scenario) tool._current_arguments = None # Should fall back to validating the full user content validation_content = tool.get_prompt_content_for_size_validation(user_content) assert validation_content == user_content def test_prompt_size_validation_with_missing_original_prompt(self): """Test fallback when _current_arguments exists but no _original_user_prompt""" tool = ChatTool() user_content = "Regular prompt without conversation history" # _current_arguments exists but no _original_user_prompt field tool._current_arguments = { "prompt": user_content, "model": "local-llama", # No _original_user_prompt field } # Should fall back to validating the full user content validation_content = tool.get_prompt_content_for_size_validation(user_content) assert validation_content == user_content def test_base_tool_default_behavior(self): """Test that BaseTool's default implementation validates full content""" from tools.shared.base_tool import BaseTool # Create a minimal tool implementation for testing class TestTool(BaseTool): def get_name(self) -> str: return "test" def get_description(self) -> str: return "Test tool" def get_input_schema(self) -> dict: return {} def get_request_model(self): return ToolRequest def get_system_prompt(self) -> str: return "Test system prompt" async def prepare_prompt(self, request) -> str: return "Test prompt" async def execute(self, arguments: dict) -> list: return [] tool = TestTool() user_content = "Test content" # Default implementation should return the same content validation_content = tool.get_prompt_content_for_size_validation(user_content) assert validation_content == user_content ================================================ FILE: tests/test_provider_retry_logic.py ================================================ """Tests covering shared retry behaviour for providers.""" from types import SimpleNamespace import pytest from providers.openai import OpenAIModelProvider def _mock_chat_response(content: str = "retry success") -> SimpleNamespace: """Create a minimal chat completion response for tests.""" usage = SimpleNamespace(prompt_tokens=10, completion_tokens=5, total_tokens=15) message = SimpleNamespace(content=content) choice = SimpleNamespace(message=message, finish_reason="stop") return SimpleNamespace(choices=[choice], model="gpt-4.1", id="resp-1", created=123, usage=usage) def test_openai_provider_retries_on_transient_error(monkeypatch): """Provider should retry once for retryable errors and eventually succeed.""" monkeypatch.setattr("providers.base.time.sleep", lambda _: None) provider = OpenAIModelProvider(api_key="test-key") attempts = {"count": 0} def create_completion(**kwargs): attempts["count"] += 1 if attempts["count"] == 1: raise RuntimeError("temporary network interruption") return _mock_chat_response("second attempt response") provider._client = SimpleNamespace( chat=SimpleNamespace(completions=SimpleNamespace(create=create_completion)), responses=SimpleNamespace(create=lambda **_: None), ) result = provider.generate_content("hello", "gpt-4.1") assert attempts["count"] == 2, "Expected a retry before succeeding" assert result.content == "second attempt response" def test_openai_provider_bails_on_non_retryable_error(monkeypatch): """Provider should stop immediately when the error is marked non-retryable.""" monkeypatch.setattr("providers.base.time.sleep", lambda _: None) provider = OpenAIModelProvider(api_key="test-key") attempts = {"count": 0} def create_completion(**kwargs): attempts["count"] += 1 raise RuntimeError("context length exceeded 429") provider._client = SimpleNamespace( chat=SimpleNamespace(completions=SimpleNamespace(create=create_completion)), responses=SimpleNamespace(create=lambda **_: None), ) monkeypatch.setattr( OpenAIModelProvider, "_is_error_retryable", lambda self, error: False, ) with pytest.raises(RuntimeError) as excinfo: provider.generate_content("hello", "gpt-4.1") assert "after 1 attempt" in str(excinfo.value) assert attempts["count"] == 1 ================================================ FILE: tests/test_provider_routing_bugs.py ================================================ """ Tests that reproduce and prevent provider routing bugs. These tests specifically cover bugs that were found in production: 1. Fallback provider registration bypassing API key validation 2. OpenRouter alias-based restrictions not working 3. Double restriction filtering 4. Missing provider_used metadata """ import os from unittest.mock import Mock import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from tools.chat import ChatTool from tools.shared.base_models import ToolRequest class MockRequest(ToolRequest): """Mock request for testing.""" pass class TestProviderRoutingBugs: """Test cases that reproduce provider routing bugs.""" def setup_method(self): """Set up clean state before each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() def teardown_method(self): """Clean up after each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None @pytest.mark.no_mock_provider def test_fallback_routing_bug_reproduction(self): """ CRITICAL BUG TEST: Reproduce the bug where fallback logic auto-registers Google provider for 'flash' model without checking GEMINI_API_KEY. Scenario: User has only OPENROUTER_API_KEY, requests 'flash' model. Bug: System incorrectly uses Google provider instead of OpenRouter. """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up bug scenario: only OpenRouter API key os.environ.pop("GEMINI_API_KEY", None) # No Google API key os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("XAI_API_KEY", None) os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) # Clear any restrictions os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" # Register only OpenRouter provider (like in server.py:configure_providers) from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Create tool to test fallback logic tool = ChatTool() # Test: Request 'flash' model - should use OpenRouter, not auto-register Google provider = tool.get_model_provider("flash") # ASSERTION: Should get OpenRouter provider, not Google assert provider is not None, "Should find a provider for 'flash' model" assert provider.get_provider_type() == ProviderType.OPENROUTER, ( f"Expected OpenRouter provider for 'flash' model with only OPENROUTER_API_KEY set, " f"but got {provider.get_provider_type()}" ) # Test common aliases that should all route to OpenRouter test_models = ["flash", "pro", "o3", "o3-mini", "o4-mini"] for model_name in test_models: provider = tool.get_model_provider(model_name) assert provider is not None, f"Should find provider for '{model_name}'" assert provider.get_provider_type() == ProviderType.OPENROUTER, ( f"Model '{model_name}' should route to OpenRouter when only OPENROUTER_API_KEY is set, " f"but got {provider.get_provider_type()}" ) finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_fallback_should_not_register_without_api_key(self): """ Test that fallback logic correctly validates API keys before registering providers. This test ensures the fix in tools/base.py:2067-2081 works correctly. """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up scenario: NO API keys at all for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: os.environ.pop(key, None) # Create tool to test fallback logic tool = ChatTool() # Test: Request 'flash' model with no API keys - should fail gracefully with pytest.raises(ValueError, match="Model 'flash' is not available"): tool.get_model_provider("flash") # Test: Request 'o3' model with no API keys - should fail gracefully with pytest.raises(ValueError, match="Model 'o3' is not available"): tool.get_model_provider("o3") # Verify no providers were auto-registered registry = ModelProviderRegistry() assert len(registry._providers) == 0, "No providers should be registered without API keys" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_mixed_api_keys_correct_routing(self): """ Test that when multiple API keys are available, provider routing works correctly. """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up scenario: Multiple API keys available os.environ["GEMINI_API_KEY"] = "test-gemini-key" os.environ["OPENAI_API_KEY"] = "test-openai-key" os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" os.environ.pop("XAI_API_KEY", None) os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) # Clear any restrictions # Register providers in priority order (like server.py) from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) tool = ChatTool() # Test priority order: Native APIs should be preferred over OpenRouter # Google models should use Google provider flash_provider = tool.get_model_provider("flash") assert ( flash_provider.get_provider_type() == ProviderType.GOOGLE ), "When both Google and OpenRouter API keys are available, 'flash' should prefer Google provider" # OpenAI models should use OpenAI provider o3_provider = tool.get_model_provider("o3") assert ( o3_provider.get_provider_type() == ProviderType.OPENAI ), "When both OpenAI and OpenRouter API keys are available, 'o3' should prefer OpenAI provider" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value class TestOpenRouterAliasRestrictions: """Test OpenRouter model restrictions with aliases - reproduces restriction bug.""" def setup_method(self): """Set up clean state before each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() def teardown_method(self): """Clean up after each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None @pytest.mark.no_mock_provider def test_openrouter_alias_restrictions_bug_reproduction(self): """ CRITICAL BUG TEST: Reproduce the bug where OpenRouter restrictions with aliases resulted in "no models available" error. Bug scenario: OPENROUTER_ALLOWED_MODELS=o3-mini,pro,flash,o4-mini,o3 Expected: 5 models available (aliases resolve to full names) Bug: 0 models available due to alias resolution failure """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up bug scenario: Only OpenRouter with alias-based restrictions os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("XAI_API_KEY", None) os.environ["OPENROUTER_API_KEY"] = "test-key" os.environ["OPENROUTER_ALLOWED_MODELS"] = "o3-mini,pro,gpt4.1,flash,o4-mini,o3" # User's exact config # Register OpenRouter provider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Test: Get available models with restrictions available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) # ASSERTION: Should have models available, not 0 assert len(available_models) > 0, ( f"Expected models available with alias restrictions 'o3-mini,pro,gpt4.1,flash,o4-mini,o3', " f"but got {len(available_models)} models. Available: {list(available_models.keys())}" ) # Expected aliases that should resolve to models: # o3-mini -> openai/o3-mini # pro -> google/gemini-2.5-pro # flash -> google/gemini-2.5-flash # o4-mini -> openai/o4-mini # o3 -> openai/o3 # gpt4.1 -> should not exist (expected to be filtered out) expected_models = {"o3-mini", "pro", "flash", "o4-mini", "o3"} available_model_names = set(available_models.keys()) # Should have at least the resolvable aliases (5 out of 6) assert len(available_model_names) >= 5, ( f"Expected at least 5 models from alias restrictions, got {len(available_model_names)}: " f"{available_model_names}" ) # Check that expected models are present missing_models = expected_models - available_model_names assert len(missing_models) == 0, ( f"Missing expected models from alias restrictions: {missing_models}. " f"Available: {available_model_names}" ) finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_openrouter_mixed_alias_and_full_names(self): """Test OpenRouter restrictions with mix of aliases and full model names.""" # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up mixed restrictions: some aliases, some full names os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("XAI_API_KEY", None) os.environ["OPENROUTER_API_KEY"] = "test-key" os.environ["OPENROUTER_ALLOWED_MODELS"] = "o3-mini,anthropic/claude-opus-4.1,flash" # Register OpenRouter provider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Test: Get available models available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) expected_models = { "o3-mini", # alias "openai/o3-mini", # canonical "anthropic/claude-opus-4.1", # full name "flash", # alias "google/gemini-2.5-flash", # canonical } available_model_names = set(available_models.keys()) assert ( available_model_names == expected_models ), f"Expected models {expected_models}, got {available_model_names}" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value class TestProviderMetadataBug: """Test for missing provider_used metadata bug.""" def test_provider_used_metadata_included(self): """ Test that provider_used metadata is included in tool responses. Bug: Only model_used was included, provider_used was missing. Fix: Added provider_used field in tools/base.py """ # Test the actual _parse_response method with model_info tool = ChatTool() # Create mock provider mock_provider = Mock() mock_provider.get_provider_type.return_value = ProviderType.OPENROUTER # Create model_info like the execute method does model_info = {"provider": mock_provider, "model_name": "test-model", "model_response": Mock()} # Test _parse_response directly with a simple response request = MockRequest() result = tool._parse_response("Test response", request, model_info) # Verify metadata includes both model_used and provider_used assert hasattr(result, "metadata"), "ToolOutput should have metadata" assert result.metadata is not None, "Metadata should not be None" assert "model_used" in result.metadata, "Metadata should include model_used" assert result.metadata["model_used"] == "test-model", "model_used should be correct" assert "provider_used" in result.metadata, "Metadata should include provider_used (bug fix)" assert result.metadata["provider_used"] == "openrouter", "provider_used should be correct" ================================================ FILE: tests/test_provider_utf8.py ================================================ """ Unit tests to validate UTF-8 encoding in providers and integration with language models. """ import json import os import unittest from unittest.mock import Mock, patch import pytest from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.shared import ProviderType class TestProviderUTF8Encoding(unittest.TestCase): """Tests for UTF-8 encoding in providers.""" def setUp(self): """Test setup.""" self.original_locale = os.getenv("LOCALE") def tearDown(self): """Cleanup after tests.""" if self.original_locale is not None: os.environ["LOCALE"] = self.original_locale else: os.environ.pop("LOCALE", None) def test_base_provider_utf8_support(self): """Test that the OpenAI provider supports UTF-8.""" provider = OpenAIModelProvider(api_key="test") # Test with UTF-8 characters test_text = "Développement en français avec émojis 🚀" tokens = provider.count_tokens(test_text, "gpt-4") # Should return a valid number (character-based estimate) self.assertIsInstance(tokens, int) self.assertGreater(tokens, 0) @pytest.mark.skip(reason="Requires real Gemini API access") @patch("google.generativeai.GenerativeModel") def test_gemini_provider_utf8_request(self, mock_model_class): """Test that the Gemini provider handles UTF-8 correctly.""" # Mock Gemini response mock_response = Mock() mock_response.text = "Response in French with accents: créé, développé, préféré 🎉" mock_response.usage_metadata = Mock() mock_response.usage_metadata.prompt_token_count = 10 mock_response.usage_metadata.candidates_token_count = 15 mock_response.usage_metadata.total_token_count = 25 mock_model = Mock() mock_model.generate_content.return_value = mock_response mock_model_class.return_value = mock_model # Test Gemini provider provider = GeminiModelProvider(api_key="test-key") # Request with UTF-8 characters response = provider.generate_content( prompt="Can you explain software development?", model_name="gemini-2.5-flash", system_prompt="Reply in French with emojis.", ) # Checks self.assertIsNotNone(response) self.assertIn("French", response.content) self.assertIn("🎉", response.content) # Check that the request contains UTF-8 characters mock_model.generate_content.assert_called_once() call_args = mock_model.generate_content.call_args parts = call_args[0][0] # First argument (parts) # Check for UTF-8 content in the request request_content = str(parts) self.assertIn("développement", request_content) @pytest.mark.skip(reason="Requires real OpenAI API access") @patch("openai.OpenAI") def test_openai_provider_utf8_logging(self, mock_openai_class): """Test that the OpenAI provider logs UTF-8 correctly.""" # Mock OpenAI response mock_response = Mock() mock_response.choices = [Mock()] mock_response.choices[0].message = Mock() mock_response.choices[0].message.content = "Python code created successfully! ✅" mock_response.usage = Mock() mock_response.usage.prompt_tokens = 20 mock_response.usage.completion_tokens = 10 mock_response.usage.total_tokens = 30 mock_client = Mock() mock_client.chat.completions.create.return_value = mock_response mock_openai_class.return_value = mock_client # Test OpenAI provider provider = OpenAIModelProvider(api_key="test-key") # Test with UTF-8 logging with patch("logging.info"): response = provider.generate_content( prompt="Generate Python code to process data", model_name="gpt-4", system_prompt="You are an expert Python developer.", ) # Response checks self.assertIsNotNone(response) self.assertIn("created", response.content) self.assertIn("✅", response.content) @pytest.mark.skip(reason="Requires real OpenAI API access") @patch("openai.OpenAI") def test_openai_compatible_o3_pro_utf8(self, mock_openai_class): """Test for o3-pro with /responses endpoint and UTF-8.""" # Mock o3-pro response mock_response = Mock() mock_response.output = Mock() mock_response.output.content = [Mock()] mock_response.output.content[0].type = "output_text" mock_response.output.content[0].text = "Analysis complete: code is well structured! 🎯" mock_response.usage = Mock() mock_response.usage.input_tokens = 50 mock_response.usage.output_tokens = 25 mock_response.model = "o3-pro" mock_response.id = "test-id" mock_response.created_at = 1234567890 mock_client = Mock() mock_client.responses.create.return_value = mock_response mock_openai_class.return_value = mock_client # Test OpenAI Compatible provider with o3-pro provider = OpenAIModelProvider(api_key="test-key") # Test with UTF-8 logging for o3-pro with patch("logging.info") as mock_logging: response = provider.generate_content( prompt="Analyze this Python code for issues", model_name="o3-pro", system_prompt="You are a code review expert.", ) # Response checks self.assertIsNotNone(response) self.assertIn("complete", response.content) self.assertIn("🎯", response.content) # Check that logging was called with ensure_ascii=False mock_logging.assert_called() log_calls = [call for call in mock_logging.call_args_list if "API request payload" in str(call)] self.assertTrue(len(log_calls) > 0, "No API payload log found") def test_provider_type_enum_utf8_safe(self): """Test that ProviderType enum is UTF-8 safe.""" # Test all provider types provider_types = list(ProviderType) for provider_type in provider_types: # Test JSON serialization data = {"provider": provider_type.value, "message": "UTF-8 test: emojis 🚀"} json_str = json.dumps(data, ensure_ascii=False) # Checks self.assertIn(provider_type.value, json_str) self.assertIn("emojis", json_str) self.assertIn("🚀", json_str) # Test deserialization parsed = json.loads(json_str) self.assertEqual(parsed["provider"], provider_type.value) self.assertEqual(parsed["message"], "UTF-8 test: emojis 🚀") def test_model_response_utf8_serialization(self): """Test UTF-8 serialization of model responses.""" from providers.shared import ModelResponse response = ModelResponse( content="Development successful! Code generated successfully. 🎉✅", usage={"input_tokens": 10, "output_tokens": 15, "total_tokens": 25}, model_name="test-model", friendly_name="Test Model", provider=ProviderType.OPENAI, # Pass enum, not .value metadata={"created": "2024-01-01", "developer": "Test", "emojis": "🚀🎯🔥"}, ) response_dict = getattr(response, "to_dict", None) if callable(response_dict): response_dict = response.to_dict() else: # Convert ProviderType to string for JSON serialization d = response.__dict__.copy() if isinstance(d.get("provider"), ProviderType): d["provider"] = d["provider"].value response_dict = d json_str = json.dumps(response_dict, ensure_ascii=False, indent=2) # Checks self.assertIn("Development", json_str) self.assertIn("successful", json_str) self.assertIn("generated", json_str) self.assertIn("🎉", json_str) self.assertIn("✅", json_str) self.assertIn("created", json_str) self.assertIn("developer", json_str) self.assertIn("🚀", json_str) # Test deserialization parsed = json.loads(json_str) self.assertEqual(parsed["content"], response.content) self.assertEqual(parsed["friendly_name"], "Test Model") def test_error_handling_with_utf8(self): """Test error handling with UTF-8 characters.""" provider = OpenAIModelProvider(api_key="test") # Test validation with UTF-8 error message (no exception expected) error_message = None try: provider.validate_parameters("gpt-4", -1.0) # Invalid temperature except Exception as e: error_message = str(e) # Error message may contain UTF-8 characters or be None if error_message: self.assertIsInstance(error_message, str) else: # No exception: test passes (current provider logs a warning only) self.assertTrue(True) def test_temperature_handling_utf8_locale(self): """Test temperature handling with UTF-8 locale.""" # Set French locale os.environ["LOCALE"] = "fr-FR" provider = OpenAIModelProvider(api_key="test") # Test different temperatures test_temps = [0.0, 0.5, 1.0, 1.5, 2.0] for temp in test_temps: try: provider.validate_parameters("gpt-4", temp) # If no exception, temperature is valid self.assertLessEqual(temp, 2.0) except ValueError: # If exception, temperature must be > 2.0 self.assertGreater(temp, 2.0) def test_provider_registry_utf8(self): """Test that the provider registry handles UTF-8.""" from providers.registry import ModelProviderRegistry # Test listing providers with UTF-8 descriptions providers = ModelProviderRegistry.get_available_providers() # Should contain valid providers self.assertGreater(len(providers), 0) # Test serialization provider_data = { "providers": [p.value for p in providers], "description": "Available providers for development 🚀", } json_str = json.dumps(provider_data, ensure_ascii=False) # Checks self.assertIn("development", json_str) self.assertIn("🚀", json_str) # Test parsing parsed = json.loads(json_str) self.assertEqual(parsed["description"], provider_data["description"]) @pytest.mark.skip(reason="Requires real Gemini API access") @patch("google.generativeai.GenerativeModel") def test_gemini_provider_handles_api_encoding_error(self, mock_model_class): """Test that the Gemini provider handles a non-UTF-8 API response.""" from unittest.mock import PropertyMock mock_response = Mock() type(mock_response).text = PropertyMock( side_effect=UnicodeDecodeError("utf-8", b"\xfa", 0, 1, "invalid start byte") ) mock_model = Mock() mock_model.generate_content.return_value = mock_response mock_model_class.return_value = mock_model provider = GeminiModelProvider(api_key="test-key") with self.assertRaises(Exception) as context: provider.generate_content( prompt="Explain something", model_name="gemini-2.5-flash", system_prompt="Reply in French.", ) # Accept any error message containing UnicodeDecodeError self.assertIn("UnicodeDecodeError", str(context.exception)) class DummyToolForLocaleTest: """Utility class to test language instruction generation.""" def get_language_instruction(self): locale = os.environ.get("LOCALE", "") if not locale or not locale.strip(): return "" return f"Always respond in {locale.strip()}.\n\n" class TestLocaleModelIntegration(unittest.TestCase): """Integration tests between locale and models.""" def setUp(self): """Integration test setup.""" self.original_locale = os.getenv("LOCALE") def tearDown(self): """Cleanup after integration tests.""" if self.original_locale is not None: os.environ["LOCALE"] = self.original_locale else: os.environ.pop("LOCALE", None) def test_system_prompt_enhancement_french(self): """Test system prompt enhancement with French locale.""" os.environ["LOCALE"] = "fr-FR" OpenAIModelProvider(api_key="test") # Simulate language instruction tool = DummyToolForLocaleTest() instruction = tool.get_language_instruction() self.assertIn("fr-FR", instruction) self.assertTrue(instruction.startswith("Always respond in fr-FR")) def test_system_prompt_enhancement_multiple_locales(self): """Test enhancement with different locales.""" OpenAIModelProvider(api_key="test") locales = ["fr-FR", "es-ES", "de-DE", "it-IT", "pt-BR", "ja-JP", "zh-CN"] for locale in locales: os.environ["LOCALE"] = locale tool = DummyToolForLocaleTest() instruction = tool.get_language_instruction() self.assertIn(locale, instruction) self.assertTrue(instruction.startswith(f"Always respond in {locale}")) prompt_data = {"system_prompt": instruction, "locale": locale} json_str = json.dumps(prompt_data, ensure_ascii=False) parsed = json.loads(json_str) self.assertEqual(parsed["locale"], locale) def test_model_name_resolution_utf8(self): """Test model name resolution with UTF-8.""" provider = OpenAIModelProvider(api_key="test") model_names = ["gpt-4", "gemini-2.5-flash", "anthropic/claude-opus-4.1", "o3-pro"] for model_name in model_names: resolved_model_name = provider._resolve_model_name(model_name) self.assertIsInstance(resolved_model_name, str) model_data = { "model": resolved_model_name, "description": f"Model {model_name} - advanced development 🚀", "capabilities": ["generation", "review", "creation"], } json_str = json.dumps(model_data, ensure_ascii=False) self.assertIn("development", json_str) self.assertIn("generation", json_str) self.assertIn("review", json_str) self.assertIn("creation", json_str) self.assertIn("🚀", json_str) def test_system_prompt_enhancement_with_unusual_locale_formats(self): """Test language instruction with various locale formats.""" test_locales = [ "fr", # Language only "fr_FR", # Language and region with underscore "de-DE.UTF-8", # Full locale with encoding ] for locale in test_locales: with self.subTest(locale=locale): os.environ["LOCALE"] = locale tool = DummyToolForLocaleTest() instruction = tool.get_language_instruction() self.assertTrue(instruction.startswith(f"Always respond in {locale}")) ================================================ FILE: tests/test_providers.py ================================================ """Tests for the model provider abstraction system""" import os from unittest.mock import Mock, patch import pytest from providers import ModelProviderRegistry, ModelResponse from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.shared import ProviderType class TestModelProviderRegistry: """Test the model provider registry""" def setup_method(self): """Clear registry before each test""" # Store the original providers to restore them later registry = ModelProviderRegistry() self._original_providers = registry._providers.copy() registry._providers.clear() registry._initialized_providers.clear() def teardown_method(self): """Restore original providers after each test""" # Restore the original providers that were registered in conftest.py registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() registry._providers.update(self._original_providers) def test_register_provider(self): """Test registering a provider""" ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) registry = ModelProviderRegistry() assert ProviderType.GOOGLE in registry._providers assert registry._providers[ProviderType.GOOGLE] == GeminiModelProvider @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key"}) def test_get_provider(self): """Test getting a provider instance""" ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) provider = ModelProviderRegistry.get_provider(ProviderType.GOOGLE) assert provider is not None assert isinstance(provider, GeminiModelProvider) assert provider.api_key == "test-key" @patch.dict(os.environ, {}, clear=True) def test_get_provider_no_api_key(self): """Test getting provider without API key returns None""" ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) provider = ModelProviderRegistry.get_provider(ProviderType.GOOGLE) assert provider is None @patch.dict(os.environ, {"GEMINI_API_KEY": "test-key"}) @pytest.mark.no_mock_provider def test_get_provider_for_model(self): """Test getting provider for a specific model""" ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) provider = ModelProviderRegistry.get_provider_for_model("gemini-2.5-flash") assert provider is not None assert isinstance(provider, GeminiModelProvider) def test_get_available_providers(self): """Test getting list of available providers""" ModelProviderRegistry.register_provider(ProviderType.GOOGLE, GeminiModelProvider) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) providers = ModelProviderRegistry.get_available_providers() assert len(providers) == 2 assert ProviderType.GOOGLE in providers assert ProviderType.OPENAI in providers class TestGeminiProvider: """Test Gemini model provider""" def test_provider_initialization(self): """Test provider initialization""" provider = GeminiModelProvider(api_key="test-key") assert provider.api_key == "test-key" assert provider.get_provider_type() == ProviderType.GOOGLE def test_get_capabilities(self): """Test getting model capabilities""" provider = GeminiModelProvider(api_key="test-key") capabilities = provider.get_capabilities("gemini-2.5-flash") assert capabilities.provider == ProviderType.GOOGLE assert capabilities.model_name == "gemini-2.5-flash" assert capabilities.context_window == 1_048_576 assert capabilities.supports_extended_thinking def test_get_capabilities_pro_model(self): """Test getting capabilities for Pro model with thinking support""" provider = GeminiModelProvider(api_key="test-key") capabilities = provider.get_capabilities("gemini-2.5-pro") assert capabilities.supports_extended_thinking def test_model_shorthand_resolution(self): """Test model shorthand resolution""" provider = GeminiModelProvider(api_key="test-key") assert provider.validate_model_name("flash") assert provider.validate_model_name("pro") capabilities = provider.get_capabilities("flash") assert capabilities.model_name == "gemini-2.5-flash" @patch("google.genai.Client") def test_generate_content(self, mock_client_class): """Test content generation""" # Mock the client mock_client = Mock() mock_response = Mock() mock_response.text = "Generated content" # Mock candidates for finish_reason mock_candidate = Mock() mock_candidate.finish_reason = "STOP" mock_response.candidates = [mock_candidate] # Mock usage metadata mock_usage = Mock() mock_usage.prompt_token_count = 10 mock_usage.candidates_token_count = 20 mock_response.usage_metadata = mock_usage mock_client.models.generate_content.return_value = mock_response mock_client_class.return_value = mock_client provider = GeminiModelProvider(api_key="test-key") response = provider.generate_content(prompt="Test prompt", model_name="gemini-2.5-flash", temperature=0.7) assert isinstance(response, ModelResponse) assert response.content == "Generated content" assert response.model_name == "gemini-2.5-flash" assert response.provider == ProviderType.GOOGLE assert response.usage["input_tokens"] == 10 assert response.usage["output_tokens"] == 20 assert response.usage["total_tokens"] == 30 class TestOpenAIProvider: """Test OpenAI model provider""" def setup_method(self): """Clear restriction service cache before each test""" import utils.model_restrictions utils.model_restrictions._restriction_service = None def teardown_method(self): """Clear restriction service cache after each test""" import utils.model_restrictions utils.model_restrictions._restriction_service = None def test_provider_initialization(self): """Test provider initialization""" provider = OpenAIModelProvider(api_key="test-key", organization="test-org") assert provider.api_key == "test-key" assert provider.organization == "test-org" assert provider.get_provider_type() == ProviderType.OPENAI def test_get_capabilities_o3(self): """Test getting O3 model capabilities""" provider = OpenAIModelProvider(api_key="test-key") capabilities = provider.get_capabilities("o3-mini") assert capabilities.provider == ProviderType.OPENAI assert capabilities.model_name == "o3-mini" assert capabilities.context_window == 200_000 assert not capabilities.supports_extended_thinking def test_get_capabilities_o4_mini(self): """Test getting O4-mini model capabilities""" provider = OpenAIModelProvider(api_key="test-key") capabilities = provider.get_capabilities("o4-mini") assert capabilities.provider == ProviderType.OPENAI assert capabilities.model_name == "o4-mini" assert capabilities.context_window == 200_000 assert not capabilities.supports_extended_thinking # Check temperature constraint is fixed at 1.0 assert capabilities.temperature_constraint.value == 1.0 def test_validate_model_names(self): """Test model name validation""" provider = OpenAIModelProvider(api_key="test-key") assert provider.validate_model_name("o3") assert provider.validate_model_name("o3mini") assert provider.validate_model_name("o3-mini") # Backwards compatibility assert provider.validate_model_name("o4-mini") assert provider.validate_model_name("o4mini") assert provider.validate_model_name("o4-mini") assert provider.validate_model_name("gpt-5.2") assert provider.validate_model_name("gpt-5.1-codex") assert provider.validate_model_name("gpt-5.1-codex-mini") assert not provider.validate_model_name("gpt-4o") assert not provider.validate_model_name("invalid-model") def test_openai_models_do_not_support_extended_thinking(self): """OpenAI catalogue exposes extended thinking capability via ModelCapabilities.""" provider = OpenAIModelProvider(api_key="test-key") aliases = ["o3", "o3mini", "o3-mini", "o4-mini", "o4mini"] for alias in aliases: assert not provider.get_capabilities(alias).supports_extended_thinking def test_gpt52_family_capabilities(self): """Ensure GPT-5.2 base model exposes correct capability flags.""" provider = OpenAIModelProvider(api_key="test-key") base = provider.get_capabilities("gpt-5.2") assert base.supports_streaming assert base.allow_code_generation codex = provider.get_capabilities("gpt-5.1-codex") assert not codex.supports_streaming assert codex.use_openai_response_api assert codex.allow_code_generation codex_mini = provider.get_capabilities("gpt-5.1-codex-mini") assert codex_mini.supports_streaming assert codex_mini.allow_code_generation ================================================ FILE: tests/test_rate_limit_patterns.py ================================================ """ Test to verify structured error code-based retry logic. """ from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider def test_openai_structured_error_retry_logic(): """Test OpenAI provider's structured error code retry logic.""" provider = OpenAIModelProvider(api_key="test-key") # Test structured token-related 429 error (should NOT be retried) class MockTokenError(Exception): def __init__(self): # Simulate the actual error format from OpenAI API self.args = ( "Error code: 429 - {'error': {'message': 'Request too large for o3', 'type': 'tokens', 'code': 'rate_limit_exceeded'}}", ) token_error = MockTokenError() assert not provider._is_error_retryable(token_error), "Token-related 429 should not be retryable" # Test standard rate limiting 429 error (should be retried) class MockRateLimitError(Exception): def __init__(self): self.args = ( "Error code: 429 - {'error': {'message': 'Too many requests', 'type': 'requests', 'code': 'rate_limit_exceeded'}}", ) rate_limit_error = MockRateLimitError() assert provider._is_error_retryable(rate_limit_error), "Request rate limiting should be retryable" # Test context length error (should NOT be retried) class MockContextError(Exception): def __init__(self): self.args = ( "Error code: 429 - {'error': {'message': 'Context length exceeded', 'code': 'context_length_exceeded'}}", ) context_error = MockContextError() assert not provider._is_error_retryable(context_error), "Context length errors should not be retryable" def test_gemini_structured_error_retry_logic(): """Test Gemini provider's structured error code retry logic.""" provider = GeminiModelProvider(api_key="test-key") # Test quota exceeded error (should NOT be retried) class MockQuotaError(Exception): def __init__(self): self.args = ("429 Resource exhausted: Quota exceeded for model",) self.details = "quota_exceeded" quota_error = MockQuotaError() assert not provider._is_error_retryable(quota_error), "Quota exceeded should not be retryable" # Test resource exhausted error (should NOT be retried) class MockResourceError(Exception): def __init__(self): self.args = ("429 Resource exhausted: Token limit exceeded",) resource_error = MockResourceError() assert not provider._is_error_retryable(resource_error), "Resource exhausted should not be retryable" # Test temporary rate limiting (should be retried) class MockTempError(Exception): def __init__(self): self.args = ("429 Too many requests, please try again later",) temp_error = MockTempError() assert provider._is_error_retryable(temp_error), "Temporary rate limiting should be retryable" def test_actual_log_error_from_issue_with_structured_parsing(): """Test the specific error from the user's log using structured parsing.""" provider = OpenAIModelProvider(api_key="test-key") # Create the exact error from the user's log class MockUserLogError(Exception): def __init__(self): # This is the exact error message from the user's issue self.args = ( "Error code: 429 - {'error': {'message': 'Request too large for o3 in organization org-MWp466of2XGyS90J8huQk4R6 on tokens per min (TPM): Limit 30000, Requested 31756. The input or output tokens must be reduced in order to run successfully. Visit https://platform.openai.com/account/rate-limits to learn more.', 'type': 'tokens', 'param': None, 'code': 'rate_limit_exceeded'}}", ) user_error = MockUserLogError() # This specific error should NOT be retryable because it has type='tokens' assert not provider._is_error_retryable(user_error), "The user's specific error should be non-retryable" def test_non_429_errors_still_work(): """Test that non-429 errors are still handled correctly.""" provider = OpenAIModelProvider(api_key="test-key") # Test retryable non-429 errors class MockTimeoutError(Exception): def __init__(self): self.args = ("Connection timeout",) timeout_error = MockTimeoutError() assert provider._is_error_retryable(timeout_error), "Timeout errors should be retryable" class Mock500Error(Exception): def __init__(self): self.args = ("500 Internal Server Error",) server_error = Mock500Error() assert provider._is_error_retryable(server_error), "500 errors should be retryable" # Test non-retryable non-429 errors class MockAuthError(Exception): def __init__(self): self.args = ("401 Unauthorized",) auth_error = MockAuthError() assert not provider._is_error_retryable(auth_error), "Auth errors should not be retryable" def test_edge_cases_and_fallbacks(): """Test edge cases and fallback behavior.""" provider = OpenAIModelProvider(api_key="test-key") # Test malformed JSON in error (should fall back gracefully) class MockMalformedError(Exception): def __init__(self): self.args = ("Error code: 429 - {invalid json}",) malformed_error = MockMalformedError() # Should still be retryable since it's a 429 without clear non-retryable indicators assert provider._is_error_retryable(malformed_error), "Malformed 429 errors should default to retryable" # Test 429 without structured data (should be retryable by default) class MockSimple429Error(Exception): def __init__(self): self.args = ("429 Too Many Requests",) simple_429_error = MockSimple429Error() assert provider._is_error_retryable(simple_429_error), "Simple 429 without type info should be retryable" ================================================ FILE: tests/test_refactor.py ================================================ """ Tests for the refactor tool functionality """ import json import pytest from tools.refactor import RefactorTool from utils.file_utils import read_file_content class TestRefactorTool: """Test suite for the refactor tool""" @pytest.fixture def refactor_tool(self): """Create a refactor tool instance for testing""" return RefactorTool() @pytest.fixture def mock_model_response(self): """Create a mock model response with valid JSON""" def _create_response(content=None): if content is None: content = json.dumps( { "refactor_opportunities": [ { "id": "refactor-001", "type": "codesmells", "severity": "high", "file": "/test/file.py", "start_line": 10, "end_line": 25, "context_start_text": "def long_method():", "context_end_text": " return result", "issue": "Method too long with multiple responsibilities", "suggestion": "Extract helper methods", "rationale": "Improves readability and maintainability", "code_to_replace": "# original code", "replacement_code_snippet": "# refactored code", "new_code_snippets": [], } ], "priority_sequence": ["refactor-001"], "next_actions": [], }, ensure_ascii=False, ) from unittest.mock import Mock return Mock( content=content, usage={"input_tokens": 100, "output_tokens": 200, "total_tokens": 300}, model_name="test-model", metadata={"finish_reason": "STOP"}, ) return _create_response def test_get_name(self, refactor_tool): """Test that the tool returns the correct name""" assert refactor_tool.get_name() == "refactor" def test_get_description(self, refactor_tool): """Test that the tool returns a comprehensive description""" description = refactor_tool.get_description() assert "refactoring" in description assert "code smell detection" in description assert "decomposition planning" in description assert "modernization" in description assert "maintainability improvements" in description def test_get_input_schema(self, refactor_tool): """Test that the input schema includes all required workflow fields""" schema = refactor_tool.get_input_schema() assert schema["type"] == "object" # Check workflow-specific fields assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "findings" in schema["properties"] assert "files_checked" in schema["properties"] assert "relevant_files" in schema["properties"] # Check refactor-specific fields assert "refactor_type" in schema["properties"] assert "confidence" in schema["properties"] # Check refactor_type enum values refactor_enum = schema["properties"]["refactor_type"]["enum"] expected_types = ["codesmells", "decompose", "modernize", "organization"] assert all(rt in refactor_enum for rt in expected_types) # Note: Old language detection and execution tests removed - # new workflow-based refactor tool has different architecture def test_model_category(self, refactor_tool): """Test that the refactor tool uses EXTENDED_REASONING category""" from tools.models import ToolModelCategory category = refactor_tool.get_model_category() assert category == ToolModelCategory.EXTENDED_REASONING def test_default_temperature(self, refactor_tool): """Test that the refactor tool uses analytical temperature""" from config import TEMPERATURE_ANALYTICAL temp = refactor_tool.get_default_temperature() assert temp == TEMPERATURE_ANALYTICAL # Note: format_response tests removed - workflow tools use different response format class TestFileUtilsLineNumbers: """Test suite for line numbering functionality in file_utils""" def test_read_file_content_with_line_numbers(self, project_path): """Test reading file content with line numbers enabled""" # Create a test file within the workspace temp_path = project_path / "test_file.py" with open(temp_path, "w") as f: f.write("def hello():\n print('Hello')\n return True") # Read with line numbers explicitly enabled content, tokens = read_file_content(str(temp_path), include_line_numbers=True) # Check that line numbers are present assert "1│ def hello():" in content assert "2│ print('Hello')" in content assert "3│ return True" in content assert "--- BEGIN FILE:" in content assert "--- END FILE:" in content def test_read_file_content_without_line_numbers(self, project_path): """Test reading file content with line numbers disabled""" # Create a test file within the workspace temp_path = project_path / "test_file.txt" with open(temp_path, "w") as f: f.write("Line 1\nLine 2\nLine 3") # Read with line numbers explicitly disabled content, tokens = read_file_content(str(temp_path), include_line_numbers=False) # Check that line numbers are NOT present assert "1│" not in content assert "Line 1" in content assert "Line 2" in content assert "--- BEGIN FILE:" in content def test_read_file_content_auto_detect_programming(self, project_path): """Test that auto-detection is OFF by default (backwards compatibility)""" # Create a test file within the workspace temp_path = project_path / "test_auto.py" with open(temp_path, "w") as f: f.write("import os\nprint('test')") # Read without specifying line numbers (should NOT auto-detect for backwards compatibility) content, tokens = read_file_content(str(temp_path)) # Should NOT automatically add line numbers for .py files (default behavior) assert "1│" not in content assert "import os" in content assert "print('test')" in content def test_read_file_content_auto_detect_text(self, project_path): """Test auto-detection of line numbers for text files""" # Create a test file within the workspace temp_path = project_path / "test_auto.txt" with open(temp_path, "w") as f: f.write("This is a text file\nWith multiple lines") # Read without specifying line numbers (should auto-detect) content, tokens = read_file_content(str(temp_path)) # Should NOT automatically add line numbers for .txt files assert "1│" not in content assert "This is a text file" in content def test_line_ending_normalization(self): """Test that different line endings are normalized consistently""" from utils.file_utils import _add_line_numbers, _normalize_line_endings # Test different line ending formats content_crlf = "Line 1\r\nLine 2\r\nLine 3" content_cr = "Line 1\rLine 2\rLine 3" content_lf = "Line 1\nLine 2\nLine 3" # All should normalize to the same result normalized_crlf = _normalize_line_endings(content_crlf) normalized_cr = _normalize_line_endings(content_cr) normalized_lf = _normalize_line_endings(content_lf) assert normalized_crlf == normalized_cr == normalized_lf assert normalized_lf == "Line 1\nLine 2\nLine 3" # Line numbering should work consistently numbered = _add_line_numbers(content_crlf) assert " 1│ Line 1" in numbered assert " 2│ Line 2" in numbered assert " 3│ Line 3" in numbered def test_detect_file_type(self): """Test file type detection""" from utils.file_utils import detect_file_type # Test programming language files assert detect_file_type("test.py") == "text" assert detect_file_type("test.js") == "text" assert detect_file_type("test.java") == "text" # Test image files assert detect_file_type("image.png") == "image" assert detect_file_type("photo.jpg") == "image" # Test binary files assert detect_file_type("program.exe") == "binary" assert detect_file_type("library.dll") == "binary" def test_should_add_line_numbers(self): """Test line number detection logic""" from utils.file_utils import should_add_line_numbers # NO files should get line numbers by default (backwards compatibility) assert not should_add_line_numbers("test.py") assert not should_add_line_numbers("app.js") assert not should_add_line_numbers("Main.java") assert not should_add_line_numbers("readme.txt") assert not should_add_line_numbers("data.csv") # Explicit override should work assert should_add_line_numbers("readme.txt", True) assert not should_add_line_numbers("test.py", False) def test_line_numbers_double_triple_digits(self, project_path): """Test line numbering with double and triple digit line numbers""" from utils.file_utils import _add_line_numbers # Create content with many lines to test double and triple digit formatting lines = [] for i in range(1, 125): # Lines 1-124 for testing up to triple digits if i < 10: lines.append(f"# Single digit line {i}") elif i < 100: lines.append(f"# Double digit line {i}") else: lines.append(f"# Triple digit line {i}") content = "\n".join(lines) numbered_content = _add_line_numbers(content) # Test single digit formatting (should be right-aligned with spaces) assert " 1│ # Single digit line 1" in numbered_content assert " 9│ # Single digit line 9" in numbered_content # Test double digit formatting (should be right-aligned) assert " 10│ # Double digit line 10" in numbered_content # Line 10 has "double digit" content assert " 50│ # Double digit line 50" in numbered_content assert " 99│ # Double digit line 99" in numbered_content # Test triple digit formatting (should be right-aligned) assert " 100│ # Triple digit line 100" in numbered_content assert " 124│ # Triple digit line 124" in numbered_content # Verify consistent alignment - all line numbers should end with "│ " lines_with_numbers = numbered_content.split("\n") for line in lines_with_numbers: if "│" in line: # Find the pipe character position pipe_pos = line.find("│") # Ensure the character before pipe is a digit assert line[pipe_pos - 1].isdigit(), f"Line format issue: {line}" # Ensure the character after pipe is a space assert line[pipe_pos + 1] == " ", f"Line format issue: {line}" def test_line_numbers_with_file_reading(self, project_path): """Test line numbering through file reading with large file""" # Create a test file with 150 functions (600 total lines: 4 lines per function) temp_path = project_path / "large_test_file.py" with open(temp_path, "w") as f: for i in range(1, 151): # Functions 1-150 f.write(f"def function_{i}():\n") f.write(f" # This is function number {i}\n") f.write(f" return {i}\n") f.write("\n") # Read with line numbers enabled content, tokens = read_file_content(str(temp_path), include_line_numbers=True) # Calculate actual line numbers based on file structure (4 lines per function) # Function 1: lines 1-4, Function 2: lines 5-8, etc. # Line 1: def function_1(): # Line 2: # This is function number 1 # Line 3: return 1 # Line 4: (empty) # Test various line number formats in the actual file content assert " 1│ def function_1():" in content # Function 13 starts at line 49 (12*4 + 1), so line 50 is " # This is function number 13" assert " 50│ # This is function number 13" in content # Line 100 is actually an empty line after function 25 (line 99 was "return 25") assert " 100│ " in content # Empty line # Line 99 is "return 25" from function 25 assert " 99│ return 25" in content # Test more line numbers - line 147 is "return 37" from function 37 assert " 147│ return 37" in content # Test that we have the final lines (600 total lines) assert " 599│ return 150" in content assert " 600│ " in content # Final empty line # Verify the file structure is preserved assert "--- BEGIN FILE:" in content assert "--- END FILE:" in content assert str(temp_path) in content def test_line_numbers_large_files_22k_lines(self, project_path): """Test line numbering for very large files (22,500+ lines)""" from utils.file_utils import _add_line_numbers # Create content simulating a very large file with 25,000 lines lines = [] for i in range(1, 25001): # Lines 1-25000 lines.append(f"// Large file line {i}") content = "\n".join(lines) numbered_content = _add_line_numbers(content) # Test that width dynamically adjusts to 5 digits for large files # Small line numbers should now have 5-digit width assert " 1│ // Large file line 1" in numbered_content assert " 9│ // Large file line 9" in numbered_content assert " 10│ // Large file line 10" in numbered_content assert " 99│ // Large file line 99" in numbered_content assert " 100│ // Large file line 100" in numbered_content assert " 999│ // Large file line 999" in numbered_content assert " 1000│ // Large file line 1000" in numbered_content assert " 9999│ // Large file line 9999" in numbered_content assert "10000│ // Large file line 10000" in numbered_content assert "22500│ // Large file line 22500" in numbered_content assert "25000│ // Large file line 25000" in numbered_content # Verify consistent alignment - all line numbers should end with "│ " lines_with_numbers = numbered_content.split("\n") for i, line in enumerate(lines_with_numbers[:100]): # Check first 100 lines if "│" in line: pipe_pos = line.find("│") # For large files, should be 5-character width plus pipe assert line[pipe_pos - 1].isdigit(), f"Line {i+1} format issue: {line}" assert line[pipe_pos + 1] == " ", f"Line {i+1} format issue: {line}" def test_line_numbers_boundary_conditions(self): """Test line numbering at boundary conditions (9999 vs 10000 lines)""" from utils.file_utils import _add_line_numbers # Test exactly 9999 lines (should use 4-digit width) lines_9999 = [f"Line {i}" for i in range(1, 10000)] # 9999 lines content_9999 = "\n".join(lines_9999) numbered_9999 = _add_line_numbers(content_9999) # Should use 4-digit format assert " 1│ Line 1" in numbered_9999 assert "9999│ Line 9999" in numbered_9999 # Test exactly 10000 lines (should use 5-digit width) lines_10000 = [f"Line {i}" for i in range(1, 10001)] # 10000 lines content_10000 = "\n".join(lines_10000) numbered_10000 = _add_line_numbers(content_10000) # Should use 5-digit format assert " 1│ Line 1" in numbered_10000 assert "10000│ Line 10000" in numbered_10000 if __name__ == "__main__": pytest.main([__file__, "-v"]) ================================================ FILE: tests/test_secaudit.py ================================================ """ Tests for the secaudit tool using WorkflowTool architecture. """ import pytest from tools.models import ToolModelCategory from tools.secaudit import SecauditRequest, SecauditTool class TestSecauditTool: """Test suite for SecauditTool using WorkflowTool architecture.""" def test_tool_metadata(self): """Test basic tool metadata and configuration.""" tool = SecauditTool() assert tool.get_name() == "secaudit" assert "security audit" in tool.get_description() assert tool.get_default_temperature() == 1.0 # TEMPERATURE_ANALYTICAL assert tool.get_model_category() == ToolModelCategory.EXTENDED_REASONING assert tool.requires_model() is True def test_request_validation(self): """Test Pydantic request model validation.""" # Valid security audit step request step_request = SecauditRequest( step="Beginning comprehensive security audit of web application", step_number=1, total_steps=6, next_step_required=True, findings="Identified React/Node.js e-commerce application with payment processing", files_checked=["/src/auth.py", "/src/payment.py"], relevant_files=["/src/auth.py", "/src/payment.py"], relevant_context=["AuthController.login", "PaymentService.process"], security_scope="Web application - e-commerce platform", threat_level="high", compliance_requirements=["PCI DSS", "SOC2"], audit_focus="comprehensive", confidence="medium", ) assert step_request.step_number == 1 assert step_request.threat_level == "high" assert step_request.compliance_requirements == ["PCI DSS", "SOC2"] assert step_request.audit_focus == "comprehensive" assert len(step_request.relevant_context) == 2 def test_request_validation_defaults(self): """Test default values for optional fields.""" minimal_request = SecauditRequest( step="Security audit step", step_number=1, total_steps=4, next_step_required=True, findings="Initial findings", ) assert minimal_request.threat_level == "medium" # Default value assert minimal_request.audit_focus == "comprehensive" # Default value assert minimal_request.confidence == "low" # Default value assert minimal_request.compliance_requirements == [] # Default empty list def test_request_validation_invalid_threat_level(self): """Test validation with invalid threat level.""" with pytest.raises(ValueError): SecauditRequest( step="Security audit step", step_number=1, total_steps=4, next_step_required=True, findings="Initial findings", threat_level="invalid", # Should only accept low, medium, high, critical ) def test_request_validation_invalid_audit_focus(self): """Test validation with invalid audit focus.""" with pytest.raises(ValueError): SecauditRequest( step="Security audit step", step_number=1, total_steps=4, next_step_required=True, findings="Initial findings", audit_focus="invalid", # Should only accept defined options ) def test_input_schema_generation(self): """Test that input schema is generated correctly.""" tool = SecauditTool() schema = tool.get_input_schema() # Verify required security audit fields are present assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "findings" in schema["properties"] # Verify security-specific fields assert "security_scope" in schema["properties"] assert "threat_level" in schema["properties"] assert "compliance_requirements" in schema["properties"] assert "audit_focus" in schema["properties"] # Verify field types assert schema["properties"]["threat_level"]["type"] == "string" assert schema["properties"]["compliance_requirements"]["type"] == "array" def test_step_guidance_step_1(self): """Test step-specific guidance for step 1 (Security Scope Analysis).""" tool = SecauditTool() request = SecauditRequest( step="Begin security audit", step_number=1, total_steps=6, next_step_required=True, findings="Starting security assessment", ) actions = tool.get_required_actions( request.step_number, request.confidence, request.findings, request.total_steps ) assert len(actions) == 4 assert "Identify application type, technology stack, and security scope" in actions assert "Map attack surface, entry points, and data flows" in actions assert "Determine relevant security standards and compliance requirements" in actions assert "Establish threat landscape and risk context for the application" in actions def test_step_guidance_step_2(self): """Test step-specific guidance for step 2 (Authentication Assessment).""" tool = SecauditTool() request = SecauditRequest( step="Analyze authentication", step_number=2, total_steps=6, next_step_required=True, findings="Authentication analysis", ) actions = tool.get_required_actions( request.step_number, request.confidence, request.findings, request.total_steps ) assert len(actions) == 4 assert "Analyze authentication mechanisms and session management" in actions assert "Check authorization controls, access patterns, and privilege escalation risks" in actions assert "Assess multi-factor authentication, password policies, and account security" in actions assert "Review identity and access management implementations" in actions def test_step_guidance_step_4(self): """Test step-specific guidance for step 4 (OWASP Top 10 Review).""" tool = SecauditTool() request = SecauditRequest( step="OWASP Top 10 review", step_number=4, total_steps=6, next_step_required=True, findings="OWASP analysis" ) actions = tool.get_required_actions( request.step_number, request.confidence, request.findings, request.total_steps ) assert len(actions) == 4 assert "Conduct OWASP Top 10 (2021) systematic review across all categories" in actions assert "Check each OWASP category methodically with specific findings and evidence" in actions assert "Cross-reference findings with application context and technology stack" in actions assert "Prioritize vulnerabilities based on exploitability and business impact" in actions def test_expert_analysis_trigger(self): """Test when expert analysis should be triggered.""" tool = SecauditTool() # Create a mock consolidated findings object class MockConsolidatedFindings: def __init__(self, relevant_files=None, findings=None, issues_found=None): self.relevant_files = relevant_files or [] self.findings = findings or [] self.issues_found = issues_found or [] # Should trigger expert analysis when we have meaningful findings findings_with_files = MockConsolidatedFindings( relevant_files=["/src/auth.py", "/src/payment.py"], findings=["Finding 1", "Finding 2"], issues_found=[{"severity": "high", "description": "SQL injection"}], ) assert tool.should_call_expert_analysis(findings_with_files) is True # Should trigger with just findings findings_only = MockConsolidatedFindings(findings=["Finding 1", "Finding 2"]) assert tool.should_call_expert_analysis(findings_only) is True # Should trigger with just issues issues_only = MockConsolidatedFindings(issues_found=[{"severity": "high", "description": "SQL injection"}]) assert tool.should_call_expert_analysis(issues_only) is True # Should not trigger with no meaningful data no_findings = MockConsolidatedFindings() assert tool.should_call_expert_analysis(no_findings) is False def test_expert_analysis_context_preparation(self): """Test expert analysis context preparation.""" tool = SecauditTool() # Create a mock consolidated findings object class MockConsolidatedFindings: def __init__(self): self.hypotheses = [] self.files_checked = ["/app/auth.py", "/app/payment.py", "/app/api.py", "/app/db.py"] self.relevant_files = ["/app/auth.py", "/app/payment.py", "/app/api.py"] self.relevant_context = ["AuthController.login", "PaymentService.process", "APIController.validate"] self.issues_found = [ {"severity": "critical", "description": "SQL injection vulnerability in login endpoint"}, {"severity": "high", "description": "Missing input validation in payment processing"}, {"severity": "medium", "description": "Weak session management configuration"}, ] self.findings = [ "Step 1: Identified e-commerce web application with payment processing", "Step 2: Found authentication vulnerabilities", "Step 3: Discovered input validation issues", ] self.hypotheses = [ {"step": 1, "confidence": "low", "hypothesis": "Initial security assessment"}, {"step": 2, "confidence": "medium", "hypothesis": "Authentication issues confirmed"}, {"step": 3, "confidence": "high", "hypothesis": "Multiple security vulnerabilities identified"}, ] self.images = [] # Set initial request to provide context tool.initial_request = "Perform security audit of e-commerce web application" tool.security_config = { "security_scope": "Web application - e-commerce platform with payment processing", "threat_level": "high", "compliance_requirements": ["PCI DSS", "SOC2", "GDPR"], "audit_focus": "comprehensive", "severity_filter": "all", } consolidated_findings = MockConsolidatedFindings() context = tool.prepare_expert_analysis_context(consolidated_findings) # Verify context contains all security-specific information assert "SECURITY AUDIT REQUEST" in context assert "Perform security audit of e-commerce web application" in context assert "SECURITY CONFIGURATION" in context assert "security_scope: Web application - e-commerce platform with payment processing" in context assert "threat_level: high" in context assert "compliance_requirements: ['PCI DSS', 'SOC2', 'GDPR']" in context assert "/app/auth.py" in context assert "AuthController.login" in context assert "CRITICAL SEVERITY:" in context assert "SQL injection vulnerability" in context assert "HIGH SEVERITY:" in context assert "Missing input validation" in context def test_security_issues_formatting_empty(self): """Test security issues formatting with no issues.""" tool = SecauditTool() formatted = tool._format_security_issues([]) assert "No security issues identified during systematic investigation." in formatted def test_security_issues_formatting_with_issues(self): """Test security issues formatting with multiple severity levels.""" tool = SecauditTool() issues = [ {"severity": "critical", "description": "Remote code execution vulnerability"}, {"severity": "high", "description": "Authentication bypass"}, {"severity": "medium", "description": "Information disclosure"}, {"severity": "low", "description": "Missing security headers"}, {"severity": "unknown", "description": "Unclassified issue"}, # Should go to low ] formatted = tool._format_security_issues(issues) assert "CRITICAL SEVERITY:" in formatted assert "Remote code execution vulnerability" in formatted assert "HIGH SEVERITY:" in formatted assert "Authentication bypass" in formatted assert "MEDIUM SEVERITY:" in formatted assert "Information disclosure" in formatted assert "LOW SEVERITY:" in formatted assert "Missing security headers" in formatted assert "[UNKNOWN] Unclassified issue" in formatted def test_tool_field_definitions(self): """Test that all security-specific tool fields are properly defined.""" tool = SecauditTool() fields = tool.get_tool_fields() # Verify all expected fields are present expected_fields = [ "step", "step_number", "total_steps", "next_step_required", "findings", "files_checked", "relevant_files", "relevant_context", "issues_found", "confidence", "images", "security_scope", "threat_level", "compliance_requirements", "audit_focus", "severity_filter", ] for field in expected_fields: assert field in fields, f"Field '{field}' not found in tool field definitions" # Verify field descriptions are comprehensive assert "OWASP Top 10" in fields["step"] assert "OWASP Top 10" in fields["step"] assert "MANDATORY" in fields["step"] assert "Security context" in fields["security_scope"] assert "threat level" in fields["threat_level"] assert "compliance frameworks" in fields["compliance_requirements"] def test_workflow_request_model(self): """Test that the workflow request model is correctly configured.""" tool = SecauditTool() request_model = tool.get_workflow_request_model() assert request_model == SecauditRequest def test_workflow_system_prompt(self): """Test that the workflow system prompt is correctly configured.""" tool = SecauditTool() system_prompt = tool.get_system_prompt() # Verify it contains key security audit elements assert "OWASP Top 10" in system_prompt assert "security_analysis_complete" in system_prompt assert "vulnerability" in system_prompt assert "compliance_assessment" in system_prompt def test_compliance_requirements_validation(self): """Test compliance requirements validation in model validator.""" # Test with valid compliance requirements valid_request = SecauditRequest( step="Security audit with compliance", step_number=1, total_steps=6, next_step_required=True, findings="Starting audit", compliance_requirements=["SOC2", "PCI DSS", "HIPAA"], ) assert valid_request.compliance_requirements == ["SOC2", "PCI DSS", "HIPAA"] # Test with unknown compliance requirement (should warn but not fail) unknown_compliance_request = SecauditRequest( step="Security audit with unknown compliance", step_number=1, total_steps=6, next_step_required=True, findings="Starting audit", compliance_requirements=["UNKNOWN_COMPLIANCE"], ) # Should still create the request but log a warning assert unknown_compliance_request.compliance_requirements == ["UNKNOWN_COMPLIANCE"] def test_comprehensive_workflow_scenario(self): """Test a complete workflow scenario from start to finish.""" tool = SecauditTool() # Step 1: Initial security scope analysis step1_request = SecauditRequest( step="Begin comprehensive security audit of e-commerce web application", step_number=1, total_steps=6, next_step_required=True, findings="Identified Node.js/React application with payment processing and user management", security_scope="Web application - e-commerce platform", threat_level="high", compliance_requirements=["PCI DSS"], relevant_files=["/src/auth.js", "/src/payment.js"], ) step1_actions = tool.get_required_actions( step1_request.step_number, step1_request.confidence, step1_request.findings, step1_request.total_steps ) assert "Identify application type" in step1_actions[0] # Test should_call_expert_analysis with mock consolidated findings class MockConsolidatedFindings: def __init__(self): self.hypotheses = [] self.relevant_files = [] self.findings = [] self.issues_found = [] mock_findings = MockConsolidatedFindings() assert not tool.should_call_expert_analysis(mock_findings) # Step 6: Final assessment step6_request = SecauditRequest( step="Complete security assessment and risk evaluation", step_number=6, total_steps=6, next_step_required=False, findings="Comprehensive security audit completed with findings documented", security_scope="Web application - e-commerce platform", threat_level="high", compliance_requirements=["PCI DSS"], relevant_files=["/src/auth.js", "/src/payment.js", "/src/api.js"], relevant_context=["AuthService.authenticate", "PaymentProcessor.charge"], issues_found=[ {"severity": "high", "description": "SQL injection in user search"}, {"severity": "medium", "description": "Weak password policy"}, ], confidence="high", ) step6_actions = tool.get_required_actions( step6_request.step_number, step6_request.confidence, step6_request.findings, step6_request.total_steps ) assert "Evaluate compliance requirements" in step6_actions[0] # Create mock consolidated findings for final step final_findings = MockConsolidatedFindings() final_findings.relevant_files = step6_request.relevant_files final_findings.findings = ["Comprehensive security audit completed with findings documented"] final_findings.issues_found = step6_request.issues_found final_findings.relevant_context = [] final_findings.images = [] assert tool.should_call_expert_analysis(final_findings) # Test expert analysis context generation with mock consolidated findings # Set up tool state as it would be after processing tool.initial_request = "Complete security assessment and risk evaluation" tool.security_config = { "security_scope": step6_request.security_scope, "threat_level": step6_request.threat_level, "compliance_requirements": step6_request.compliance_requirements, "audit_focus": step6_request.audit_focus, "severity_filter": step6_request.severity_filter, } # Create a complete mock consolidated findings complete_findings = MockConsolidatedFindings() complete_findings.relevant_files = step6_request.relevant_files complete_findings.relevant_context = step6_request.relevant_context complete_findings.issues_found = step6_request.issues_found complete_findings.findings = ["Security audit findings from all steps"] complete_findings.files_checked = [] complete_findings.images = [] context = tool.prepare_expert_analysis_context(complete_findings) assert "PCI DSS" in context assert "SQL injection" in context assert "HIGH SEVERITY:" in context ================================================ FILE: tests/test_server.py ================================================ """ Tests for the main server functionality """ import pytest from server import handle_call_tool class TestServerTools: """Test server tool handling""" @pytest.mark.asyncio async def test_handle_call_tool_unknown(self): """Test calling an unknown tool""" result = await handle_call_tool("unknown_tool", {}) assert len(result) == 1 assert "Unknown tool: unknown_tool" in result[0].text @pytest.mark.asyncio async def test_handle_chat(self): """Test chat functionality using real integration testing""" import importlib import os # Set test environment os.environ["PYTEST_CURRENT_TEST"] = "test" # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for real provider resolution os.environ["OPENAI_API_KEY"] = "sk-test-key-server-chat-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None # Test with real provider resolution try: result = await handle_call_tool("chat", {"prompt": "Hello Gemini", "model": "o3-mini"}) # If we get here, check the response format assert len(result) == 1 # Parse JSON response import json response_data = json.loads(result[0].text) assert "status" in response_data except Exception as e: # Expected: API call will fail with fake key error_msg = str(e) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_handle_version(self): """Test getting version info""" result = await handle_call_tool("version", {}) assert len(result) == 1 response = result[0].text # Parse the JSON response import json data = json.loads(response) assert data["status"] == "success" content = data["content"] # Check for expected content in the markdown output assert "# PAL MCP Server Version" in content assert "## Server Information" in content assert "## Configuration" in content assert "Current Version" in content ================================================ FILE: tests/test_supported_models_aliases.py ================================================ """Test the MODEL_CAPABILITIES aliases structure across all providers.""" from providers.dial import DIALModelProvider from providers.gemini import GeminiModelProvider from providers.openai import OpenAIModelProvider from providers.xai import XAIModelProvider class TestSupportedModelsAliases: """Test that all providers have correctly structured MODEL_CAPABILITIES with aliases.""" def test_gemini_provider_aliases(self): """Test Gemini provider's alias structure.""" provider = GeminiModelProvider("test-key") # Check that all models have ModelCapabilities with aliases for model_name, config in provider.MODEL_CAPABILITIES.items(): assert hasattr(config, "aliases"), f"{model_name} must have aliases attribute" assert isinstance(config.aliases, list), f"{model_name} aliases must be a list" # Test specific aliases assert "flash" in provider.MODEL_CAPABILITIES["gemini-2.5-flash"].aliases assert "pro" in provider.MODEL_CAPABILITIES["gemini-3-pro-preview"].aliases assert "flash-2.0" in provider.MODEL_CAPABILITIES["gemini-2.0-flash"].aliases assert "flash2" in provider.MODEL_CAPABILITIES["gemini-2.0-flash"].aliases assert "flashlite" in provider.MODEL_CAPABILITIES["gemini-2.0-flash-lite"].aliases assert "flash-lite" in provider.MODEL_CAPABILITIES["gemini-2.0-flash-lite"].aliases # Test alias resolution assert provider._resolve_model_name("flash") == "gemini-2.5-flash" assert provider._resolve_model_name("pro") == "gemini-3-pro-preview" assert provider._resolve_model_name("flash-2.0") == "gemini-2.0-flash" assert provider._resolve_model_name("flash2") == "gemini-2.0-flash" assert provider._resolve_model_name("flashlite") == "gemini-2.0-flash-lite" # Test case insensitive resolution assert provider._resolve_model_name("Flash") == "gemini-2.5-flash" assert provider._resolve_model_name("PRO") == "gemini-3-pro-preview" def test_openai_provider_aliases(self): """Test OpenAI provider's alias structure.""" provider = OpenAIModelProvider("test-key") # Check that all models have ModelCapabilities with aliases for model_name, config in provider.MODEL_CAPABILITIES.items(): assert hasattr(config, "aliases"), f"{model_name} must have aliases attribute" assert isinstance(config.aliases, list), f"{model_name} aliases must be a list" # Test specific aliases # "mini" is now an alias for gpt-5-mini, not o4-mini assert "mini" in provider.MODEL_CAPABILITIES["gpt-5-mini"].aliases assert "o4mini" in provider.MODEL_CAPABILITIES["o4-mini"].aliases # o4-mini is no longer in its own aliases (removed self-reference) assert "o3mini" in provider.MODEL_CAPABILITIES["o3-mini"].aliases assert "o3pro" in provider.MODEL_CAPABILITIES["o3-pro"].aliases assert "gpt4.1" in provider.MODEL_CAPABILITIES["gpt-4.1"].aliases assert "gpt5.2" in provider.MODEL_CAPABILITIES["gpt-5.2"].aliases assert "gpt5.1-codex" in provider.MODEL_CAPABILITIES["gpt-5.1-codex"].aliases assert "codex-mini" in provider.MODEL_CAPABILITIES["gpt-5.1-codex-mini"].aliases # Test alias resolution assert provider._resolve_model_name("mini") == "gpt-5-mini" # mini -> gpt-5-mini now assert provider._resolve_model_name("o3mini") == "o3-mini" assert provider._resolve_model_name("o3pro") == "o3-pro" # o3pro resolves to o3-pro assert provider._resolve_model_name("o4mini") == "o4-mini" assert provider._resolve_model_name("gpt4.1") == "gpt-4.1" # gpt4.1 resolves to gpt-4.1 assert provider._resolve_model_name("gpt5.2") == "gpt-5.2" assert provider._resolve_model_name("gpt5.1") == "gpt-5.2" assert provider._resolve_model_name("gpt5.1-codex") == "gpt-5.1-codex" assert provider._resolve_model_name("codex-mini") == "gpt-5.1-codex-mini" # Test case insensitive resolution assert provider._resolve_model_name("Mini") == "gpt-5-mini" # mini -> gpt-5-mini now assert provider._resolve_model_name("O3MINI") == "o3-mini" assert provider._resolve_model_name("Gpt5.1") == "gpt-5.2" def test_xai_provider_aliases(self): """Test XAI provider's alias structure.""" provider = XAIModelProvider("test-key") # Check that all models have ModelCapabilities with aliases for model_name, config in provider.MODEL_CAPABILITIES.items(): assert hasattr(config, "aliases"), f"{model_name} must have aliases attribute" assert isinstance(config.aliases, list), f"{model_name} aliases must be a list" # Test specific aliases assert "grok" in provider.MODEL_CAPABILITIES["grok-4"].aliases assert "grok4" in provider.MODEL_CAPABILITIES["grok-4"].aliases assert "grok-4.1-fast-reasoning" in provider.MODEL_CAPABILITIES["grok-4-1-fast-reasoning"].aliases # Test alias resolution assert provider._resolve_model_name("grok") == "grok-4" assert provider._resolve_model_name("grok4") == "grok-4" assert provider._resolve_model_name("grok-4.1-fast-reasoning") == "grok-4-1-fast-reasoning" assert provider._resolve_model_name("grok-4.1-fast-reasoning-latest") == "grok-4-1-fast-reasoning" # Test case insensitive resolution assert provider._resolve_model_name("Grok") == "grok-4" assert provider._resolve_model_name("GROK-4.1-FAST-REASONING") == "grok-4-1-fast-reasoning" def test_dial_provider_aliases(self): """Test DIAL provider's alias structure.""" provider = DIALModelProvider("test-key") # Check that all models have ModelCapabilities with aliases for model_name, config in provider.MODEL_CAPABILITIES.items(): assert hasattr(config, "aliases"), f"{model_name} must have aliases attribute" assert isinstance(config.aliases, list), f"{model_name} aliases must be a list" # Test specific aliases assert "o3" in provider.MODEL_CAPABILITIES["o3-2025-04-16"].aliases assert "o4-mini" in provider.MODEL_CAPABILITIES["o4-mini-2025-04-16"].aliases assert "sonnet-4.1" in provider.MODEL_CAPABILITIES["anthropic.claude-sonnet-4.1-20250805-v1:0"].aliases assert "opus-4.1" in provider.MODEL_CAPABILITIES["anthropic.claude-opus-4.1-20250805-v1:0"].aliases assert "gemini-2.5-pro" in provider.MODEL_CAPABILITIES["gemini-2.5-pro-preview-05-06"].aliases # Test alias resolution assert provider._resolve_model_name("o3") == "o3-2025-04-16" assert provider._resolve_model_name("o4-mini") == "o4-mini-2025-04-16" assert provider._resolve_model_name("sonnet-4.1") == "anthropic.claude-sonnet-4.1-20250805-v1:0" assert provider._resolve_model_name("opus-4.1") == "anthropic.claude-opus-4.1-20250805-v1:0" # Test case insensitive resolution assert provider._resolve_model_name("O3") == "o3-2025-04-16" assert provider._resolve_model_name("SONNET-4.1") == "anthropic.claude-sonnet-4.1-20250805-v1:0" def test_list_models_includes_aliases(self): """Test that list_models returns both base models and aliases.""" # Test Gemini gemini_provider = GeminiModelProvider("test-key") gemini_models = gemini_provider.list_models(respect_restrictions=False) assert "gemini-2.5-flash" in gemini_models assert "flash" in gemini_models assert "gemini-3-pro-preview" in gemini_models assert "pro" in gemini_models # Test OpenAI openai_provider = OpenAIModelProvider("test-key") openai_models = openai_provider.list_models(respect_restrictions=False) assert "o4-mini" in openai_models assert "mini" in openai_models assert "o3-mini" in openai_models assert "o3mini" in openai_models # Test XAI xai_provider = XAIModelProvider("test-key") xai_models = xai_provider.list_models(respect_restrictions=False) assert "grok-4" in xai_models assert "grok" in xai_models assert "grok-4.1-fast" in xai_models assert "grok-4.1-fast-reasoning" in xai_models # Test DIAL dial_provider = DIALModelProvider("test-key") dial_models = dial_provider.list_models(respect_restrictions=False) assert "o3-2025-04-16" in dial_models assert "o3" in dial_models def test_list_models_all_known_variant_includes_aliases(self): """Unified list_models should support lowercase, alias-inclusive listings.""" # Test Gemini gemini_provider = GeminiModelProvider("test-key") gemini_all = gemini_provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) assert "gemini-2.5-flash" in gemini_all assert "flash" in gemini_all assert "gemini-3-pro-preview" in gemini_all assert "pro" in gemini_all # All should be lowercase assert all(model == model.lower() for model in gemini_all) # Test OpenAI openai_provider = OpenAIModelProvider("test-key") openai_all = openai_provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) assert "o4-mini" in openai_all assert "mini" in openai_all assert "o3-mini" in openai_all assert "o3mini" in openai_all # All should be lowercase assert all(model == model.lower() for model in openai_all) def test_no_string_shorthand_in_supported_models(self): """Test that no provider has string-based shorthands anymore.""" providers = [ GeminiModelProvider("test-key"), OpenAIModelProvider("test-key"), XAIModelProvider("test-key"), DIALModelProvider("test-key"), ] for provider in providers: for model_name, config in provider.MODEL_CAPABILITIES.items(): # All values must be ModelCapabilities objects, not strings or dicts from providers.shared import ModelCapabilities assert isinstance(config, ModelCapabilities), ( f"{provider.__class__.__name__}.MODEL_CAPABILITIES['{model_name}'] " f"must be a ModelCapabilities object, not {type(config).__name__}" ) def test_resolve_returns_original_if_not_found(self): """Test that _resolve_model_name returns original name if alias not found.""" providers = [ GeminiModelProvider("test-key"), OpenAIModelProvider("test-key"), XAIModelProvider("test-key"), DIALModelProvider("test-key"), ] for provider in providers: # Test with unknown model name assert provider._resolve_model_name("unknown-model") == "unknown-model" assert provider._resolve_model_name("gpt-4") == "gpt-4" assert provider._resolve_model_name("claude-3") == "claude-3" ================================================ FILE: tests/test_thinking_modes.py ================================================ """ Tests for thinking_mode functionality across all tools """ from unittest.mock import patch import pytest from tools.analyze import AnalyzeTool from tools.codereview import CodeReviewTool from tools.debug import DebugIssueTool from tools.thinkdeep import ThinkDeepTool @pytest.fixture(autouse=True) def setup_test_env(): """Set up test environment""" # PYTEST_CURRENT_TEST is already set by pytest yield class TestThinkingModes: """Test thinking modes across all tools""" @patch("config.DEFAULT_THINKING_MODE_THINKDEEP", "high") def test_default_thinking_modes(self): """Test that tools have correct default thinking modes""" tools = [ (ThinkDeepTool(), "high"), (AnalyzeTool(), "medium"), (CodeReviewTool(), "medium"), (DebugIssueTool(), "medium"), ] for tool, expected_default in tools: assert ( tool.get_default_thinking_mode() == expected_default ), f"{tool.__class__.__name__} should default to {expected_default}" @pytest.mark.asyncio async def test_thinking_mode_minimal(self): """Test minimal thinking mode with real provider resolution""" import importlib import os # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for OpenAI provider (which supports thinking mode) os.environ["OPENAI_API_KEY"] = "sk-test-key-minimal-thinking-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Use a model that supports thinking # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = AnalyzeTool() # This should attempt to use the real OpenAI provider # Even with a fake API key, we can test the provider resolution logic # The test will fail at the API call level, but we can verify the thinking mode logic try: result = await tool.execute( { "absolute_file_paths": ["/absolute/path/test.py"], "prompt": "What is this?", "model": "o3-mini", "thinking_mode": "minimal", } ) # If we get here, great! The provider resolution worked # Check that thinking mode was properly handled assert result is not None except Exception as e: # Expected: API call will fail with fake key, but we can check the error # If we get a provider resolution error, that's what we're testing error_msg = getattr(e, "payload", str(e)) # Should NOT be a mock-related error - should be a real API or key error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error (API key, network, etc.) import json try: parsed = json.loads(error_msg) except Exception: parsed = None if isinstance(parsed, dict) and parsed.get("status", "").endswith("_failed"): assert "validation errors" in parsed.get("error", "") else: assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection", "Model"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_thinking_mode_low(self): """Test low thinking mode with real provider resolution""" import importlib import os # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for OpenAI provider (which supports thinking mode) os.environ["OPENAI_API_KEY"] = "sk-test-key-low-thinking-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = CodeReviewTool() # Test with real provider resolution try: result = await tool.execute( { "absolute_file_paths": ["/absolute/path/test.py"], "thinking_mode": "low", "prompt": "Test code review for validation purposes", "model": "o3-mini", } ) # If we get here, provider resolution worked assert result is not None except Exception as e: # Expected: API call will fail with fake key error_msg = getattr(e, "payload", str(e)) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error import json try: parsed = json.loads(error_msg) except Exception: parsed = None if isinstance(parsed, dict) and parsed.get("status", "").endswith("_failed"): assert "validation errors" in parsed.get("error", "") else: assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection", "Model"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_thinking_mode_medium(self): """Test medium thinking mode (default for most tools) using real integration testing""" import importlib import os # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for OpenAI provider (which supports thinking mode) os.environ["OPENAI_API_KEY"] = "sk-test-key-medium-thinking-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = DebugIssueTool() # Test with real provider resolution try: result = await tool.execute( { "prompt": "Test error", "model": "o3-mini", # Not specifying thinking_mode, should use default (medium) } ) # If we get here, provider resolution worked assert result is not None # Should be a valid debug response assert len(result) == 1 except Exception as e: # Expected: API call will fail with fake key error_msg = getattr(e, "payload", str(e)) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error import json try: parsed = json.loads(error_msg) except Exception: parsed = None if isinstance(parsed, dict) and parsed.get("status", "").endswith("_failed"): assert "validation errors" in parsed.get("error", "") else: assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection", "Model"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_thinking_mode_high(self): """Test high thinking mode with real provider resolution""" import importlib import os # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for OpenAI provider (which supports thinking mode) os.environ["OPENAI_API_KEY"] = "sk-test-key-high-thinking-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = AnalyzeTool() # Test with real provider resolution try: result = await tool.execute( { "absolute_file_paths": ["/absolute/path/complex.py"], "prompt": "Analyze architecture", "thinking_mode": "high", "model": "o3-mini", } ) # If we get here, provider resolution worked assert result is not None except Exception as e: # Expected: API call will fail with fake key error_msg = getattr(e, "payload", str(e)) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error import json try: parsed = json.loads(error_msg) except Exception: parsed = None if isinstance(parsed, dict) and parsed.get("status", "").endswith("_failed"): assert "validation errors" in parsed.get("error", "") else: assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection", "Model"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None @pytest.mark.asyncio async def test_thinking_mode_max(self): """Test max thinking mode (default for thinkdeep) using real integration testing""" import importlib import os # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), "DEFAULT_THINKING_MODE_THINKDEEP": os.environ.get("DEFAULT_THINKING_MODE_THINKDEEP"), } try: # Set up environment for OpenAI provider (which supports thinking mode) os.environ["OPENAI_API_KEY"] = "sk-test-key-max-thinking-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" os.environ["DEFAULT_THINKING_MODE_THINKDEEP"] = "high" # Set default to high for thinkdeep # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None tool = ThinkDeepTool() # Test with real provider resolution try: result = await tool.execute( { "prompt": "Initial analysis", "model": "o3-mini", # Not specifying thinking_mode, should use default (high) } ) # If we get here, provider resolution worked assert result is not None # Should be a valid thinkdeep response assert len(result) == 1 except Exception as e: # Expected: API call will fail with fake key error_msg = getattr(e, "payload", str(e)) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error import json try: parsed = json.loads(error_msg) except Exception: parsed = None if isinstance(parsed, dict) and parsed.get("status", "").endswith("_failed"): assert "validation errors" in parsed.get("error", "") else: assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection", "Model"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None ================================================ FILE: tests/test_tools.py ================================================ """ Tests for individual tool implementations """ import json import shutil import tempfile import pytest from tools import AnalyzeTool, ChatTool, CodeReviewTool, ThinkDeepTool from tools.shared.exceptions import ToolExecutionError class TestThinkDeepTool: """Test the thinkdeep tool""" @pytest.fixture def tool(self): return ThinkDeepTool() def test_tool_metadata(self, tool): """Test tool metadata""" assert tool.get_name() == "thinkdeep" assert "investigation and reasoning" in tool.get_description() assert tool.get_default_temperature() == 1.0 schema = tool.get_input_schema() # ThinkDeep is now a workflow tool with step-based fields assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "findings" in schema["properties"] # Required fields for workflow expected_required = {"step", "step_number", "total_steps", "next_step_required", "findings"} assert expected_required.issubset(set(schema["required"])) @pytest.mark.asyncio async def test_execute_success(self, tool): """Test successful execution using real integration testing""" import importlib import os # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for real provider resolution os.environ["OPENAI_API_KEY"] = "sk-test-key-thinkdeep-success-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys to isolate to OpenAI for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None # Test with real provider resolution try: result = await tool.execute( { "step": "Initial analysis", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial thinking about building a cache", "problem_context": "Building a cache", "focus_areas": ["performance", "scalability"], "model": "o3-mini", } ) # If we get here, check the response format assert len(result) == 1 # Should be a valid JSON response output = json.loads(result[0].text) assert "status" in output except Exception as e: # Expected: API call will fail with fake key error_msg = str(e) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None class TestCodeReviewTool: """Test the codereview tool""" @pytest.fixture def tool(self): return CodeReviewTool() def test_tool_metadata(self, tool): """Test tool metadata""" assert tool.get_name() == "codereview" assert "code review" in tool.get_description() assert tool.get_default_temperature() == 1.0 schema = tool.get_input_schema() assert "relevant_files" in schema["properties"] assert "step" in schema["properties"] assert "step_number" in schema["required"] @pytest.mark.asyncio async def test_execute_with_review_type(self, tool, tmp_path): """Test execution with specific review type using real provider resolution""" import importlib import os # Create test file test_file = tmp_path / "test.py" test_file.write_text("def insecure(): pass", encoding="utf-8") # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for testing os.environ["OPENAI_API_KEY"] = "sk-test-key-codereview-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None # Test with real provider resolution - expect it to fail at API level try: result = await tool.execute( { "step": "Review for security issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial security review", "relevant_files": [str(test_file)], "model": "o3-mini", } ) # If we somehow get here, that's fine too assert result is not None except Exception as e: # Expected: API call will fail with fake key error_msg = str(e) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None class TestAnalyzeTool: """Test the analyze tool""" @pytest.fixture def tool(self): return AnalyzeTool() def test_tool_metadata(self, tool): """Test tool metadata""" assert tool.get_name() == "analyze" assert "code analysis" in tool.get_description() assert tool.get_default_temperature() == 1.0 schema = tool.get_input_schema() # New workflow tool requires step-based fields assert "step" in schema["properties"] assert "step_number" in schema["properties"] assert "total_steps" in schema["properties"] assert "next_step_required" in schema["properties"] assert "findings" in schema["properties"] # Workflow tools use relevant_files instead of files assert "relevant_files" in schema["properties"] # Required fields for workflow expected_required = {"step", "step_number", "total_steps", "next_step_required", "findings"} assert expected_required.issubset(set(schema["required"])) @pytest.mark.asyncio async def test_execute_with_analysis_type(self, tool, tmp_path): """Test execution with specific analysis type using real provider resolution""" import importlib import os # Create test file test_file = tmp_path / "module.py" test_file.write_text("class Service: pass", encoding="utf-8") # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for testing os.environ["OPENAI_API_KEY"] = "sk-test-key-analyze-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None # Test with real provider resolution - expect it to fail at API level try: result = await tool.execute( { "step": "Analyze the structure of this code", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial analysis of code structure", "relevant_files": [str(test_file)], "analysis_type": "architecture", "output_format": "summary", "model": "o3-mini", } ) # If we somehow get here, that's fine too assert result is not None except Exception as e: # Expected: API call will fail with fake key error_msg = str(e) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None class TestAbsolutePathValidation: """Test absolute path validation across all tools""" # Removed: test_analyze_tool_relative_path_rejected - workflow tool handles validation differently # NOTE: CodeReview tool test has been commented out because the codereview tool has been # refactored to use a workflow-based pattern. The workflow tools handle path validation # differently and may accept relative paths in step 1 since validation happens at the # file reading stage. See simulator_tests/test_codereview_validation.py for comprehensive # workflow testing of the new codereview tool. @pytest.mark.asyncio async def test_thinkdeep_tool_relative_path_rejected(self): """Test that thinkdeep tool rejects relative paths""" tool = ThinkDeepTool() with pytest.raises(ToolExecutionError) as exc_info: await tool.execute( { "step": "My analysis", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial analysis", "files_checked": ["./local/file.py"], } ) response = json.loads(exc_info.value.payload) assert response["status"] == "error" assert "must be FULL absolute paths" in response["content"] assert "./local/file.py" in response["content"] @pytest.mark.asyncio async def test_chat_tool_relative_path_rejected(self): """Test that chat tool rejects relative paths""" tool = ChatTool() temp_dir = tempfile.mkdtemp() try: with pytest.raises(ToolExecutionError) as exc_info: await tool.execute( { "prompt": "Explain this code", "absolute_file_paths": ["code.py"], # relative path without ./ "working_directory_absolute_path": temp_dir, } ) finally: shutil.rmtree(temp_dir, ignore_errors=True) response = json.loads(exc_info.value.payload) assert response["status"] == "error" assert "must be FULL absolute paths" in response["content"] assert "code.py" in response["content"] @pytest.mark.asyncio async def test_analyze_tool_accepts_absolute_paths(self): """Test that analyze tool accepts absolute paths using real provider resolution""" import importlib import os tool = AnalyzeTool() # Save original environment original_env = { "OPENAI_API_KEY": os.environ.get("OPENAI_API_KEY"), "DEFAULT_MODEL": os.environ.get("DEFAULT_MODEL"), } try: # Set up environment for testing os.environ["OPENAI_API_KEY"] = "sk-test-key-absolute-path-test-not-real" os.environ["DEFAULT_MODEL"] = "o3-mini" # Clear other provider keys for key in ["GEMINI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY"]: os.environ.pop(key, None) # Reload config and clear registry import config importlib.reload(config) from providers.registry import ModelProviderRegistry ModelProviderRegistry._instance = None # Test with real provider resolution - expect it to fail at API level try: result = await tool.execute( { "step": "Analyze this code file", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Initial code analysis", "relevant_files": ["/absolute/path/file.py"], "model": "o3-mini", } ) # If we somehow get here, that's fine too assert result is not None except Exception as e: # Expected: API call will fail with fake key error_msg = str(e) # Should NOT be a mock-related error assert "MagicMock" not in error_msg assert "'<' not supported between instances" not in error_msg # Should be a real provider error assert any( phrase in error_msg for phrase in ["API", "key", "authentication", "provider", "network", "connection"] ) finally: # Restore environment for key, value in original_env.items(): if value is not None: os.environ[key] = value else: os.environ.pop(key, None) # Reload config and clear registry importlib.reload(config) ModelProviderRegistry._instance = None class TestSpecialStatusModels: """Test SPECIAL_STATUS_MODELS registry and structured response handling""" def test_trace_complete_status_in_registry(self): """Test that trace_complete status is properly registered""" from tools.models import SPECIAL_STATUS_MODELS, TraceComplete assert "trace_complete" in SPECIAL_STATUS_MODELS assert SPECIAL_STATUS_MODELS["trace_complete"] == TraceComplete def test_trace_complete_model_validation(self): """Test TraceComplete model validation""" from tools.models import TraceComplete # Test precision mode precision_data = { "status": "trace_complete", "trace_type": "precision", "entry_point": { "file": "/path/to/file.py", "class_or_struct": "MyClass", "method": "myMethod", "signature": "def myMethod(self, param1: str) -> bool", "parameters": {"param1": "test"}, }, "call_path": [ { "from": {"file": "/path/to/file.py", "class": "MyClass", "method": "myMethod", "line": 10}, "to": {"file": "/path/to/other.py", "class": "OtherClass", "method": "otherMethod", "line": 20}, "reason": "direct call", "condition": None, "ambiguous": False, } ], } model = TraceComplete(**precision_data) assert model.status == "trace_complete" assert model.trace_type == "precision" assert model.entry_point.file == "/path/to/file.py" assert len(model.call_path) == 1 # Test dependencies mode dependencies_data = { "status": "trace_complete", "trace_type": "dependencies", "target": { "file": "/path/to/file.py", "class_or_struct": "MyClass", "method": "myMethod", "signature": "def myMethod(self, param1: str) -> bool", }, "incoming_dependencies": [ { "from_file": "/path/to/caller.py", "from_class": "CallerClass", "from_method": "callerMethod", "line": 15, "type": "direct_call", } ], "outgoing_dependencies": [ { "to_file": "/path/to/dependency.py", "to_class": "DepClass", "to_method": "depMethod", "line": 25, "type": "method_call", } ], } model = TraceComplete(**dependencies_data) assert model.status == "trace_complete" assert model.trace_type == "dependencies" assert model.target.file == "/path/to/file.py" assert len(model.incoming_dependencies) == 1 assert len(model.outgoing_dependencies) == 1 ================================================ FILE: tests/test_tracer.py ================================================ """ Tests for the tracer tool functionality """ import pytest from tools.models import ToolModelCategory from tools.tracer import TracerRequest, TracerTool class TestTracerTool: """Test suite for the Tracer tool""" @pytest.fixture def tracer_tool(self): """Create a tracer tool instance for testing""" return TracerTool() def test_get_name(self, tracer_tool): """Test that the tool returns the correct name""" assert tracer_tool.get_name() == "tracer" def test_get_description(self, tracer_tool): """Test that the tool returns a comprehensive description""" description = tracer_tool.get_description() assert "code tracing" in description assert "precision" in description assert "dependencies" in description assert "systematic" in description def test_get_input_schema(self, tracer_tool): """Test that the input schema includes required fields""" schema = tracer_tool.get_input_schema() assert schema["type"] == "object" assert "target_description" in schema["properties"] assert "trace_mode" in schema["properties"] assert "step" in schema["properties"] assert "step_number" in schema["properties"] # Check trace_mode enum values trace_enum = schema["properties"]["trace_mode"]["enum"] assert "precision" in trace_enum assert "dependencies" in trace_enum # Check required fields include workflow fields required_fields = set(schema["required"]) assert "target_description" in required_fields assert "trace_mode" in required_fields def test_get_model_category(self, tracer_tool): """Test that the tracer tool uses EXTENDED_REASONING category""" category = tracer_tool.get_model_category() assert category == ToolModelCategory.EXTENDED_REASONING def test_request_model_validation(self, tracer_tool): """Test TracerRequest model validation""" # Valid request request = TracerRequest( step="Analyze BookingManager finalizeInvoice method execution flow", step_number=1, total_steps=3, next_step_required=True, findings="Initial investigation of booking finalization process", target_description="BookingManager finalizeInvoice method", trace_mode="precision", ) assert request.target_description == "BookingManager finalizeInvoice method" assert request.trace_mode == "precision" assert request.step_number == 1 # Test invalid trace_mode with pytest.raises(ValueError): TracerRequest( step="Test step", step_number=1, total_steps=1, next_step_required=False, findings="Test findings", trace_mode="invalid_mode", ) def test_get_required_actions(self, tracer_tool): """Test that required actions are provided for each step""" # Step 1 - initial investigation (in ask mode by default) actions = tracer_tool.get_required_actions(1, "exploring", "Initial findings", 3) assert len(actions) > 0 # Default is ask mode, so should ask for mode selection if tracer_tool.get_trace_mode() == "ask": assert any("ask user" in action.lower() for action in actions) assert any("precision mode" in action.lower() for action in actions) # Test with initialized trace_config for non-ask mode tracer_tool.trace_config = {"trace_mode": "precision"} actions = tracer_tool.get_required_actions(1, "exploring", "Initial findings", 3) assert len(actions) > 0 assert any("search" in action.lower() for action in actions) assert any("locate" in action.lower() for action in actions) # Later steps with low confidence actions = tracer_tool.get_required_actions(2, "low", "Some findings", 3) assert len(actions) > 0 assert any("trace" in action.lower() for action in actions) # High confidence steps actions = tracer_tool.get_required_actions(3, "high", "Strong findings", 3) assert len(actions) > 0 assert any("verify" in action.lower() for action in actions) def test_workflow_tool_characteristics(self, tracer_tool): """Test that tracer has proper workflow tool characteristics""" # Should not require external expert analysis assert not tracer_tool.requires_expert_analysis() # Should return TracerRequest as the workflow model assert tracer_tool.get_workflow_request_model() == TracerRequest # Should not require AI model at MCP boundary assert not tracer_tool.requires_model() def test_get_rendering_instructions_precision(self, tracer_tool): """Test rendering instructions for precision mode""" instructions = tracer_tool._get_rendering_instructions("precision") assert "PRECISION TRACE" in instructions assert "CALL FLOW DIAGRAM" in instructions assert "ADDITIONAL ANALYSIS VIEWS" in instructions assert "ClassName::MethodName" in instructions assert "↓" in instructions def test_get_rendering_instructions_dependencies(self, tracer_tool): """Test rendering instructions for dependencies mode""" instructions = tracer_tool._get_rendering_instructions("dependencies") assert "DEPENDENCIES TRACE" in instructions assert "DEPENDENCY FLOW DIAGRAM" in instructions assert "DEPENDENCY TABLE" in instructions assert "INCOMING DEPENDENCIES" in instructions assert "OUTGOING DEPENDENCIES" in instructions assert "←" in instructions assert "→" in instructions def test_rendering_instructions_consistency(self, tracer_tool): """Test that rendering instructions are consistent between modes""" precision_instructions = tracer_tool._get_precision_rendering_instructions() dependencies_instructions = tracer_tool._get_dependencies_rendering_instructions() # Both should have mandatory instructions assert "MANDATORY RENDERING INSTRUCTIONS" in precision_instructions assert "MANDATORY RENDERING INSTRUCTIONS" in dependencies_instructions # Both should have specific styling requirements assert "ONLY" in precision_instructions assert "ONLY" in dependencies_instructions # Both should have absolute requirements assert "ABSOLUTE REQUIREMENTS" in precision_instructions assert "ABSOLUTE REQUIREMENTS" in dependencies_instructions def test_mode_selection_guidance(self, tracer_tool): """Test that the schema provides clear guidance on when to use each mode""" schema = tracer_tool.get_input_schema() trace_mode_desc = schema["properties"]["trace_mode"]["description"] # Should clearly indicate precision is for methods/functions assert "execution flow" in trace_mode_desc # Should clearly indicate dependencies is for structural relationships assert "structural relationships" in trace_mode_desc ================================================ FILE: tests/test_utf8_localization.py ================================================ """ Unit tests to validate UTF-8 localization and encoding of French characters. These tests check: 1. Language instruction generation according to LOCALE 2. UTF-8 encoding with json.dumps(ensure_ascii=False) 3. French characters and emojis are displayed correctly 4. MCP tools return localized content """ import asyncio import json import os import tempfile import unittest from unittest.mock import Mock from tools.shared.base_tool import BaseTool class MockTestTool(BaseTool): """Concrete implementation of BaseTool for testing.""" def __init__(self): super().__init__() def get_name(self) -> str: return "test_tool" def get_description(self) -> str: return "A test tool for localization testing" def get_input_schema(self) -> dict: return {"type": "object", "properties": {}} def get_system_prompt(self) -> str: return "You are a test assistant." def get_request_model(self): from tools.shared.base_models import ToolRequest return ToolRequest async def prepare_prompt(self, request) -> str: return "Test prompt" async def execute(self, arguments: dict) -> list: return [Mock(text="test response")] class TestUTF8Localization(unittest.TestCase): """Tests for UTF-8 localization and French character encoding.""" def setUp(self): """Test setup.""" self.original_locale = os.getenv("LOCALE") def tearDown(self): """Cleanup after tests.""" if self.original_locale is not None: os.environ["LOCALE"] = self.original_locale else: os.environ.pop("LOCALE", None) def test_language_instruction_generation_french(self): """Test language instruction generation for French.""" # Set LOCALE to French os.environ["LOCALE"] = "fr-FR" # Test get_language_instruction method tool = MockTestTool() instruction = tool.get_language_instruction() # Checks self.assertIsInstance(instruction, str) self.assertIn("fr-FR", instruction) self.assertTrue(instruction.endswith("\n\n")) def test_language_instruction_generation_english(self): """Test language instruction generation for English.""" # Set LOCALE to English os.environ["LOCALE"] = "en-US" tool = MockTestTool() instruction = tool.get_language_instruction() # Checks self.assertIsInstance(instruction, str) self.assertIn("en-US", instruction) self.assertTrue(instruction.endswith("\n\n")) def test_language_instruction_empty_locale(self): """Test with empty LOCALE.""" # Set LOCALE to empty os.environ["LOCALE"] = "" tool = MockTestTool() instruction = tool.get_language_instruction() # Should return empty string self.assertEqual(instruction, "") def test_language_instruction_no_locale(self): """Test with no LOCALE variable set.""" # Remove LOCALE os.environ.pop("LOCALE", None) tool = MockTestTool() instruction = tool.get_language_instruction() # Should return empty string self.assertEqual(instruction, "") def test_json_dumps_utf8_encoding(self): """Test that json.dumps uses ensure_ascii=False for UTF-8.""" # Test data with French characters and emojis test_data = { "status": "succès", "message": "Tâche terminée avec succès", "details": { "créé": "2024-01-01", "développeur": "Jean Dupont", "préférences": ["français", "développement"], "emojis": "🔴 🟠 🟡 🟢 ✅ ❌", }, } # Test with ensure_ascii=False (correct) json_correct = json.dumps(test_data, ensure_ascii=False, indent=2) # Check that UTF-8 characters are preserved self.assertIn("succès", json_correct) self.assertIn("terminée", json_correct) self.assertIn("créé", json_correct) self.assertIn("développeur", json_correct) self.assertIn("préférences", json_correct) self.assertIn("français", json_correct) self.assertIn("développement", json_correct) self.assertIn("🔴", json_correct) self.assertIn("🟢", json_correct) self.assertIn("✅", json_correct) # Check that characters are NOT escaped self.assertNotIn("\\u", json_correct) self.assertNotIn("\\ud83d", json_correct) def test_json_dumps_ascii_encoding_comparison(self): """Test comparison between ensure_ascii=True and False.""" test_data = {"message": "Développement réussi! 🎉"} # With ensure_ascii=True (old, incorrect behavior) json_escaped = json.dumps(test_data, ensure_ascii=True) # With ensure_ascii=False (new, correct behavior) json_utf8 = json.dumps(test_data, ensure_ascii=False) # Checks self.assertIn("\\u", json_escaped) # Characters are escaped self.assertNotIn("é", json_escaped) # UTF-8 characters are escaped self.assertNotIn("\\u", json_utf8) # No escaped characters self.assertIn("é", json_utf8) # UTF-8 characters preserved self.assertIn("🎉", json_utf8) # Emojis preserved def test_french_characters_in_file_content(self): """Test reading and writing files with French characters.""" # Test content with French characters test_content = """ # System configuration # Created by: Lead Developer # Creation date: December 15, 2024 def process_data(preferences, parameters): ""\" Processes data according to user preferences. Args: preferences: User preferences dictionary parameters: Configuration parameters Returns: Processing result ""\" return "Processing completed successfully! ✅" # Helper functions def generate_report(): ""\"Generates a summary report.""\" return { "status": "success", "data": "Report generated", "emojis": "📊 📈 📉" } """ # Test writing and reading with tempfile.NamedTemporaryFile(mode="w+", encoding="utf-8", delete=False) as f: f.write(test_content) temp_file = f.name try: # Read file with open(temp_file, encoding="utf-8") as f: read_content = f.read() # Checks self.assertEqual(read_content, test_content) self.assertIn("Lead Developer", read_content) self.assertIn("Creation", read_content) self.assertIn("preferences", read_content) self.assertIn("parameters", read_content) self.assertIn("completed", read_content) self.assertIn("successfully", read_content) self.assertIn("✅", read_content) self.assertIn("success", read_content) self.assertIn("generated", read_content) self.assertIn("📊", read_content) finally: # Cleanup os.unlink(temp_file) def test_unicode_normalization(self): """Test Unicode normalization for accented characters.""" # Test with different Unicode encodings test_cases = [ "café", # e + acute accent combined "café", # e with precomposed acute accent "naïf", # i + diaeresis "coeur", # oe ligature "été", # e + acute accent ] for text in test_cases: # Test that json.dumps preserves characters json_output = json.dumps({"text": text}, ensure_ascii=False) self.assertIn(text, json_output) # Parse and check parsed = json.loads(json_output) self.assertEqual(parsed["text"], text) def test_emoji_preservation(self): """Test emoji preservation in JSON encoding.""" # Emojis used in PAL MCP tools emojis = [ "🔴", # Critical "🟠", # High "🟡", # Medium "🟢", # Low "✅", # Success "❌", # Error "⚠️", # Warning "📊", # Charts "🎉", # Celebration "🚀", # Rocket "🇫🇷", # French flag ] test_data = {"emojis": emojis, "message": " ".join(emojis)} # Test with ensure_ascii=False json_output = json.dumps(test_data, ensure_ascii=False) # Checks for emoji in emojis: self.assertIn(emoji, json_output) # No escaped characters self.assertNotIn("\\u", json_output) # Test parsing parsed = json.loads(json_output) self.assertEqual(parsed["emojis"], emojis) self.assertEqual(parsed["message"], " ".join(emojis)) class TestLocalizationIntegration(unittest.TestCase): """Integration tests for localization with real tools.""" def setUp(self): """Integration test setup.""" self.original_locale = os.getenv("LOCALE") def tearDown(self): """Cleanup after integration tests.""" if self.original_locale is not None: os.environ["LOCALE"] = self.original_locale else: os.environ.pop("LOCALE", None) def test_codereview_tool_french_locale_simple(self): """Test that the codereview tool correctly handles French locale configuration.""" # Set to French original_locale = os.environ.get("LOCALE") os.environ["LOCALE"] = "fr-FR" try: # Test language instruction generation from tools.codereview import CodeReviewTool codereview_tool = CodeReviewTool() # Test that the tool correctly gets language instruction for French language_instruction = codereview_tool.get_language_instruction() # Should contain French locale self.assertIn("fr-FR", language_instruction) # Should contain language instruction format self.assertIn("respond in", language_instruction.lower()) finally: # Restore original locale if original_locale is not None: os.environ["LOCALE"] = original_locale else: os.environ.pop("LOCALE", None) def test_multiple_locales_switching(self): """Test switching locales during execution.""" tool = MockTestTool() # French os.environ["LOCALE"] = "fr-FR" instruction_fr = tool.get_language_instruction() self.assertIn("fr-FR", instruction_fr) # English os.environ["LOCALE"] = "en-US" instruction_en = tool.get_language_instruction() self.assertIn("en-US", instruction_en) # Spanish os.environ["LOCALE"] = "es-ES" instruction_es = tool.get_language_instruction() self.assertIn("es-ES", instruction_es) # Chinese os.environ["LOCALE"] = "zh-CN" instruction_zh = tool.get_language_instruction() self.assertIn("zh-CN", instruction_zh) # Check that all instructions are different instructions = [ instruction_fr, instruction_en, instruction_es, instruction_zh, ] for i, inst1 in enumerate(instructions): for j, inst2 in enumerate(instructions): if i != j: self.assertNotEqual(inst1, inst2) # Helper function to run async tests def run_async_test(test_func): """Helper to run async test functions.""" return asyncio.run(test_func()) if __name__ == "__main__": unittest.main(verbosity=2) ================================================ FILE: tests/test_utils.py ================================================ """ Tests for utility functions """ from utils import check_token_limit, estimate_tokens, read_file_content, read_files class TestFileUtils: """Test file reading utilities""" def test_read_file_content_success(self, project_path): """Test successful file reading""" test_file = project_path / "test.py" test_file.write_text("def hello():\n return 'world'", encoding="utf-8") content, tokens = read_file_content(str(test_file)) assert "--- BEGIN FILE:" in content assert "--- END FILE:" in content assert "def hello():" in content assert "return 'world'" in content assert tokens > 0 # Should have estimated tokens def test_read_file_content_not_found(self, project_path): """Test reading non-existent file""" # Use a non-existent file within the project path nonexistent = project_path / "nonexistent" / "file.py" content, tokens = read_file_content(str(nonexistent)) assert "--- FILE NOT FOUND:" in content assert "Error: File does not exist" in content assert tokens > 0 def test_read_file_content_dangerous_files_blocked(self): """Test that dangerous system files are blocked""" # /etc/passwd should be blocked as it's under /etc (dangerous path) content, tokens = read_file_content("/etc/passwd") assert "--- ERROR ACCESSING FILE:" in content assert "Access to system directory denied" in content assert tokens > 0 def test_read_file_content_relative_path_rejected(self): """Test that relative paths are rejected""" # Try to use a relative path content, tokens = read_file_content("./some/relative/path.py") assert "--- ERROR ACCESSING FILE:" in content assert "Relative paths are not supported" in content assert tokens > 0 def test_read_file_content_directory(self, project_path): """Test reading a directory""" content, tokens = read_file_content(str(project_path)) assert "--- NOT A FILE:" in content assert "Error: Path is not a file" in content assert tokens > 0 def test_read_files_multiple(self, project_path): """Test reading multiple files""" file1 = project_path / "file1.py" file1.write_text("print('file1')", encoding="utf-8") file2 = project_path / "file2.py" file2.write_text("print('file2')", encoding="utf-8") content = read_files([str(file1), str(file2)]) assert "--- BEGIN FILE:" in content assert "file1.py" in content assert "file2.py" in content assert "print('file1')" in content assert "print('file2')" in content # Check that both files are included assert "file1.py" in content and "file2.py" in content def test_read_files_with_code(self): """Test reading with direct code""" code = "def test():\n pass" content = read_files([], code) assert "--- BEGIN DIRECT CODE ---" in content assert "--- END DIRECT CODE ---" in content assert code in content # Check that direct code is included assert code in content def test_read_files_directory_support(self, project_path): """Test reading all files from a directory""" # Create directory structure (project_path / "file1.py").write_text("print('file1')", encoding="utf-8") (project_path / "file2.js").write_text("console.log('file2')", encoding="utf-8") (project_path / "readme.md").write_text("# README", encoding="utf-8") # Create subdirectory subdir = project_path / "src" subdir.mkdir() (subdir / "module.py").write_text("class Module: pass", encoding="utf-8") # Create hidden file (should be skipped) (project_path / ".hidden").write_text("secret", encoding="utf-8") # Read the directory content = read_files([str(project_path)]) # Check files are included assert "file1.py" in content assert "file2.js" in content assert "readme.md" in content # Handle both forward and backslashes for cross-platform compatibility assert "module.py" in content assert "class Module: pass" in content # Check content assert "print('file1')" in content assert "console.log('file2')" in content assert "# README" in content assert "class Module: pass" in content # Hidden file should not be included assert ".hidden" not in content assert "secret" not in content # Check that all files are included assert all(filename in content for filename in ["file1.py", "file2.js", "readme.md", "module.py"]) def test_read_files_mixed_paths(self, project_path): """Test reading mix of files and directories""" # Create files file1 = project_path / "direct.py" file1.write_text("# Direct file", encoding="utf-8") # Create directory with files subdir = project_path / "subdir" subdir.mkdir() (subdir / "sub1.py").write_text("# Sub file 1", encoding="utf-8") (subdir / "sub2.py").write_text("# Sub file 2", encoding="utf-8") # Read mix of direct file and directory content = read_files([str(file1), str(subdir)]) assert "direct.py" in content assert "sub1.py" in content assert "sub2.py" in content assert "# Direct file" in content assert "# Sub file 1" in content assert "# Sub file 2" in content # Check that all files are included assert all(filename in content for filename in ["direct.py", "sub1.py", "sub2.py"]) def test_read_files_token_limit(self, project_path): """Test token limit handling""" # Create files with known token counts # ~250 tokens each (1000 chars) large_content = "x" * 1000 for i in range(5): (project_path / f"file{i}.txt").write_text(large_content, encoding="utf-8") # Read with small token limit (should skip some files) # Reserve 50k tokens, limit to 51k total = 1k available # Each file ~250 tokens, so should read ~3-4 files content = read_files([str(project_path)], max_tokens=51_000) # Check that token limit handling is present assert "--- SKIPPED FILES (TOKEN LIMIT) ---" in content # Count how many files were read read_count = content.count("--- BEGIN FILE:") assert 2 <= read_count <= 4 # Should read some but not all def test_read_files_large_file(self, project_path): """Test handling of large files""" # Create a file larger than max_size (1MB) large_file = project_path / "large.txt" large_file.write_text("x" * 2_000_000, encoding="utf-8") # 2MB content = read_files([str(large_file)]) assert "--- FILE TOO LARGE:" in content assert "2,000,000 bytes" in content # File too large message should be present assert "--- FILE TOO LARGE:" in content def test_read_files_file_extensions(self, project_path): """Test file extension filtering""" # Create various file types (project_path / "code.py").write_text("python", encoding="utf-8") (project_path / "style.css").write_text("css", encoding="utf-8") (project_path / "binary.exe").write_text("exe", encoding="utf-8") (project_path / "image.jpg").write_text("jpg", encoding="utf-8") content = read_files([str(project_path)]) # Code files should be included assert "code.py" in content assert "style.css" in content # Binary files should not be included (not in CODE_EXTENSIONS) assert "binary.exe" not in content assert "image.jpg" not in content class TestTokenUtils: """Test token counting utilities""" def test_estimate_tokens(self): """Test token estimation""" # Rough estimate: 1 token ≈ 4 characters text = "a" * 400 # 400 characters assert estimate_tokens(text) == 100 def test_check_token_limit_within(self): """Test token limit check - within limit""" text = "a" * 4000 # 1000 tokens within_limit, tokens = check_token_limit(text) assert within_limit is True assert tokens == 1000 def test_check_token_limit_exceeded(self): """Test token limit check - exceeded""" text = "a" * 5_000_000 # 1.25M tokens within_limit, tokens = check_token_limit(text) assert within_limit is False assert tokens == 1_250_000 ================================================ FILE: tests/test_uvx_resource_packaging.py ================================================ """Tests for uvx path resolution functionality.""" import json import tempfile from pathlib import Path from unittest.mock import patch from providers.registries.openrouter import OpenRouterModelRegistry class TestUvxPathResolution: """Test uvx path resolution for OpenRouter model registry.""" def test_normal_operation(self): """Test that normal operation works in development environment.""" registry = OpenRouterModelRegistry() assert len(registry.list_models()) > 0 assert len(registry.list_aliases()) > 0 def test_config_path_resolution(self): """Test that the config path resolution finds the config file in multiple locations.""" # Check that the config file exists in the development location config_file = Path(__file__).parent.parent / "conf" / "openrouter_models.json" assert config_file.exists(), "Config file should exist in conf/openrouter_models.json" # Test that a registry can find and use the config registry = OpenRouterModelRegistry() # When using resources, config_path is None; when using file system, it should exist if registry.use_resources: assert registry.config_path is None, "When using resources, config_path should be None" else: assert registry.config_path.exists(), "When using file system, config path should exist" assert len(registry.list_models()) > 0, "Registry should load models from config" def test_explicit_config_path_override(self): """Test that explicit config path works correctly.""" config_path = Path(__file__).parent.parent / "conf" / "openrouter_models.json" registry = OpenRouterModelRegistry(config_path=str(config_path)) # Should use the provided file path assert registry.config_path == config_path assert len(registry.list_models()) > 0 def test_environment_variable_override(self): """Test that CUSTOM_MODELS_CONFIG_PATH environment variable works.""" config_path = Path(__file__).parent.parent / "conf" / "openrouter_models.json" with patch.dict("os.environ", {"OPENROUTER_MODELS_CONFIG_PATH": str(config_path)}): registry = OpenRouterModelRegistry() # Should use environment path assert registry.config_path == config_path assert len(registry.list_models()) > 0 @patch("providers.registries.base.importlib.resources.files") def test_multiple_path_fallback(self, mock_files): """Test that file-system fallback works when resource loading fails.""" mock_files.side_effect = Exception("Resource loading failed") with tempfile.TemporaryDirectory() as tmpdir: temp_dir = Path(tmpdir) conf_dir = temp_dir / "conf" conf_dir.mkdir(parents=True, exist_ok=True) config_path = conf_dir / "openrouter_models.json" config_path.write_text( json.dumps( { "models": [ { "model_name": "test/model", "aliases": ["testalias"], "context_window": 1024, "max_output_tokens": 512, } ] }, indent=2, ) ) original_exists = Path.exists def fake_exists(path_self): if str(path_self).endswith("conf/openrouter_models.json") and path_self != config_path: return False if path_self == config_path: return True return original_exists(path_self) with patch("pathlib.Path.cwd", return_value=temp_dir), patch("pathlib.Path.exists", fake_exists): registry = OpenRouterModelRegistry() assert not registry.use_resources assert registry.config_path == config_path assert "test/model" in registry.list_models() def test_missing_config_handling(self): """Test behavior when config file is missing.""" # Use a non-existent path with patch.dict("os.environ", {}, clear=True): registry = OpenRouterModelRegistry(config_path="/nonexistent/path/config.json") # Should gracefully handle missing config assert len(registry.list_models()) == 0 assert len(registry.list_aliases()) == 0 def test_resource_loading_success(self): """Test successful resource loading via importlib.resources.""" # Just test that the registry works normally in our environment # This validates the resource loading mechanism indirectly registry = OpenRouterModelRegistry() # Should load successfully using either resources or file system fallback assert len(registry.list_models()) > 0 assert len(registry.list_aliases()) > 0 def test_use_resources_attribute(self): """Test that the use_resources attribute is properly set.""" registry = OpenRouterModelRegistry() # Should have the use_resources attribute assert hasattr(registry, "use_resources") assert isinstance(registry.use_resources, bool) ================================================ FILE: tests/test_uvx_support.py ================================================ """ Test cases for uvx support and environment handling. """ import os import sys import tempfile from pathlib import Path from unittest import mock import pytest class TestUvxEnvironmentHandling: """Test uvx-specific environment handling features.""" def test_dotenv_import_success(self): """Test that dotenv is imported successfully when available.""" # Mock successful dotenv import mock_load = mock.MagicMock() mock_values = mock.MagicMock(return_value={}) fake_dotenv = mock.MagicMock(load_dotenv=mock_load, dotenv_values=mock_values) with mock.patch.dict("sys.modules", {"dotenv": fake_dotenv}): if "utils.env" in sys.modules: del sys.modules["utils.env"] if "server" in sys.modules: del sys.modules["server"] import importlib import utils.env as env_config with tempfile.NamedTemporaryFile("w", delete=False) as tmp_env: temp_env_path = Path(tmp_env.name) tmp_env.write("PAL_MCP_FORCE_ENV_OVERRIDE=false\n") try: importlib.reload(env_config) env_config._ENV_PATH = temp_env_path env_config.reload_env() import server # noqa: F401 assert mock_load.call_count >= 1 _, kwargs = mock_load.call_args assert "dotenv_path" in kwargs finally: temp_env_path.unlink(missing_ok=True) def test_dotenv_import_failure_graceful_handling(self): """Test that ImportError for dotenv is handled gracefully (uvx scenario).""" # Mock only the dotenv import to fail original_import = __builtins__["__import__"] def mock_import(name, *args, **kwargs): if name == "dotenv": raise ImportError("No module named 'dotenv'") return original_import(name, *args, **kwargs) with mock.patch("builtins.__import__", side_effect=mock_import): # This should not raise an exception when trying to import dotenv try: from dotenv import load_dotenv # noqa: F401 pytest.fail("Should have raised ImportError for dotenv") except ImportError: # Expected behavior - ImportError should be caught gracefully in server.py pass def test_env_file_path_resolution(self): """Test that .env file path is correctly resolved relative to server.py.""" import server # Test that the server module correctly resolves .env path script_dir = Path(server.__file__).parent expected_env_file = script_dir / ".env" # The logic should create a path relative to server.py assert expected_env_file.name == ".env" assert expected_env_file.parent == script_dir def test_environment_variables_still_work_without_dotenv(self): """Test that environment variables work even when dotenv is not available.""" # Set a test environment variable test_key = "TEST_PAL_MCP_VAR" test_value = "test_value_123" with mock.patch.dict(os.environ, {test_key: test_value}): # Environment variable should still be accessible regardless of dotenv assert os.getenv(test_key) == test_value def test_dotenv_graceful_fallback_behavior(self): """Test the actual graceful fallback behavior in server module.""" # Test that server module handles missing dotenv gracefully # This is tested by the fact that the server can be imported even if dotenv fails import server # If we can import server, the graceful handling works assert hasattr(server, "run") # Test that environment variables still work test_key = "TEST_FALLBACK_VAR" test_value = "fallback_test_123" with mock.patch.dict(os.environ, {test_key: test_value}): assert os.getenv(test_key) == test_value class TestUvxProjectConfiguration: """Test uvx-specific project configuration features.""" def test_pyproject_toml_has_required_uvx_fields(self): """Test that pyproject.toml has all required fields for uvx support.""" try: import tomllib except ImportError: # tomllib is only available in Python 3.11+ # For older versions, use tomli or skip the test try: import tomli as tomllib except ImportError: pytest.skip("tomllib/tomli not available for TOML parsing") pyproject_path = Path(__file__).parent.parent / "pyproject.toml" assert pyproject_path.exists(), "pyproject.toml should exist" with open(pyproject_path, "rb") as f: pyproject_data = tomllib.load(f) # Check required uvx fields assert "project" in pyproject_data project = pyproject_data["project"] # Essential fields for uvx assert "name" in project assert project["name"] == "pal-mcp-server" assert "dependencies" in project assert "requires-python" in project # Script entry point for uvx assert "scripts" in project assert "pal-mcp-server" in project["scripts"] assert project["scripts"]["pal-mcp-server"] == "server:run" def test_pyproject_dependencies_match_requirements(self): """Test that pyproject.toml dependencies align with requirements.txt.""" try: import tomllib except ImportError: # tomllib is only available in Python 3.11+ try: import tomli as tomllib except ImportError: pytest.skip("tomllib/tomli not available for TOML parsing") # Read pyproject.toml pyproject_path = Path(__file__).parent.parent / "pyproject.toml" with open(pyproject_path, "rb") as f: pyproject_data = tomllib.load(f) pyproject_deps = set(pyproject_data["project"]["dependencies"]) # Read requirements.txt requirements_path = Path(__file__).parent.parent / "requirements.txt" if requirements_path.exists(): # Note: We primarily validate pyproject.toml has core dependencies # requirements.txt might have additional dev dependencies # Core dependencies should be present in both core_packages = {"mcp", "openai", "google-genai", "pydantic", "python-dotenv"} for pkg in core_packages: pyproject_has = any(pkg in dep for dep in pyproject_deps) assert pyproject_has, f"{pkg} should be in pyproject.toml dependencies" # requirements.txt might have additional dev dependencies def test_uvx_entry_point_callable(self): """Test that the uvx entry point (server:run) is callable.""" import server # The entry point should reference a callable function assert hasattr(server, "run"), "server module should have a 'run' function" assert callable(server.run), "server.run should be callable" ================================================ FILE: tests/test_workflow_file_embedding.py ================================================ """ Unit tests for workflow file embedding behavior Tests the critical file embedding logic for workflow tools: - Intermediate steps: Only reference file names (save Claude's context) - Final steps: Embed full file content for expert analysis """ import os import tempfile from unittest.mock import Mock, patch import pytest from tools.workflow.workflow_mixin import BaseWorkflowMixin class TestWorkflowFileEmbedding: """Test workflow file embedding behavior""" def setup_method(self): """Set up test fixtures""" # Create a mock workflow tool self.mock_tool = Mock() self.mock_tool.get_name.return_value = "test_workflow" # Bind the methods we want to test - use bound methods self.mock_tool._should_embed_files_in_workflow_step = ( BaseWorkflowMixin._should_embed_files_in_workflow_step.__get__(self.mock_tool) ) self.mock_tool._force_embed_files_for_expert_analysis = ( BaseWorkflowMixin._force_embed_files_for_expert_analysis.__get__(self.mock_tool) ) # Create test files self.test_files = [] for i in range(2): fd, path = tempfile.mkstemp(suffix=f"_test_{i}.py") with os.fdopen(fd, "w") as f: f.write(f"# Test file {i}\nprint('hello world {i}')\n") self.test_files.append(path) def teardown_method(self): """Clean up test files""" for file_path in self.test_files: try: os.unlink(file_path) except OSError: pass def test_intermediate_step_no_embedding(self): """Test that intermediate steps only reference files, don't embed""" # Intermediate step: step_number=1, next_step_required=True step_number = 1 continuation_id = None # New conversation is_final_step = False # next_step_required=True should_embed = self.mock_tool._should_embed_files_in_workflow_step(step_number, continuation_id, is_final_step) assert should_embed is False, "Intermediate steps should NOT embed files" def test_intermediate_step_with_continuation_no_embedding(self): """Test that intermediate steps with continuation only reference files""" # Intermediate step with continuation: step_number=2, next_step_required=True step_number = 2 continuation_id = "test-thread-123" # Continuing conversation is_final_step = False # next_step_required=True should_embed = self.mock_tool._should_embed_files_in_workflow_step(step_number, continuation_id, is_final_step) assert should_embed is False, "Intermediate steps with continuation should NOT embed files" def test_final_step_embeds_files(self): """Test that final steps embed full file content for expert analysis""" # Final step: any step_number, next_step_required=False step_number = 3 continuation_id = "test-thread-123" is_final_step = True # next_step_required=False should_embed = self.mock_tool._should_embed_files_in_workflow_step(step_number, continuation_id, is_final_step) assert should_embed is True, "Final steps SHOULD embed files for expert analysis" def test_final_step_new_conversation_embeds_files(self): """Test that final steps in new conversations embed files""" # Final step in new conversation (rare but possible): step_number=1, next_step_required=False step_number = 1 continuation_id = None # New conversation is_final_step = True # next_step_required=False (one-step workflow) should_embed = self.mock_tool._should_embed_files_in_workflow_step(step_number, continuation_id, is_final_step) assert should_embed is True, "Final steps in new conversations SHOULD embed files" @patch("utils.file_utils.read_files") @patch("utils.file_utils.expand_paths") @patch("utils.conversation_memory.get_thread") @patch("utils.conversation_memory.get_conversation_file_list") def test_comprehensive_file_collection_for_expert_analysis( self, mock_get_conversation_file_list, mock_get_thread, mock_expand_paths, mock_read_files ): """Test that expert analysis collects relevant files from current workflow and conversation history""" # Setup test files for different sources conversation_files = [self.test_files[0]] # relevant_files from conversation history current_relevant_files = [ self.test_files[0], self.test_files[1], ] # current step's relevant_files (overlap with conversation) # Setup mocks mock_thread_context = Mock() mock_get_thread.return_value = mock_thread_context mock_get_conversation_file_list.return_value = conversation_files mock_expand_paths.return_value = self.test_files mock_read_files.return_value = "# File content\nprint('test')" # Mock model context for token allocation mock_model_context = Mock() mock_token_allocation = Mock() mock_token_allocation.file_tokens = 100000 mock_model_context.calculate_token_allocation.return_value = mock_token_allocation # Set up the tool methods and state self.mock_tool.get_current_model_context.return_value = mock_model_context self.mock_tool.wants_line_numbers_by_default.return_value = True self.mock_tool.get_name.return_value = "test_workflow" # Set up consolidated findings self.mock_tool.consolidated_findings = Mock() self.mock_tool.consolidated_findings.relevant_files = set(current_relevant_files) # Set up current arguments with continuation self.mock_tool._current_arguments = {"continuation_id": "test-thread-123"} self.mock_tool.get_current_arguments.return_value = {"continuation_id": "test-thread-123"} # Bind the method we want to test self.mock_tool._prepare_files_for_expert_analysis = ( BaseWorkflowMixin._prepare_files_for_expert_analysis.__get__(self.mock_tool) ) self.mock_tool._force_embed_files_for_expert_analysis = ( BaseWorkflowMixin._force_embed_files_for_expert_analysis.__get__(self.mock_tool) ) # Call the method file_content = self.mock_tool._prepare_files_for_expert_analysis() # Verify it collected files from conversation history mock_get_thread.assert_called_once_with("test-thread-123") mock_get_conversation_file_list.assert_called_once_with(mock_thread_context) # Verify it called read_files with ALL unique relevant files # Should include files from: conversation_files + current_relevant_files # But deduplicated: [test_files[0], test_files[1]] (unique set) expected_unique_files = list(set(conversation_files + current_relevant_files)) # The actual call will be with whatever files were collected and deduplicated mock_read_files.assert_called_once() call_args = mock_read_files.call_args called_files = call_args[0][0] # First positional argument # Verify all expected files are included for expected_file in expected_unique_files: assert expected_file in called_files, f"Expected file {expected_file} not found in {called_files}" # Verify return value assert file_content == "# File content\nprint('test')" @patch("utils.file_utils.read_files") @patch("utils.file_utils.expand_paths") def test_force_embed_bypasses_conversation_history(self, mock_expand_paths, mock_read_files): """Test that _force_embed_files_for_expert_analysis bypasses conversation filtering""" # Setup mocks mock_expand_paths.return_value = self.test_files mock_read_files.return_value = "# File content\nprint('test')" # Mock model context for token allocation mock_model_context = Mock() mock_token_allocation = Mock() mock_token_allocation.file_tokens = 100000 mock_model_context.calculate_token_allocation.return_value = mock_token_allocation # Set up the tool methods self.mock_tool.get_current_model_context.return_value = mock_model_context self.mock_tool.wants_line_numbers_by_default.return_value = True # Call the method file_content, processed_files = self.mock_tool._force_embed_files_for_expert_analysis(self.test_files) # Verify it called read_files directly (bypassing conversation history filtering) mock_read_files.assert_called_once_with( self.test_files, max_tokens=100000, reserve_tokens=1000, include_line_numbers=True, ) # Verify it expanded paths to get individual files mock_expand_paths.assert_called_once_with(self.test_files) # Verify return values assert file_content == "# File content\nprint('test')" assert processed_files == self.test_files def test_embedding_decision_logic_comprehensive(self): """Comprehensive test of the embedding decision logic""" test_cases = [ # (step_number, continuation_id, is_final_step, expected_embed, description) (1, None, False, False, "Step 1 new conversation, intermediate"), (1, None, True, True, "Step 1 new conversation, final (one-step workflow)"), (2, "thread-123", False, False, "Step 2 with continuation, intermediate"), (2, "thread-123", True, True, "Step 2 with continuation, final"), (5, "thread-456", False, False, "Step 5 with continuation, intermediate"), (5, "thread-456", True, True, "Step 5 with continuation, final"), ] for step_number, continuation_id, is_final_step, expected_embed, description in test_cases: should_embed = self.mock_tool._should_embed_files_in_workflow_step( step_number, continuation_id, is_final_step ) assert should_embed == expected_embed, f"Failed for: {description}" if __name__ == "__main__": pytest.main([__file__]) ================================================ FILE: tests/test_workflow_metadata.py ================================================ """ Tests for workflow tool metadata functionality. This test ensures that workflow tools include metadata (provider_used and model_used) in their responses, similar to regular tools, for consistent tracking across all tool types. """ import json import os import pytest from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from tools.debug import DebugIssueTool from tools.shared.exceptions import ToolExecutionError class TestWorkflowMetadata: """Test cases for workflow tool metadata functionality.""" def setup_method(self): """Set up clean state before each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None # Clear provider registry registry = ModelProviderRegistry() registry._providers.clear() registry._initialized_providers.clear() def teardown_method(self): """Clean up after each test.""" # Clear restriction service cache import utils.model_restrictions utils.model_restrictions._restriction_service = None @pytest.mark.no_mock_provider def test_workflow_metadata_in_response(self): """ Test that workflow tools include metadata in their responses. This test verifies that workflow tools (like debug) include provider_used and model_used metadata in their responses, ensuring consistency with regular tools for tracking purposes. """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up test environment with OpenRouter API key os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("XAI_API_KEY", None) os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) # Clear any restrictions os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" # Register OpenRouter provider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Create debug tool debug_tool = DebugIssueTool() # Create mock model context like server.py does from utils.model_context import ModelContext model_name = "flash" model_context = ModelContext(model_name) # Create arguments with model context (like server.py provides) arguments = { "step": "Investigating the test issue to check metadata functionality", "step_number": 1, "total_steps": 2, "next_step_required": False, # Final step to trigger completion "findings": "Initial findings for test", "model": model_name, "confidence": "high", "_model_context": model_context, "_resolved_model_name": model_name, } # Execute the workflow tool import asyncio result = asyncio.run(debug_tool.execute_workflow(arguments)) # Parse the JSON response assert len(result) == 1 response_text = result[0].text response_data = json.loads(response_text) # Verify metadata is present assert "metadata" in response_data, "Workflow response should include metadata" metadata = response_data["metadata"] # Verify required metadata fields assert "tool_name" in metadata, "Metadata should include tool_name" assert "model_used" in metadata, "Metadata should include model_used" assert "provider_used" in metadata, "Metadata should include provider_used" # Verify metadata values assert metadata["tool_name"] == "debug", "tool_name should be 'debug'" assert metadata["model_used"] == model_name, f"model_used should be '{model_name}'" assert metadata["provider_used"] == "openrouter", "provider_used should be 'openrouter'" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_workflow_metadata_in_error_response(self): """ Test that workflow tools include metadata even in error responses. """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up test environment with OpenRouter API key os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("XAI_API_KEY", None) os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) # Clear any restrictions os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" # Register OpenRouter provider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Create debug tool debug_tool = DebugIssueTool() # Create arguments with invalid data to trigger error model_name = "flash" arguments = { "step": "Test step", "step_number": "invalid", # This should cause an error during validation "_resolved_model_name": model_name, } # Execute the workflow tool - should fail gracefully import asyncio with pytest.raises(ToolExecutionError) as exc_info: asyncio.run(debug_tool.execute(arguments)) response_data = json.loads(exc_info.value.payload) # Verify it's an error response with metadata assert "status" in response_data assert "error" in response_data or "content" in response_data assert "metadata" in response_data, "Error responses should include metadata" metadata = response_data["metadata"] assert "tool_name" in metadata, "Error metadata should include tool_name" assert metadata["tool_name"] == "debug", "tool_name should be 'debug'" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_workflow_metadata_fallback_handling(self): """ Test that workflow tools handle metadata gracefully when model context is missing. """ # Save original environment original_env = {} for key in ["OPENROUTER_ALLOWED_MODELS"]: original_env[key] = os.environ.get(key) try: # Clear any restrictions os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) # Create debug tool debug_tool = DebugIssueTool() # Create arguments without model context (fallback scenario) arguments = { "step": "Test step without model context", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Test findings", "model": "flash", "confidence": "low", # No _model_context or _resolved_model_name } # Execute the workflow tool import asyncio result = asyncio.run(debug_tool.execute_workflow(arguments)) # Parse the JSON response assert len(result) == 1 response_text = result[0].text response_data = json.loads(response_text) # Verify metadata is still present with fallback values assert "metadata" in response_data, "Workflow response should include metadata even in fallback" metadata = response_data["metadata"] # Verify fallback metadata assert "tool_name" in metadata, "Fallback metadata should include tool_name" assert "model_used" in metadata, "Fallback metadata should include model_used" assert "provider_used" in metadata, "Fallback metadata should include provider_used" assert metadata["tool_name"] == "debug", "tool_name should be 'debug'" assert metadata["model_used"] == "flash", "model_used should be from request" assert metadata["provider_used"] == "unknown", "provider_used should be 'unknown' in fallback" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value @pytest.mark.no_mock_provider def test_workflow_metadata_preserves_existing_response_fields(self): """ Test that adding metadata doesn't interfere with existing workflow response fields. """ # Save original environment original_env = {} for key in [ "GEMINI_API_KEY", "OPENAI_API_KEY", "XAI_API_KEY", "OPENROUTER_API_KEY", "OPENROUTER_ALLOWED_MODELS", ]: original_env[key] = os.environ.get(key) try: # Set up test environment os.environ.pop("GEMINI_API_KEY", None) os.environ.pop("OPENAI_API_KEY", None) os.environ.pop("XAI_API_KEY", None) os.environ.pop("OPENROUTER_ALLOWED_MODELS", None) # Clear any restrictions os.environ["OPENROUTER_API_KEY"] = "test-openrouter-key" # Register OpenRouter provider from providers.openrouter import OpenRouterProvider ModelProviderRegistry.register_provider(ProviderType.OPENROUTER, OpenRouterProvider) # Create debug tool debug_tool = DebugIssueTool() # Create mock model context from utils.model_context import ModelContext model_name = "flash" model_context = ModelContext(model_name) # Create arguments for intermediate step arguments = { "step": "Testing intermediate step for metadata preservation", "step_number": 1, "total_steps": 3, "next_step_required": True, # Intermediate step "findings": "Intermediate findings", "model": model_name, "confidence": "medium", "_model_context": model_context, "_resolved_model_name": model_name, } # Execute the workflow tool import asyncio result = asyncio.run(debug_tool.execute_workflow(arguments)) # Parse the JSON response assert len(result) == 1 response_text = result[0].text response_data = json.loads(response_text) # Verify standard workflow fields are preserved assert "status" in response_data, "Standard workflow status should be preserved" assert "step_number" in response_data, "Standard workflow step_number should be preserved" assert "total_steps" in response_data, "Standard workflow total_steps should be preserved" assert "next_step_required" in response_data, "Standard workflow next_step_required should be preserved" # Verify metadata was added without breaking existing fields assert "metadata" in response_data, "Metadata should be added" metadata = response_data["metadata"] assert metadata["tool_name"] == "debug" assert metadata["model_used"] == model_name assert metadata["provider_used"] == "openrouter" # Verify specific intermediate step fields assert response_data["next_step_required"] is True, "next_step_required should be preserved" assert response_data["step_number"] == 1, "step_number should be preserved" finally: # Restore original environment for key, value in original_env.items(): if value is None: os.environ.pop(key, None) else: os.environ[key] = value ================================================ FILE: tests/test_workflow_prompt_size_validation_simple.py ================================================ """Integration tests for workflow step size validation. These tests exercise the debug workflow tool end-to-end to ensure that step size validation operates on the real execution path rather than mocked helpers. """ from __future__ import annotations import json import pytest from config import MCP_PROMPT_SIZE_LIMIT from tools.debug import DebugIssueTool from tools.shared.exceptions import ToolExecutionError def build_debug_arguments(**overrides) -> dict[str, object]: """Create a minimal set of workflow arguments for DebugIssueTool.""" base_arguments: dict[str, object] = { "step": "Investigate the authentication issue in the login module", "step_number": 1, "total_steps": 3, "next_step_required": True, "findings": "Initial observations about the login failure", "files_checked": [], "relevant_files": [], "relevant_context": [], "issues_found": [], "confidence": "low", "use_assistant_model": False, # WorkflowRequest accepts optional fields; leave hypothesis/continuation unset } base_arguments.update(overrides) return base_arguments @pytest.mark.asyncio async def test_workflow_tool_accepts_normal_step_content() -> None: """Verify a typical step executes through the real workflow path.""" tool = DebugIssueTool() arguments = build_debug_arguments() responses = await tool.execute(arguments) assert len(responses) == 1 payload = json.loads(responses[0].text) assert payload["status"] == "pause_for_investigation" assert payload["step_number"] == 1 assert "error" not in payload @pytest.mark.asyncio async def test_workflow_tool_rejects_oversized_step_with_guidance() -> None: """Large step content should trigger the size safeguard with helpful guidance.""" oversized_step = "Investigate this issue: " + ("A" * (MCP_PROMPT_SIZE_LIMIT + 1000)) tool = DebugIssueTool() arguments = build_debug_arguments(step=oversized_step) with pytest.raises(ToolExecutionError) as exc_info: await tool.execute(arguments) output_payload = json.loads(exc_info.value.payload) assert output_payload["status"] == "resend_prompt" assert output_payload["metadata"]["prompt_size"] > MCP_PROMPT_SIZE_LIMIT guidance = output_payload["content"].lower() assert "shorter instructions" in guidance assert "file paths" in guidance ================================================ FILE: tests/test_workflow_utf8.py ================================================ """ Unit tests to validate UTF-8 encoding in workflow tools and the generation of properly encoded JSON responses. """ import json import os import unittest from unittest.mock import AsyncMock, Mock, patch from tools.analyze import AnalyzeTool from tools.codereview import CodeReviewTool from tools.debug import DebugIssueTool class TestWorkflowToolsUTF8(unittest.IsolatedAsyncioTestCase): """Tests for UTF-8 encoding in workflow tools.""" def setUp(self): """Test setup.""" self.original_locale = os.getenv("LOCALE") # Default to French for tests os.environ["LOCALE"] = "fr-FR" def tearDown(self): """Cleanup after tests.""" if self.original_locale is not None: os.environ["LOCALE"] = self.original_locale else: os.environ.pop("LOCALE", None) def test_workflow_json_response_structure(self): """Test the structure of JSON responses from workflow tools.""" # Mock response with UTF-8 characters test_response = { "status": "pause_for_analysis", "step_number": 1, "total_steps": 3, "next_step_required": True, "findings": "Code analysis reveals performance issues 🔍", "files_checked": ["/src/main.py"], "relevant_files": ["/src/main.py"], "issues_found": [{"severity": "high", "description": "Function too complex - refactoring needed"}], "investigation_required": True, "required_actions": ["Review code dependencies", "Analyze architectural patterns"], } # Test JSON serialization with ensure_ascii=False json_str = json.dumps(test_response, indent=2, ensure_ascii=False) # Check UTF-8 characters are preserved self.assertIn("🔍", json_str) # No escaped characters self.assertNotIn("\\u", json_str) # Test parsing parsed = json.loads(json_str) self.assertEqual(parsed["findings"], test_response["findings"]) self.assertEqual(len(parsed["issues_found"]), 1) @patch("tools.shared.base_tool.BaseTool.get_model_provider") @patch("utils.model_context.ModelContext") async def test_analyze_tool_utf8_response(self, mock_model_context, mock_get_provider): """Test that the analyze tool returns correct UTF-8 responses.""" # Mock ModelContext to bypass model validation mock_context_instance = Mock() # Mock token allocation for file processing mock_token_allocation = Mock() mock_token_allocation.file_tokens = 1000 mock_token_allocation.total_tokens = 2000 mock_context_instance.calculate_token_allocation.return_value = mock_token_allocation # Mock provider with more complete setup (same as codereview test) mock_provider = Mock() mock_provider.get_provider_type.return_value = Mock(value="test") mock_provider.get_capabilities.return_value = Mock(supports_extended_thinking=False) mock_provider.generate_content = AsyncMock( return_value=Mock( content=json.dumps( { "status": "analysis_complete", "raw_analysis": "Analysis completed successfully", }, ensure_ascii=False, ), usage={}, model_name="flash", metadata={}, ) ) # Use the same provider for both contexts mock_get_provider.return_value = mock_provider mock_context_instance.provider = mock_provider mock_context_instance.capabilities = Mock(supports_extended_thinking=False) mock_model_context.return_value = mock_context_instance # Test the tool analyze_tool = AnalyzeTool() result = await analyze_tool.execute( { "step": "Analyze system architecture to identify issues", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Starting architectural analysis of Python code", "relevant_files": ["/test/main.py"], "model": "flash", } ) # Checks self.assertIsNotNone(result) self.assertEqual(len(result), 1) # Parse the response - must be valid UTF-8 JSON response_text = result[0].text response_data = json.loads(response_text) # Structure checks self.assertIn("status", response_data) # Check that the French instruction was added # The mock provider's generate_content should be called mock_provider.generate_content.assert_called() # The call was successful, which means our fix worked @patch("tools.shared.base_tool.BaseTool.get_model_provider") async def test_codereview_tool_french_findings(self, mock_get_provider): """Test that the codereview tool produces findings in French.""" # Mock with analysis in French mock_provider = Mock() mock_provider.get_provider_type.return_value = Mock(value="test") mock_provider.get_capabilities.return_value = Mock(supports_extended_thinking=False) mock_provider.generate_content = AsyncMock( return_value=Mock( content=json.dumps( { "status": "analysis_complete", "raw_analysis": """ 🔴 CRITIQUE: Aucun problème critique trouvé. 🟠 ÉLEVÉ: Fichier example.py:42 - Fonction trop complexe → Problème: La fonction process_data() contient trop de responsabilités → Solution: Décomposer en fonctions plus petites et spécialisées 🟡 MOYEN: Gestion d'erreurs insuffisante → Problème: Plusieurs fonctions n'ont pas de gestion d'erreurs appropriée → Solution: Ajouter des try-catch et validation des paramètres ✅ Points positifs: • Code bien commenté et lisible • Nomenclature cohérente • Tests unitaires présents """, }, ensure_ascii=False, ), usage={}, model_name="test-model", metadata={}, ) ) mock_get_provider.return_value = mock_provider # Test the tool codereview_tool = CodeReviewTool() result = await codereview_tool.execute( { "step": "Complete review of Python code", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Code review complete", "relevant_files": ["/test/example.py"], "model": "test-model", } ) # Checks self.assertIsNotNone(result) response_text = result[0].text response_data = json.loads(response_text) # Check UTF-8 characters in analysis if "expert_analysis" in response_data: analysis = response_data["expert_analysis"]["raw_analysis"] # Check for French characters self.assertIn("ÉLEVÉ", analysis) self.assertIn("problème", analysis) self.assertIn("spécialisées", analysis) self.assertIn("appropriée", analysis) self.assertIn("paramètres", analysis) self.assertIn("présents", analysis) # Check for emojis self.assertIn("🔴", analysis) self.assertIn("🟠", analysis) self.assertIn("🟡", analysis) self.assertIn("✅", analysis) @patch("tools.shared.base_tool.BaseTool.get_model_provider") async def test_debug_tool_french_error_analysis(self, mock_get_provider): """Test that the debug tool analyzes errors in French.""" # Mock provider mock_provider = Mock() mock_provider.get_provider_type.return_value = Mock(value="test") mock_provider.get_capabilities.return_value = Mock(supports_extended_thinking=False) mock_provider.generate_content = AsyncMock( return_value=Mock( content=json.dumps( { "status": "pause_for_investigation", "step_number": 1, "total_steps": 2, "next_step_required": True, "findings": ( "Erreur analysée: variable 'données' non définie. " "Cause probable: import manquant." ), "files_checked": ["/src/data_processor.py"], "relevant_files": ["/src/data_processor.py"], "hypothesis": ("Variable 'données' not defined - missing import"), "confidence": "medium", "investigation_status": "in_progress", "error_analysis": ("L'erreur concerne la variable 'données' qui " "n'est pas définie."), }, ensure_ascii=False, ), usage={}, model_name="test-model", metadata={}, ) ) mock_get_provider.return_value = mock_provider # Test the debug tool debug_tool = DebugIssueTool() result = await debug_tool.execute( { "step": "Analyze NameError in data processing file", "step_number": 1, "total_steps": 1, "next_step_required": False, "findings": "Error detected during script execution", "files_checked": ["/src/data_processor.py"], "relevant_files": ["/src/data_processor.py"], "hypothesis": ("Variable 'données' not defined - missing import"), "confidence": "medium", "model": "test-model", } ) # Checks self.assertIsNotNone(result) response_text = result[0].text response_data = json.loads(response_text) # Check response structure self.assertIn("status", response_data) self.assertIn("investigation_status", response_data) # Check that UTF-8 characters are preserved response_str = json.dumps(response_data, ensure_ascii=False) self.assertIn("données", response_str) def test_utf8_emoji_preservation_in_workflow_responses(self): """Test that emojis are preserved in workflow tool responses.""" # Mock workflow response with various emojis test_data = { "status": "analysis_complete", "severity_indicators": { "critical": "🔴", "high": "🟠", "medium": "🟡", "low": "🟢", "success": "✅", "error": "❌", "warning": "⚠️", }, "progress": "Analysis completed 🎉", "recommendations": [ "Optimize performance 🚀", "Improve documentation 📚", "Add unit tests 🧪", ], } # Test JSON encoding with ensure_ascii=False json_str = json.dumps(test_data, ensure_ascii=False, indent=2) # Check emojis are preserved self.assertIn("🔴", json_str) self.assertIn("🟠", json_str) self.assertIn("🟡", json_str) self.assertIn("🟢", json_str) self.assertIn("✅", json_str) self.assertIn("❌", json_str) self.assertIn("⚠️", json_str) self.assertIn("🎉", json_str) self.assertIn("🚀", json_str) self.assertIn("📚", json_str) self.assertIn("🧪", json_str) # No escaped Unicode self.assertNotIn("\\u", json_str) # Test parsing preserves emojis parsed = json.loads(json_str) self.assertEqual(parsed["severity_indicators"]["critical"], "🔴") self.assertEqual(parsed["progress"], "Analysis completed 🎉") if __name__ == "__main__": unittest.main(verbosity=2) ================================================ FILE: tests/test_xai_provider.py ================================================ """Tests for X.AI provider implementation.""" import os from unittest.mock import MagicMock, patch import pytest from providers.shared import ProviderType from providers.xai import XAIModelProvider class TestXAIProvider: """Test X.AI provider functionality.""" def setup_method(self): """Set up clean state before each test.""" # Clear restriction service cache before each test import utils.model_restrictions utils.model_restrictions._restriction_service = None def teardown_method(self): """Clean up after each test to avoid singleton issues.""" # Clear restriction service cache after each test import utils.model_restrictions utils.model_restrictions._restriction_service = None @patch.dict(os.environ, {"XAI_API_KEY": "test-key"}) def test_initialization(self): """Test provider initialization.""" provider = XAIModelProvider("test-key") assert provider.api_key == "test-key" assert provider.get_provider_type() == ProviderType.XAI assert provider.base_url == "https://api.x.ai/v1" def test_initialization_with_custom_url(self): """Test provider initialization with custom base URL.""" provider = XAIModelProvider("test-key", base_url="https://custom.x.ai/v1") assert provider.api_key == "test-key" assert provider.base_url == "https://custom.x.ai/v1" def test_model_validation(self): """Test model name validation.""" provider = XAIModelProvider("test-key") # Test valid models assert provider.validate_model_name("grok-4") is True assert provider.validate_model_name("grok4") is True assert provider.validate_model_name("grok") is True assert provider.validate_model_name("grok-4.1-fast") is True assert provider.validate_model_name("grok-4.1-fast-reasoning") is True assert provider.validate_model_name("grok-4.1-fast-reasoning-latest") is True assert provider.validate_model_name("grok-4.1-fast") is True assert provider.validate_model_name("grok-4.1-fast-reasoning") is True assert provider.validate_model_name("grok-4.1-fast-reasoning-latest") is True # Test invalid model assert provider.validate_model_name("invalid-model") is False assert provider.validate_model_name("gpt-4") is False assert provider.validate_model_name("gemini-pro") is False assert provider.validate_model_name("grok-3") is False assert provider.validate_model_name("grok-3-fast") is False assert provider.validate_model_name("grokfast") is False def test_resolve_model_name(self): """Test model name resolution.""" provider = XAIModelProvider("test-key") # Test shorthand resolution assert provider._resolve_model_name("grok") == "grok-4" assert provider._resolve_model_name("grok4") == "grok-4" assert provider._resolve_model_name("grok-4.1-fast-reasoning") == "grok-4-1-fast-reasoning" assert provider._resolve_model_name("grok-4.1-fast-reasoning-latest") == "grok-4-1-fast-reasoning" # Test full name passthrough assert provider._resolve_model_name("grok-4") == "grok-4" assert provider._resolve_model_name("grok-4.1-fast") == "grok-4-1-fast-reasoning" def test_get_capabilities_grok4(self): """Test getting model capabilities for GROK-4.""" provider = XAIModelProvider("test-key") capabilities = provider.get_capabilities("grok-4") assert capabilities.model_name == "grok-4" assert capabilities.friendly_name == "X.AI (Grok 4)" assert capabilities.context_window == 256_000 assert capabilities.provider == ProviderType.XAI assert capabilities.supports_extended_thinking is True assert capabilities.supports_system_prompts is True assert capabilities.supports_streaming is True assert capabilities.supports_function_calling is True assert capabilities.supports_json_mode is True assert capabilities.supports_images is True # Test temperature range assert capabilities.temperature_constraint.min_temp == 0.0 assert capabilities.temperature_constraint.max_temp == 2.0 assert capabilities.temperature_constraint.default_temp == 0.3 def test_get_capabilities_grok4_1_fast(self): """Test getting model capabilities for GROK-4.1 Fast Reasoning.""" provider = XAIModelProvider("test-key") capabilities = provider.get_capabilities("grok-4.1-fast") assert capabilities.model_name == "grok-4-1-fast-reasoning" assert capabilities.friendly_name == "X.AI (Grok 4.1 Fast Reasoning)" assert capabilities.context_window == 2_000_000 assert capabilities.provider == ProviderType.XAI assert capabilities.supports_extended_thinking is True assert capabilities.supports_function_calling is True assert capabilities.supports_json_mode is True assert capabilities.supports_images is True def test_get_capabilities_with_shorthand(self): """Test getting model capabilities with shorthand.""" provider = XAIModelProvider("test-key") capabilities = provider.get_capabilities("grok") assert capabilities.model_name == "grok-4" # Should resolve to full name assert capabilities.context_window == 256_000 capabilities_fast = provider.get_capabilities("grok-4.1-fast-reasoning") assert capabilities_fast.model_name == "grok-4-1-fast-reasoning" # Should resolve to full name def test_unsupported_model_capabilities(self): """Test error handling for unsupported models.""" provider = XAIModelProvider("test-key") with pytest.raises(ValueError, match="Unsupported model 'invalid-model' for provider xai"): provider.get_capabilities("invalid-model") def test_extended_thinking_flags(self): """X.AI capabilities should expose extended thinking support correctly.""" provider = XAIModelProvider("test-key") thinking_aliases = [ "grok-4", "grok", "grok4", "grok-4.1-fast", "grok-4.1-fast-reasoning", "grok-4.1-fast-reasoning-latest", ] for alias in thinking_aliases: assert provider.get_capabilities(alias).supports_extended_thinking is True def test_provider_type(self): """Test provider type identification.""" provider = XAIModelProvider("test-key") assert provider.get_provider_type() == ProviderType.XAI @patch.dict(os.environ, {"XAI_ALLOWED_MODELS": "grok-4"}) def test_model_restrictions(self): """Test model restrictions functionality.""" # Clear cached restriction service import utils.model_restrictions from providers.registry import ModelProviderRegistry utils.model_restrictions._restriction_service = None ModelProviderRegistry.reset_for_testing() provider = XAIModelProvider("test-key") # grok-4 should be allowed (including alias) assert provider.validate_model_name("grok-4") is True assert provider.validate_model_name("grok") is True # grok-4.1-fast should be blocked by restrictions assert provider.validate_model_name("grok-4.1-fast") is False assert provider.validate_model_name("grok-4.1-fast-reasoning") is False @patch.dict(os.environ, {"XAI_ALLOWED_MODELS": "grok-4.1-fast-reasoning"}) def test_multiple_model_restrictions(self): """Restrictions should allow aliases for Grok 4.1 Fast.""" # Clear cached restriction service import utils.model_restrictions from providers.registry import ModelProviderRegistry utils.model_restrictions._restriction_service = None ModelProviderRegistry.reset_for_testing() provider = XAIModelProvider("test-key") # Alias should be allowed (resolves to grok-4.1-fast) assert provider.validate_model_name("grok-4.1-fast-reasoning") is True # Canonical name is not allowed unless explicitly listed assert provider.validate_model_name("grok-4.1-fast") is False # grok-4 should NOT be allowed assert provider.validate_model_name("grok-4") is False @patch.dict(os.environ, {"XAI_ALLOWED_MODELS": "grok,grok-4,grok-4.1-fast,grok-4-1-fast-reasoning"}) def test_both_shorthand_and_full_name_allowed(self): """Test that aliases and canonical names can be allowed together.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = XAIModelProvider("test-key") # Both shorthand and full name should be allowed when explicitly listed assert provider.validate_model_name("grok") is True # Alias explicitly allowed assert provider.validate_model_name("grok-4") is True # Canonical name explicitly allowed assert provider.validate_model_name("grok-4.1-fast") is True # Alias explicitly allowed assert provider.validate_model_name("grok-4-1-fast-reasoning") is True # Canonical name explicitly allowed @patch.dict(os.environ, {"XAI_ALLOWED_MODELS": ""}) def test_empty_restrictions_allows_all(self): """Test that empty restrictions allow all models.""" # Clear cached restriction service import utils.model_restrictions utils.model_restrictions._restriction_service = None provider = XAIModelProvider("test-key") assert provider.validate_model_name("grok-4") is True assert provider.validate_model_name("grok-4.1-fast") is True assert provider.validate_model_name("grok-4.1-fast-reasoning") is True assert provider.validate_model_name("grok") is True assert provider.validate_model_name("grok4") is True def test_friendly_name(self): """Test friendly name constant.""" provider = XAIModelProvider("test-key") assert provider.FRIENDLY_NAME == "X.AI" capabilities = provider.get_capabilities("grok-4") assert capabilities.friendly_name == "X.AI (Grok 4)" def test_supported_models_structure(self): """Test that MODEL_CAPABILITIES has the correct structure.""" provider = XAIModelProvider("test-key") # Check that all expected base models are present assert "grok-4" in provider.MODEL_CAPABILITIES assert "grok-4-1-fast-reasoning" in provider.MODEL_CAPABILITIES # Check model configs have required fields from providers.shared import ModelCapabilities grok4_config = provider.MODEL_CAPABILITIES["grok-4"] assert isinstance(grok4_config, ModelCapabilities) assert hasattr(grok4_config, "context_window") assert hasattr(grok4_config, "supports_extended_thinking") assert hasattr(grok4_config, "aliases") assert grok4_config.context_window == 256_000 assert grok4_config.supports_extended_thinking is True # Check aliases are correctly structured assert "grok" in grok4_config.aliases assert "grok-4" in grok4_config.aliases assert "grok4" in grok4_config.aliases grok41fast_config = provider.MODEL_CAPABILITIES["grok-4-1-fast-reasoning"] assert grok41fast_config.context_window == 2_000_000 assert grok41fast_config.supports_extended_thinking is True assert "grok-4.1-fast" in grok41fast_config.aliases assert "grok-4.1-fast-reasoning" in grok41fast_config.aliases @patch("providers.openai_compatible.OpenAI") def test_generate_content_resolves_alias_before_api_call(self, mock_openai_class): """Test that generate_content resolves aliases before making API calls. This is the CRITICAL test that ensures aliases like 'grok' get resolved to 'grok-4' before being sent to X.AI API. """ # Set up mock OpenAI client mock_client = MagicMock() mock_openai_class.return_value = mock_client # Mock the completion response mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.model = "grok-4" # API returns the resolved model name mock_response.id = "test-id" mock_response.created = 1234567890 mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response provider = XAIModelProvider("test-key") # Call generate_content with alias 'grok' result = provider.generate_content( prompt="Test prompt", model_name="grok", temperature=0.7 # This should be resolved to "grok-4" ) # Verify the API was called with the RESOLVED model name mock_client.chat.completions.create.assert_called_once() call_kwargs = mock_client.chat.completions.create.call_args[1] # CRITICAL ASSERTION: The API should receive "grok-4", not "grok" assert call_kwargs["model"] == "grok-4", f"Expected 'grok-4' but API received '{call_kwargs['model']}'" # Verify other parameters assert call_kwargs["temperature"] == 0.7 assert len(call_kwargs["messages"]) == 1 assert call_kwargs["messages"][0]["role"] == "user" assert call_kwargs["messages"][0]["content"] == "Test prompt" # Verify response assert result.content == "Test response" assert result.model_name == "grok-4" # Should be the resolved name @patch("providers.openai_compatible.OpenAI") def test_generate_content_other_aliases(self, mock_openai_class): """Test other alias resolutions in generate_content.""" from unittest.mock import MagicMock # Set up mock mock_client = MagicMock() mock_openai_class.return_value = mock_client mock_response = MagicMock() mock_response.choices = [MagicMock()] mock_response.choices[0].message.content = "Test response" mock_response.choices[0].finish_reason = "stop" mock_response.usage = MagicMock() mock_response.usage.prompt_tokens = 10 mock_response.usage.completion_tokens = 5 mock_response.usage.total_tokens = 15 mock_client.chat.completions.create.return_value = mock_response provider = XAIModelProvider("test-key") # Test grok4 -> grok-4 mock_response.model = "grok-4" provider.generate_content(prompt="Test", model_name="grok4", temperature=0.7) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "grok-4" # Test grok-4 -> grok-4 provider.generate_content(prompt="Test", model_name="grok-4", temperature=0.7) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "grok-4" # Test grok-4.1-fast-reasoning -> grok-4-1-fast-reasoning mock_response.model = "grok-4-1-fast-reasoning" provider.generate_content(prompt="Test", model_name="grok-4.1-fast-reasoning", temperature=0.7) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "grok-4-1-fast-reasoning" # Test grok-4.1-fast -> grok-4-1-fast-reasoning provider.generate_content(prompt="Test", model_name="grok-4.1-fast", temperature=0.7) call_kwargs = mock_client.chat.completions.create.call_args[1] assert call_kwargs["model"] == "grok-4-1-fast-reasoning" ================================================ FILE: tests/transport_helpers.py ================================================ """Helper functions for HTTP transport injection in tests.""" from tests.http_transport_recorder import TransportFactory def inject_transport(monkeypatch, cassette_path: str): """Inject HTTP transport into OpenAICompatibleProvider for testing. This helper simplifies the monkey patching pattern used across tests to inject custom HTTP transports for recording/replaying API calls. Also ensures OpenAI provider is properly registered for tests that need it. Args: monkeypatch: pytest monkeypatch fixture cassette_path: Path to cassette file for recording/replay Returns: The created transport instance Example: transport = inject_transport(monkeypatch, "path/to/cassette.json") """ # Ensure OpenAI provider is registered - always needed for transport injection from providers.openai import OpenAIModelProvider from providers.registry import ModelProviderRegistry from providers.shared import ProviderType # Always register OpenAI provider for transport tests (API key might be dummy) ModelProviderRegistry.register_provider(ProviderType.OPENAI, OpenAIModelProvider) # Create transport transport = TransportFactory.create_transport(str(cassette_path)) # Inject transport using the established pattern from providers.openai_compatible import OpenAICompatibleProvider original_client_property = OpenAICompatibleProvider.client def patched_client_getter(self): if self._client is None: self._test_transport = transport return original_client_property.fget(self) monkeypatch.setattr(OpenAICompatibleProvider, "client", property(patched_client_getter)) return transport ================================================ FILE: tools/__init__.py ================================================ """ Tool implementations for PAL MCP Server """ from .analyze import AnalyzeTool from .apilookup import LookupTool from .challenge import ChallengeTool from .chat import ChatTool from .clink import CLinkTool from .codereview import CodeReviewTool from .consensus import ConsensusTool from .debug import DebugIssueTool from .docgen import DocgenTool from .listmodels import ListModelsTool from .planner import PlannerTool from .precommit import PrecommitTool from .refactor import RefactorTool from .secaudit import SecauditTool from .testgen import TestGenTool from .thinkdeep import ThinkDeepTool from .tracer import TracerTool from .version import VersionTool __all__ = [ "ThinkDeepTool", "CodeReviewTool", "DebugIssueTool", "DocgenTool", "AnalyzeTool", "LookupTool", "ChatTool", "CLinkTool", "ConsensusTool", "ListModelsTool", "PlannerTool", "PrecommitTool", "ChallengeTool", "RefactorTool", "SecauditTool", "TestGenTool", "TracerTool", "VersionTool", ] ================================================ FILE: tools/analyze.py ================================================ """ AnalyzeWorkflow tool - Step-by-step code analysis with systematic investigation This tool provides a structured workflow for comprehensive code and file analysis. It guides the CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough code examination, pattern identification, and architectural assessment before proceeding. The tool supports complex analysis scenarios including architectural review, performance analysis, security assessment, and maintainability evaluation. Key features: - Step-by-step analysis workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic pattern and insight tracking with categorization - Expert analysis integration with external models - Support for focused analysis (architecture, performance, security, quality) - Confidence-based workflow optimization """ import logging from typing import TYPE_CHECKING, Any, Literal, Optional from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import ANALYZE_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for analyze workflow ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "The analysis plan. Step 1: State your strategy, including how you will map the codebase structure, " "understand business logic, and assess code quality, performance implications, and architectural patterns. " "Later steps: Report findings and adapt the approach as new insights emerge." ), "step_number": ( "The index of the current step in the analysis sequence, beginning at 1. Each step should build upon or " "revise the previous one." ), "total_steps": ( "Your current estimate for how many steps will be needed to complete the analysis. " "Adjust as new findings emerge." ), "next_step_required": ( "Set to true if you plan to continue the investigation with another step. False means you believe the " "analysis is complete and ready for expert validation." ), "findings": ( "Summary of discoveries from this step, including architectural patterns, tech stack assessment, scalability characteristics, " "performance implications, maintainability factors, and strategic improvement opportunities. " "IMPORTANT: Document both strengths (good patterns, solid architecture) and concerns (tech debt, overengineering, unnecessary complexity). " "In later steps, confirm or update past findings with additional evidence." ), "files_checked": ( "List all files examined (absolute paths). Include even ruled-out files to track exploration path." ), "relevant_files": ( "Subset of files_checked directly relevant to analysis findings (absolute paths). Include files with " "significant patterns, architectural decisions, or strategic improvement opportunities." ), "relevant_context": ( "List methods/functions central to analysis findings, in 'ClassName.methodName' or 'functionName' format. " "Prioritize those demonstrating key patterns, architectural decisions, or improvement opportunities." ), "images": ( "Optional absolute paths to architecture diagrams or visual references that help with analysis context." ), "confidence": ( "Your confidence in the analysis: exploring, low, medium, high, very_high, almost_certain, or certain. " "'certain' indicates the analysis is complete and ready for validation." ), "analysis_type": "Type of analysis to perform (architecture, performance, security, quality, general)", "output_format": "How to format the output (summary, detailed, actionable)", } class AnalyzeWorkflowRequest(WorkflowRequest): """Request model for analyze workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) # Issues found during analysis (structured with severity) issues_found: list[dict] = Field( default_factory=list, description="Issues or concerns identified during analysis, each with severity level (critical, high, medium, low)", ) # Optional images for visual context images: Optional[list[str]] = Field(default=None, description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Analyze-specific fields (only used in step 1 to initialize) # Note: Use relevant_files field instead of files for consistency across workflow tools analysis_type: Optional[Literal["architecture", "performance", "security", "quality", "general"]] = Field( "general", description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["analysis_type"] ) output_format: Optional[Literal["summary", "detailed", "actionable"]] = Field( "detailed", description=ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["output_format"] ) # Keep thinking_mode from original analyze tool; temperature is inherited from WorkflowRequest @model_validator(mode="after") def validate_step_one_requirements(self): """Ensure step 1 has required relevant_files.""" if self.step_number == 1: if not self.relevant_files: raise ValueError("Step 1 requires 'relevant_files' field to specify files or directories to analyze") return self class AnalyzeTool(WorkflowTool): """ Analyze workflow tool for step-by-step code analysis and expert validation. This tool implements a structured analysis workflow that guides users through methodical investigation steps, ensuring thorough code examination, pattern identification, and architectural assessment before reaching conclusions. It supports complex analysis scenarios including architectural review, performance analysis, security assessment, and maintainability evaluation. """ def __init__(self): super().__init__() self.initial_request = None self.analysis_config = {} def get_name(self) -> str: return "analyze" def get_description(self) -> str: return ( "Performs comprehensive code analysis with systematic investigation and expert validation. " "Use for architecture, performance, maintainability, and pattern analysis. " "Guides through structured code review and strategic planning." ) def get_system_prompt(self) -> str: return ANALYZE_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Analyze workflow requires thorough analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the analyze workflow-specific request model.""" return AnalyzeWorkflowRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with analyze-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Fields to exclude from analyze workflow (inherited from WorkflowRequest but not used) excluded_fields = {"hypothesis", "confidence"} # Analyze workflow-specific field overrides analyze_field_overrides = { "step": { "type": "string", "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, "confidence": { "type": "string", "enum": ["exploring", "low", "medium", "high", "very_high", "almost_certain", "certain"], "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["confidence"], }, "images": { "type": "array", "items": {"type": "string"}, "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, "issues_found": { "type": "array", "items": {"type": "object"}, "description": "Issues or concerns identified during analysis, each with severity level (critical, high, medium, low)", }, "analysis_type": { "type": "string", "enum": ["architecture", "performance", "security", "quality", "general"], "default": "general", "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["analysis_type"], }, "output_format": { "type": "string", "enum": ["summary", "detailed", "actionable"], "default": "detailed", "description": ANALYZE_WORKFLOW_FIELD_DESCRIPTIONS["output_format"], }, } # Use WorkflowSchemaBuilder with analyze-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=analyze_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), excluded_workflow_fields=list(excluded_fields), ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each investigation phase.""" if step_number == 1: # Initial analysis investigation tasks return [ "Read and understand the code files specified for analysis", "Map the tech stack, frameworks, and overall architecture", "Identify the main components, modules, and their relationships", "Understand the business logic and intended functionality", "Examine architectural patterns and design decisions used", "Look for strengths, risks, and strategic improvement areas", ] elif step_number < total_steps: # Need deeper investigation return [ "Examine specific architectural patterns and design decisions in detail", "Analyze scalability characteristics and performance implications", "Assess maintainability factors: module cohesion, coupling, tech debt", "Identify security posture and potential systemic vulnerabilities", "Look for overengineering, unnecessary complexity, or missing abstractions", "Evaluate how well the architecture serves business and scaling goals", ] else: # Close to completion - need final verification return [ "Verify all significant architectural insights have been documented", "Confirm strategic improvement opportunities are comprehensively captured", "Ensure both strengths and risks are properly identified with evidence", "Validate that findings align with the analysis type and goals specified", "Check that recommendations are actionable and proportional to the codebase", "Confirm the analysis provides clear guidance for strategic decisions", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Always call expert analysis for comprehensive validation. Analysis benefits from a second opinion to ensure completeness. """ # Check if user explicitly requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # For analysis, we always want expert validation if we have any meaningful data return len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 1 def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call for final analysis validation.""" context_parts = [ f"=== ANALYSIS REQUEST ===\\n{self.initial_request or 'Code analysis workflow initiated'}\\n=== END REQUEST ===" ] # Add investigation summary investigation_summary = self._build_analysis_summary(consolidated_findings) context_parts.append( f"\\n=== AGENT'S ANALYSIS INVESTIGATION ===\\n{investigation_summary}\\n=== END INVESTIGATION ===" ) # Add analysis configuration context if available if self.analysis_config: config_text = "\\n".join(f"- {key}: {value}" for key, value in self.analysis_config.items() if value) context_parts.append(f"\\n=== ANALYSIS CONFIGURATION ===\\n{config_text}\\n=== END CONFIGURATION ===") # Add relevant code elements if available if consolidated_findings.relevant_context: methods_text = "\\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\\n=== RELEVANT CODE ELEMENTS ===\\n{methods_text}\\n=== END CODE ELEMENTS ===") # Add assessment evolution if available if consolidated_findings.hypotheses: assessments_text = "\\n".join( f"Step {h['step']}: {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\\n=== ASSESSMENT EVOLUTION ===\\n{assessments_text}\\n=== END ASSESSMENTS ===") # Add images if available if consolidated_findings.images: images_text = "\\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append( f"\\n=== VISUAL ANALYSIS INFORMATION ===\\n{images_text}\\n=== END VISUAL INFORMATION ===" ) return "\\n".join(context_parts) def _build_analysis_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the analysis investigation.""" summary_parts = [ "=== SYSTEMATIC ANALYSIS INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Code elements analyzed: {len(consolidated_findings.relevant_context)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\\n".join(summary_parts) def should_include_files_in_expert_prompt(self) -> bool: """Include files in expert analysis for comprehensive validation.""" return True def should_embed_system_prompt(self) -> bool: """Embed system prompt in expert analysis for proper context.""" return True def get_expert_thinking_mode(self) -> str: """Use high thinking mode for thorough analysis.""" return "high" def get_expert_analysis_instruction(self) -> str: """Get specific instruction for analysis expert validation.""" return ( "Please provide comprehensive analysis validation based on the investigation findings. " "Focus on identifying any remaining architectural insights, validating the completeness of the analysis, " "and providing final strategic recommendations following the structured format specified in the system prompt." ) # Hook method overrides for analyze-specific behavior def prepare_step_data(self, request) -> dict: """ Map analyze-specific fields for internal processing. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": request.issues_found, # Analyze workflow uses issues_found for structured problem tracking "confidence": "medium", # Fixed value for workflow compatibility "hypothesis": request.findings, # Map findings to hypothesis for compatibility "images": request.images or [], } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Analyze workflow always uses expert analysis for comprehensive validation. Analysis benefits from a second opinion to ensure completeness and catch any missed insights or alternative perspectives. """ return False def store_initial_issue(self, step_description: str): """Store initial request for expert analysis.""" self.initial_request = step_description # Override inheritance hooks for analyze-specific behavior def get_completion_status(self) -> str: """Analyze tools use analysis-specific status.""" return "analysis_complete_ready_for_implementation" def get_completion_data_key(self) -> str: """Analyze uses 'complete_analysis' key.""" return "complete_analysis" def get_final_analysis_from_request(self, request): """Analyze tools use 'findings' field.""" return request.findings def get_confidence_level(self, request) -> str: """Analyze tools use fixed confidence for consistency.""" return "medium" def get_completion_message(self) -> str: """Analyze-specific completion message.""" return ( "Analysis complete. You have identified all significant patterns, " "architectural insights, and strategic opportunities. MANDATORY: Present the user with the complete " "analysis results organized by strategic impact, and IMMEDIATELY proceed with implementing the " "highest priority recommendations or provide specific guidance for improvements. Focus on actionable " "strategic insights." ) def get_skip_reason(self) -> str: """Analyze-specific skip reason.""" return "Completed comprehensive analysis locally" def get_skip_expert_analysis_status(self) -> str: """Analyze-specific expert analysis skip status.""" return "skipped_due_to_complete_analysis" def prepare_work_summary(self) -> str: """Analyze-specific work summary.""" return self._build_analysis_summary(self.consolidated_findings) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Analyze-specific completion message. """ base_message = ( "ANALYSIS IS COMPLETE. You MUST now summarize and present ALL analysis findings organized by " "strategic impact (Critical → High → Medium → Low), specific architectural insights with code references, " "and exact recommendations for improvement. Clearly prioritize the top 3 strategic opportunities that need " "immediate attention. Provide concrete, actionable guidance for each finding—make it easy for a developer " "to understand exactly what strategic improvements to implement and how to approach them." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Provide specific guidance for handling expert analysis in code analysis. """ return ( "IMPORTANT: Analysis from an assistant model has been provided above. You MUST thoughtfully evaluate and validate " "the expert insights rather than treating them as definitive conclusions. Cross-reference the expert " "analysis with your own systematic investigation, verify that architectural recommendations are " "appropriate for this codebase's scale and context, and ensure suggested improvements align with " "the project's goals and constraints. Present a comprehensive synthesis that combines your detailed " "analysis with validated expert perspectives, clearly distinguishing between patterns you've " "independently identified and additional strategic insights from expert validation." ) def get_step_guidance_message(self, request) -> str: """ Analyze-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_analyze_step_guidance(request.step_number, request) return step_guidance["next_steps"] def get_analyze_step_guidance(self, step_number: int, request) -> dict[str, Any]: """ Provide step-specific guidance for analyze workflow. """ # Generate the next steps instruction based on required actions required_actions = self.get_required_actions(step_number, "medium", request.findings, request.total_steps) if step_number == 1: next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first examine " f"the code files thoroughly using appropriate tools. CRITICAL AWARENESS: You need to understand " f"the architectural patterns, assess scalability and performance characteristics, identify strategic " f"improvement areas, and look for systemic risks, overengineering, and missing abstractions. " f"Use file reading tools, code analysis, and systematic examination to gather comprehensive information. " f"Only call {self.get_name()} again AFTER completing your investigation. When you call " f"{self.get_name()} next time, use step_number: {step_number + 1} and report specific " f"files examined, architectural insights found, and strategic assessment discoveries." ) elif step_number < request.total_steps: next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. Based on your findings, you've identified areas that need " f"deeper analysis. MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nOnly call {self.get_name()} again with step_number: {step_number + 1} AFTER " + "completing these analysis tasks." ) else: next_steps = ( f"WAIT! Your analysis needs final verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nREMEMBER: Ensure you have identified all significant architectural insights and strategic " f"opportunities across all areas. Document findings with specific file references and " f"code examples where applicable, then call {self.get_name()} with step_number: {step_number + 1}." ) return {"next_steps": next_steps} def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match analyze workflow format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Store analysis configuration for expert analysis if request.relevant_files: self.analysis_config = { "relevant_files": request.relevant_files, "analysis_type": request.analysis_type, "output_format": request.output_format, } # Convert generic status names to analyze-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "analysis_in_progress", f"pause_for_{tool_name}": "pause_for_analysis", f"{tool_name}_required": "analysis_required", f"{tool_name}_complete": "analysis_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match analyze workflow if f"{tool_name}_status" in response_data: response_data["analysis_status"] = response_data.pop(f"{tool_name}_status") # Add analyze-specific status fields response_data["analysis_status"]["insights_by_severity"] = {} for insight in self.consolidated_findings.issues_found: severity = insight.get("severity", "unknown") if severity not in response_data["analysis_status"]["insights_by_severity"]: response_data["analysis_status"]["insights_by_severity"][severity] = 0 response_data["analysis_status"]["insights_by_severity"][severity] += 1 response_data["analysis_status"]["analysis_confidence"] = self.get_request_confidence(request) # Map complete_analyze to complete_analysis if f"complete_{tool_name}" in response_data: response_data["complete_analysis"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match analyze workflow if f"{tool_name}_complete" in response_data: response_data["analysis_complete"] = response_data.pop(f"{tool_name}_complete") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the analyze workflow-specific request model.""" return AnalyzeWorkflowRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/apilookup.py ================================================ """API lookup tool - quickly gather the latest API/SDK information.""" from __future__ import annotations import json from typing import TYPE_CHECKING, Any from pydantic import Field from config import TEMPERATURE_ANALYTICAL from tools.shared.base_models import ToolRequest from tools.simple.base import SimpleTool if TYPE_CHECKING: from tools.models import ToolModelCategory LOOKUP_FIELD_DESCRIPTIONS = { "prompt": "The API, SDK, library, framework, or technology you need current documentation, version info, breaking changes, or migration guidance for.", } class LookupRequest(ToolRequest): prompt: str = Field(..., description=LOOKUP_FIELD_DESCRIPTIONS["prompt"]) LOOKUP_PROMPT = """ MANDATORY: You MUST perform this research in a SEPARATE SUB-TASK using your web search tool. CRITICAL RULES - READ CAREFULLY: - Launch your environment's dedicated web search capability (for example `websearch`, `web_search`, or another native web-search tool such as the one you use to perform a web search online) to gather sources - do NOT call this `apilookup` tool again during the same lookup, this is ONLY an orchestration tool to guide you and has NO web search capability of its own. - ALWAYS run the search from a separate sub-task/sub-process so the research happens outside this tool invocation. - If the environment does not expose a web search tool, immediately report that limitation instead of invoking `apilookup` again. MISSION: Research the latest, most authoritative documentation for the requested API, SDK, library, framework, programming language feature, or tool to answer the user's question accurately using a SUB-AGENT in a separate process. SEARCH STRATEGY (MAXIMUM 2-4 SEARCHES TOTAL FOR THIS MISSION - THEN STOP): - IMPORTANT: Begin by determining today's date and current year - MANDATORY FOR OS-TIED APIS/SDKs: If the request involves iOS, macOS, Windows, Linux, Android, watchOS, tvOS, or any OS-specific framework/API: * FIRST perform a web search to determine "what is the latest [OS name] version [current year]" * If the search is around a specific tool or an IDE, confirm the latest version "latest version [tool name]" * DO NOT rely on your training data or knowledge cutoff for OS versions - you MUST search for current information * ONLY AFTER confirming the current OS version, search for APIs/SDKs/frameworks for that specific version * Example workflow: Search "latest iOS version [current year]" → Find current version → Then search "[current iOS version] SwiftUI glass effect button [current year]" - MANDATORY FOR MAJOR FRAMEWORKS/LANGUAGES: For rapidly-evolving ecosystems, verify current stable version: * Languages: Node.js, Python, Ruby, Rust, Go, Java, .NET/C#, PHP, Kotlin, Swift * Web frameworks: React, Vue, Angular, Next.js, Nuxt, Svelte, SvelteKit, Remix, Astro, SolidJS * Backend frameworks: Django, Flask, FastAPI, Rails, Laravel, Spring Boot, Express, NestJS, Axum * Mobile: Flutter, React Native, Jetpack Compose, SwiftUI * Build tools: Vite, Webpack, esbuild, Turbopack, Rollup * Package managers: npm, pnpm, yarn, pip, cargo, go modules, maven, gradle * Search pattern: "latest [framework/language/SDK] version [current year]" BEFORE searching for specific APIs * ONLY consider articles, documentation, and resources dated within the current year or most recent release cycle * Ignore or deprioritize results from previous years unless they are still the current official documentation - ALWAYS find current official documentation, release notes, changelogs, migration guides, and authoritative blog posts. Newest APIs / SDKs released or updated in the current year trump older ones. - Prioritize official sources: project documentation sites, GitHub repositories, package registries (npm, PyPI, crates.io, Maven Central, NuGet, RubyGems, Packagist, etc.), and official blogs - Check version-specific documentation when relevant and add current year to ensure latest docs are retrieved (e.g., "React docs [current year]", "Python what's new [current year]", "TypeScript breaking changes [current year]", "Next.js app router [current year]") - Look for recent Stack Overflow discussions, GitHub issues, RFC documents, or official discussion forums when official docs are incomplete - Cross-reference multiple sources to validate syntax, method signatures, configuration options, and best practices - Search for deprecation warnings, security advisories, or migration paths between major versions - STOP IMMEDIATELY after 2-4 searches maximum - DO NOT continue exploring tangential topics, examples, tutorials, or supplementary material - If latest, more current, authoritative information has been found: STOP looking further - ALWAYS cite authoritative sources with links (official docs, changelogs, GitHub releases, package registry pages) """.strip() class LookupTool(SimpleTool): """Simple tool that wraps user queries with API lookup instructions.""" def get_name(self) -> str: return "apilookup" def get_description(self) -> str: return ( "Use this tool automatically when you need current API/SDK documentation, latest version info, breaking changes, deprecations, migration guides, or official release notes. " "This tool searches authoritative sources (official docs, GitHub, package registries) to ensure up-to-date accuracy." ) def get_system_prompt(self) -> str: return "" def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def requires_model(self) -> bool: return False def get_model_category(self) -> ToolModelCategory: from tools.models import ToolModelCategory return ToolModelCategory.FAST_RESPONSE def get_request_model(self): return LookupRequest def get_tool_fields(self) -> dict[str, dict[str, Any]]: return { "prompt": { "type": "string", "description": LOOKUP_FIELD_DESCRIPTIONS["prompt"], } } async def prepare_prompt(self, request) -> str: # pragma: no cover - not used return "" def get_input_schema(self) -> dict[str, Any]: return { "type": "object", "properties": { "prompt": { "type": "string", "description": LOOKUP_FIELD_DESCRIPTIONS["prompt"], }, }, "required": ["prompt"], } async def execute(self, arguments: dict[str, Any]) -> list: from mcp.types import TextContent request = self.get_request_model()(**arguments) response = { "status": "web_lookup_needed", "instructions": LOOKUP_PROMPT, "user_prompt": request.prompt, } return [TextContent(type="text", text=json.dumps(response, ensure_ascii=False, indent=2))] ================================================ FILE: tools/challenge.py ================================================ """ Challenge tool - Encourages critical thinking and thoughtful disagreement This tool takes a user's statement and returns it wrapped in instructions that encourage the CLI agent to challenge ideas and think critically before agreeing. It helps avoid reflexive agreement by prompting deeper analysis and genuine evaluation. This is a simple, self-contained tool that doesn't require AI model access. """ from typing import TYPE_CHECKING, Any, Optional from pydantic import Field if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from tools.shared.base_models import ToolRequest from tools.shared.exceptions import ToolExecutionError from .simple.base import SimpleTool # Field descriptions for the Challenge tool CHALLENGE_FIELD_DESCRIPTIONS = { "prompt": ( "Statement to scrutinize. If you invoke `challenge` manually, strip the word 'challenge' and pass just the statement. " "Automatic invocations send the full user message as-is; do not modify it." ), } class ChallengeRequest(ToolRequest): """Request model for Challenge tool""" prompt: str = Field(..., description=CHALLENGE_FIELD_DESCRIPTIONS["prompt"]) class ChallengeTool(SimpleTool): """ Challenge tool for encouraging critical thinking and avoiding automatic agreement. This tool wraps user statements in instructions that encourage the CLI agent to: - Challenge ideas and think critically before responding - Evaluate whether they actually agree or disagree - Provide thoughtful analysis rather than reflexive agreement The tool is self-contained and doesn't require AI model access - it simply transforms the input prompt into a structured critical thinking challenge. """ def get_name(self) -> str: return "challenge" def get_description(self) -> str: return ( "Prevents reflexive agreement by forcing critical thinking and reasoned analysis when a statement is challenged. " "Trigger automatically when a user critically questions, disagrees or appears to push back on earlier answers, and use it manually to sanity-check contentious claims." ) def get_system_prompt(self) -> str: # Challenge tool doesn't need a system prompt since it doesn't call AI return "" def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Challenge doesn't need a model category since it doesn't use AI""" from tools.models import ToolModelCategory return ToolModelCategory.FAST_RESPONSE # Default, but not used def requires_model(self) -> bool: """ Challenge tool doesn't require model resolution at the MCP boundary. Like the planner tool, this is a pure data processing tool that transforms the input without calling external AI models. Returns: bool: False - challenge doesn't need AI model access """ return False def get_request_model(self): """Return the Challenge-specific request model""" return ChallengeRequest def get_input_schema(self) -> dict[str, Any]: """ Generate input schema for the challenge tool. Since this tool doesn't require a model, we exclude model-related fields. """ schema = { "type": "object", "properties": { "prompt": { "type": "string", "description": CHALLENGE_FIELD_DESCRIPTIONS["prompt"], }, }, "required": ["prompt"], } return schema async def execute(self, arguments: dict[str, Any]) -> list: """ Execute the challenge tool by wrapping the prompt in critical thinking instructions. This is the main execution method that transforms the user's statement into a structured challenge that encourages thoughtful re-evaluation. """ import json from mcp.types import TextContent try: # Validate request request = self.get_request_model()(**arguments) # Wrap the prompt in challenge instructions wrapped_prompt = self._wrap_prompt_for_challenge(request.prompt) # Return the wrapped prompt as the response response_data = { "status": "challenge_accepted", "original_statement": request.prompt, "challenge_prompt": wrapped_prompt, "instructions": ( "Present the challenge_prompt to yourself and follow its instructions. " "Reassess the statement carefully and critically before responding. " "If, after reflection, you find reasons to disagree or qualify it, explain your reasoning. " "Likewise, if you find reasons to agree, articulate them clearly and justify your agreement." ), } return [TextContent(type="text", text=json.dumps(response_data, indent=2, ensure_ascii=False))] except ToolExecutionError: raise except Exception as e: import logging logger = logging.getLogger(__name__) logger.error(f"Error in challenge tool execution: {e}", exc_info=True) error_data = { "status": "error", "error": str(e), "content": f"Failed to create challenge prompt: {str(e)}", } raise ToolExecutionError(json.dumps(error_data, ensure_ascii=False)) from e def _wrap_prompt_for_challenge(self, prompt: str) -> str: """ Wrap the user's statement in instructions that encourage critical challenge. Args: prompt: The original user statement to wrap Returns: The statement wrapped in challenge instructions """ return ( f"CRITICAL REASSESSMENT – Do not automatically agree:\n\n" f'"{prompt}"\n\n' f"Carefully evaluate the statement above. Is it accurate, complete, and well-reasoned? " f"Investigate if needed before replying, and stay focused. If you identify flaws, gaps, or misleading " f"points, explain them clearly. Likewise, if you find the reasoning sound, explain why it holds up. " f"Respond with thoughtful analysis—stay to the point and avoid reflexive agreement." ) # Required method implementations from SimpleTool async def prepare_prompt(self, request: ChallengeRequest) -> str: """Not used since challenge doesn't call AI models""" return "" def format_response(self, response: str, request: ChallengeRequest, model_info: Optional[dict] = None) -> str: """Not used since challenge doesn't call AI models""" return response def get_tool_fields(self) -> dict[str, dict[str, Any]]: """Tool-specific field definitions for Challenge""" return { "prompt": { "type": "string", "description": CHALLENGE_FIELD_DESCRIPTIONS["prompt"], }, } def get_required_fields(self) -> list[str]: """Required fields for Challenge tool""" return ["prompt"] ================================================ FILE: tools/chat.py ================================================ """ Chat tool - General development chat and collaborative thinking This tool provides a conversational interface for general development assistance, brainstorming, problem-solving, and collaborative thinking. It supports file context, images, and conversation continuation for seamless multi-turn interactions. """ import logging import os import re from pathlib import Path from typing import TYPE_CHECKING, Any, Optional from pydantic import Field if TYPE_CHECKING: from providers.shared import ModelCapabilities from tools.models import ToolModelCategory from config import TEMPERATURE_BALANCED from systemprompts import CHAT_PROMPT, GENERATE_CODE_PROMPT from tools.shared.base_models import COMMON_FIELD_DESCRIPTIONS, ToolRequest from .simple.base import SimpleTool # Field descriptions matching the original Chat tool exactly CHAT_FIELD_DESCRIPTIONS = { "prompt": ( "Your question or idea for collaborative thinking to be sent to the external model. Provide detailed context, " "including your goal, what you've tried, and any specific challenges. " "WARNING: Large inline code must NOT be shared in prompt. Provide full-path to files on disk as separate parameter." ), "absolute_file_paths": ("Full, absolute file paths to relevant code in order to share with external model"), "images": "Image paths (absolute) or base64 strings for optional visual context.", "working_directory_absolute_path": ( "Absolute path to an existing directory where generated code artifacts can be saved." ), } class ChatRequest(ToolRequest): """Request model for Chat tool""" prompt: str = Field(..., description=CHAT_FIELD_DESCRIPTIONS["prompt"]) absolute_file_paths: Optional[list[str]] = Field( default_factory=list, description=CHAT_FIELD_DESCRIPTIONS["absolute_file_paths"], ) images: Optional[list[str]] = Field(default_factory=list, description=CHAT_FIELD_DESCRIPTIONS["images"]) working_directory_absolute_path: str = Field( ..., description=CHAT_FIELD_DESCRIPTIONS["working_directory_absolute_path"], ) class ChatTool(SimpleTool): """ General development chat and collaborative thinking tool using SimpleTool architecture. This tool provides identical functionality to the original Chat tool but uses the new SimpleTool architecture for cleaner code organization and better maintainability. Migration note: This tool is designed to be a drop-in replacement for the original Chat tool with 100% behavioral compatibility. """ def __init__(self) -> None: super().__init__() self._last_recordable_response: Optional[str] = None def get_name(self) -> str: return "chat" def get_description(self) -> str: return ( "General chat and collaborative thinking partner for brainstorming, development discussion, " "getting second opinions, and exploring ideas. Use for ideas, validations, questions, and thoughtful explanations." ) def get_annotations(self) -> Optional[dict[str, Any]]: """Chat writes generated artifacts when code-generation is enabled.""" return {"readOnlyHint": False} def get_system_prompt(self) -> str: return CHAT_PROMPT def get_capability_system_prompts(self, capabilities: Optional["ModelCapabilities"]) -> list[str]: prompts = list(super().get_capability_system_prompts(capabilities)) if capabilities and capabilities.allow_code_generation: prompts.append(GENERATE_CODE_PROMPT) return prompts def get_default_temperature(self) -> float: return TEMPERATURE_BALANCED def get_model_category(self) -> "ToolModelCategory": """Chat prioritizes fast responses and cost efficiency""" from tools.models import ToolModelCategory return ToolModelCategory.FAST_RESPONSE def get_request_model(self): """Return the Chat-specific request model""" return ChatRequest # === Schema Generation Utilities === def get_input_schema(self) -> dict[str, Any]: """Generate input schema matching the original Chat tool expectations.""" required_fields = ["prompt", "working_directory_absolute_path"] if self.is_effective_auto_mode(): required_fields.append("model") schema = { "type": "object", "properties": { "prompt": { "type": "string", "description": CHAT_FIELD_DESCRIPTIONS["prompt"], }, "absolute_file_paths": { "type": "array", "items": {"type": "string"}, "description": CHAT_FIELD_DESCRIPTIONS["absolute_file_paths"], }, "images": { "type": "array", "items": {"type": "string"}, "description": CHAT_FIELD_DESCRIPTIONS["images"], }, "working_directory_absolute_path": { "type": "string", "description": CHAT_FIELD_DESCRIPTIONS["working_directory_absolute_path"], }, "model": self.get_model_field_schema(), "temperature": { "type": "number", "description": COMMON_FIELD_DESCRIPTIONS["temperature"], "minimum": 0, "maximum": 1, }, "thinking_mode": { "type": "string", "enum": ["minimal", "low", "medium", "high", "max"], "description": COMMON_FIELD_DESCRIPTIONS["thinking_mode"], }, "continuation_id": { "type": "string", "description": COMMON_FIELD_DESCRIPTIONS["continuation_id"], }, }, "required": required_fields, "additionalProperties": False, } return schema def get_tool_fields(self) -> dict[str, dict[str, Any]]: """Tool-specific field definitions used by SimpleTool scaffolding.""" return { "prompt": { "type": "string", "description": CHAT_FIELD_DESCRIPTIONS["prompt"], }, "absolute_file_paths": { "type": "array", "items": {"type": "string"}, "description": CHAT_FIELD_DESCRIPTIONS["absolute_file_paths"], }, "images": { "type": "array", "items": {"type": "string"}, "description": CHAT_FIELD_DESCRIPTIONS["images"], }, "working_directory_absolute_path": { "type": "string", "description": CHAT_FIELD_DESCRIPTIONS["working_directory_absolute_path"], }, } def get_required_fields(self) -> list[str]: """Required fields for ChatSimple tool""" return ["prompt", "working_directory_absolute_path"] # === Hook Method Implementations === async def prepare_prompt(self, request: ChatRequest) -> str: """ Prepare the chat prompt with optional context files. This implementation matches the original Chat tool exactly while using SimpleTool convenience methods for cleaner code. """ # Use SimpleTool's Chat-style prompt preparation return self.prepare_chat_style_prompt(request) def _validate_file_paths(self, request) -> Optional[str]: """Extend validation to cover the working directory path.""" files = self.get_request_files(request) if files: expanded_files: list[str] = [] for file_path in files: expanded = os.path.expanduser(file_path) if not os.path.isabs(expanded): return ( "Error: All file paths must be FULL absolute paths to real files / folders - DO NOT SHORTEN. " f"Received: {file_path}" ) expanded_files.append(expanded) self.set_request_files(request, expanded_files) error = super()._validate_file_paths(request) if error: return error working_directory = request.working_directory_absolute_path if working_directory: expanded = os.path.expanduser(working_directory) if not os.path.isabs(expanded): return ( "Error: 'working_directory_absolute_path' must be an absolute path (you may use '~' which will be expanded). " f"Received: {working_directory}" ) if not os.path.isdir(expanded): return ( "Error: 'working_directory_absolute_path' must reference an existing directory. " f"Received: {working_directory}" ) return None def format_response(self, response: str, request: ChatRequest, model_info: Optional[dict] = None) -> str: """ Format the chat response to match the original Chat tool exactly. """ self._last_recordable_response = None body = response recordable_override: Optional[str] = None if self._model_supports_code_generation(): block, remainder, _ = self._extract_generated_code_block(response) if block: sanitized_text = remainder.strip() target_directory = request.working_directory_absolute_path try: artifact_path = self._persist_generated_code_block(block, target_directory) except Exception as exc: # pragma: no cover - rare filesystem failures logger.error("Failed to persist generated code block: %s", exc, exc_info=True) warning = ( f"WARNING: Unable to write pal_generated.code inside '{target_directory}'. " "Check the path permissions and re-run. The generated code block is included below for manual handling." ) history_copy_base = sanitized_text history_copy = self._join_sections(history_copy_base, warning) if history_copy_base else warning recordable_override = history_copy sanitized_warning = history_copy.strip() body = f"{sanitized_warning}\n\n{block.strip()}".strip() else: if not sanitized_text: base_message = ( "Generated code saved to pal_generated.code.\n" "\n" "CRITICAL: Contains mixed instructions + partial snippets - NOT complete code to copy as-is!\n" "\n" "You MUST:\n" " 1. Read as a proposal from partial context - you may need to read the file in sections\n" " 2. Implement ideas using YOUR complete codebase context and understanding\n" " 3. Never paste wholesale - snippets may be partial with missing lines, pasting will corrupt your code!\n" " 4. Adapt to fit your actual structure and style\n" " 5. Build/lint/test after implementation to verify correctness\n" "\n" "Treat as guidance to implement thoughtfully, not ready-to-paste code." ) sanitized_text = base_message instruction = self._build_agent_instruction(artifact_path) body = self._join_sections(sanitized_text, instruction) final_output = ( f"{body}\n\n---\n\nAGENT'S TURN: Evaluate this perspective alongside your analysis to " "form a comprehensive solution and continue with the user's request and task at hand." ) if recordable_override is not None: self._last_recordable_response = ( f"{recordable_override}\n\n---\n\nAGENT'S TURN: Evaluate this perspective alongside your analysis to " "form a comprehensive solution and continue with the user's request and task at hand." ) else: self._last_recordable_response = final_output return final_output def _record_assistant_turn( self, continuation_id: str, response_text: str, request, model_info: Optional[dict] ) -> None: recordable = self._last_recordable_response if self._last_recordable_response is not None else response_text try: super()._record_assistant_turn(continuation_id, recordable, request, model_info) finally: self._last_recordable_response = None def _model_supports_code_generation(self) -> bool: context = getattr(self, "_model_context", None) if not context: return False try: capabilities = context.capabilities except Exception: # pragma: no cover - defensive fallback return False return bool(capabilities.allow_code_generation) def _extract_generated_code_block(self, text: str) -> tuple[Optional[str], str, int]: matches = list(re.finditer(r".*?", text, flags=re.DOTALL | re.IGNORECASE)) if not matches: return None, text, 0 last_match = matches[-1] block = last_match.group(0).strip() # Merge the text before and after the final block while trimming excess whitespace before = text[: last_match.start()] after = text[last_match.end() :] remainder = self._join_sections(before, after) return block, remainder, len(matches) def _persist_generated_code_block(self, block: str, working_directory: str) -> Path: expanded = os.path.expanduser(working_directory) target_dir = Path(expanded).resolve() if not target_dir.is_dir(): raise FileNotFoundError(f"Absolute working directory path '{working_directory}' does not exist") target_file = target_dir / "pal_generated.code" if target_file.exists(): try: target_file.unlink() except OSError as exc: logger.warning("Unable to remove existing pal_generated.code: %s", exc) content = block if block.endswith("\n") else f"{block}\n" target_file.write_text(content, encoding="utf-8") logger.info("Generated code artifact written to %s", target_file) return target_file @staticmethod def _build_agent_instruction(artifact_path: Path) -> str: return ( f"CONTINUING FROM PREVIOUS DISCUSSION: Implementation plan saved to `{artifact_path}`.\n" "\n" f"CRITICAL WARNING: `{artifact_path}` may contain partial code snippets from another AI with limited context. " "Wholesale copy-pasting MAY CORRUPT your codebase with incomplete logic and missing lines.\n" "\n" "Required workflow:\n" "1. For blocks: Partial excerpts only. Understand the intent and implement using YOUR full context. " "DO NOT copy wholesale - adapt ideas to fit actual structure.\n" "2. For blocks: Understand proposal and create properly. Verify completeness (imports, syntax, logic).\n" "3. Validation: After ALL changes, verify correctness using available tools (build/compile, linters, tests, type checks, etc.).\n" f"4. Cleanup: After you're done reading and applying changes, delete `{artifact_path}` once verified to prevent stale instructions.\n" "\n" "Treat this as a patch-set requiring manual integration, not ready-to-paste code. You have full codebase context - use it." ) @staticmethod def _join_sections(*sections: str) -> str: chunks: list[str] = [] for section in sections: if section: trimmed = section.strip() if trimmed: chunks.append(trimmed) return "\n\n".join(chunks) def get_websearch_guidance(self) -> str: """ Return Chat tool-style web search guidance. """ return self.get_chat_style_websearch_guidance() logger = logging.getLogger(__name__) ================================================ FILE: tools/clink.py ================================================ """clink tool - bridge PAL MCP requests to external AI CLIs.""" from __future__ import annotations import logging import re from datetime import datetime, timezone from pathlib import Path from typing import Any from mcp.types import TextContent from pydantic import BaseModel, Field from clink import get_registry from clink.agents import AgentOutput, CLIAgentError, create_agent from clink.models import ResolvedCLIClient, ResolvedCLIRole from config import TEMPERATURE_BALANCED from tools.models import ToolModelCategory, ToolOutput from tools.shared.base_models import COMMON_FIELD_DESCRIPTIONS from tools.shared.exceptions import ToolExecutionError from tools.simple.base import SchemaBuilder, SimpleTool logger = logging.getLogger(__name__) MAX_RESPONSE_CHARS = 20_000 SUMMARY_PATTERN = re.compile(r"(.*?)", re.IGNORECASE | re.DOTALL) class CLinkRequest(BaseModel): """Request model for clink tool.""" prompt: str = Field(..., description="Prompt forwarded to the target CLI.") cli_name: str | None = Field( default=None, description="Configured CLI client name to invoke. Defaults to the first configured CLI if omitted.", ) role: str | None = Field( default=None, description="Optional role preset defined in the CLI configuration (defaults to 'default').", ) absolute_file_paths: list[str] = Field( default_factory=list, description=COMMON_FIELD_DESCRIPTIONS["absolute_file_paths"], ) images: list[str] = Field( default_factory=list, description=COMMON_FIELD_DESCRIPTIONS["images"], ) continuation_id: str | None = Field( default=None, description=COMMON_FIELD_DESCRIPTIONS["continuation_id"], ) class CLinkTool(SimpleTool): """Bridge MCP requests to configured CLI agents. Schema metadata is cached at construction time and execution relies on the shared SimpleTool hooks for conversation memory. Prompt preparation is customised so we pass instructions and file references suitable for another CLI agent. """ def __init__(self) -> None: # Cache registry metadata so the schema surfaces concrete enum values. self._registry = get_registry() self._cli_names = self._registry.list_clients() self._role_map: dict[str, list[str]] = {name: self._registry.list_roles(name) for name in self._cli_names} self._all_roles: list[str] = sorted({role for roles in self._role_map.values() for role in roles}) if "gemini" in self._cli_names: self._default_cli_name = "gemini" else: self._default_cli_name = self._cli_names[0] if self._cli_names else None self._active_system_prompt: str = "" super().__init__() def get_name(self) -> str: return "clink" def get_description(self) -> str: return ( "Link a request to an external AI CLI (Gemini CLI, Qwen CLI, etc.) through PAL MCP to reuse " "their capabilities inside existing workflows." ) def get_annotations(self) -> dict[str, Any]: return {"readOnlyHint": True} def requires_model(self) -> bool: return False def get_model_category(self) -> ToolModelCategory: return ToolModelCategory.BALANCED def get_default_temperature(self) -> float: return TEMPERATURE_BALANCED def get_system_prompt(self) -> str: return self._active_system_prompt or "" def get_request_model(self): return CLinkRequest def get_input_schema(self) -> dict[str, Any]: # Surface configured CLI names and roles directly in the schema so MCP clients # (and downstream agents) can discover available options without consulting # a separate registry call. role_descriptions = [] for name in self._cli_names: roles = ", ".join(sorted(self._role_map.get(name, ["default"]))) or "default" role_descriptions.append(f"{name}: {roles}") if role_descriptions: cli_available = ", ".join(self._cli_names) if self._cli_names else "(none configured)" default_text = ( f" Default: {self._default_cli_name}." if self._default_cli_name and len(self._cli_names) <= 1 else "" ) cli_description = ( "Configured CLI client name (from conf/cli_clients). Available: " + cli_available + default_text ) role_description = ( "Optional role preset defined for the selected CLI (defaults to 'default'). Roles per CLI: " + "; ".join(role_descriptions) ) else: cli_description = "Configured CLI client name (from conf/cli_clients)." role_description = "Optional role preset defined for the selected CLI (defaults to 'default')." properties = { "prompt": { "type": "string", "description": "User request forwarded to the CLI (conversation context is pre-applied).", }, "cli_name": { "type": "string", "enum": self._cli_names, "description": cli_description, }, "role": { "type": "string", "enum": self._all_roles or ["default"], "description": role_description, }, "absolute_file_paths": SchemaBuilder.SIMPLE_FIELD_SCHEMAS["absolute_file_paths"], "images": SchemaBuilder.COMMON_FIELD_SCHEMAS["images"], "continuation_id": SchemaBuilder.COMMON_FIELD_SCHEMAS["continuation_id"], } schema = { "type": "object", "properties": properties, "required": ["prompt"], "additionalProperties": False, } if len(self._cli_names) > 1: schema["required"].append("cli_name") return schema def get_tool_fields(self) -> dict[str, dict[str, Any]]: """Unused by clink because we override the schema end-to-end.""" return {} async def execute(self, arguments: dict[str, Any]) -> list[TextContent]: self._current_arguments = arguments request = self.get_request_model()(**arguments) path_error = self._validate_file_paths(request) if path_error: self._raise_tool_error(path_error) selected_cli = request.cli_name or self._default_cli_name if not selected_cli: self._raise_tool_error("No CLI clients are configured for clink.") try: client_config = self._registry.get_client(selected_cli) except KeyError as exc: self._raise_tool_error(str(exc)) try: role_config = client_config.get_role(request.role) except KeyError as exc: self._raise_tool_error(str(exc)) absolute_file_paths = self.get_request_files(request) images = self.get_request_images(request) continuation_id = self.get_request_continuation_id(request) self._model_context = arguments.get("_model_context") system_prompt_text = role_config.prompt_path.read_text(encoding="utf-8") include_system_prompt = not self._use_external_system_prompt(client_config) try: prompt_text = await self._prepare_prompt_for_role( request, role_config, system_prompt=system_prompt_text, include_system_prompt=include_system_prompt, ) except Exception as exc: logger.exception("Failed to prepare clink prompt") self._raise_tool_error(f"Failed to prepare prompt: {exc}") agent = create_agent(client_config) try: result = await agent.run( role=role_config, prompt=prompt_text, system_prompt=system_prompt_text if system_prompt_text.strip() else None, files=absolute_file_paths, images=images, ) except CLIAgentError as exc: metadata = self._build_error_metadata(client_config, exc) self._raise_tool_error( f"CLI '{client_config.name}' execution failed: {exc}", metadata=metadata, ) metadata = self._build_success_metadata(client_config, role_config, result) metadata = self._prune_metadata(metadata, client_config, reason="normal") content, metadata = self._apply_output_limit( client_config, result.parsed.content, metadata, ) model_info = { "provider": client_config.name, "model_name": result.parsed.metadata.get("model_used"), } if continuation_id: try: self._record_assistant_turn(continuation_id, content, request, model_info) except Exception: logger.debug("Failed to record assistant turn for continuation %s", continuation_id, exc_info=True) continuation_offer = self._create_continuation_offer(request, model_info) if continuation_offer: tool_output = self._create_continuation_offer_response( content, continuation_offer, request, model_info, ) tool_output.metadata = self._merge_metadata(tool_output.metadata, metadata) else: tool_output = ToolOutput( status="success", content=content, content_type="text", metadata=metadata, ) return [TextContent(type="text", text=tool_output.model_dump_json())] async def prepare_prompt(self, request) -> str: client_config = self._registry.get_client(request.cli_name) role_config = client_config.get_role(request.role) system_prompt_text = role_config.prompt_path.read_text(encoding="utf-8") include_system_prompt = not self._use_external_system_prompt(client_config) return await self._prepare_prompt_for_role( request, role_config, system_prompt=system_prompt_text, include_system_prompt=include_system_prompt, ) async def _prepare_prompt_for_role( self, request: CLinkRequest, role: ResolvedCLIRole, *, system_prompt: str, include_system_prompt: bool, ) -> str: """Load the role prompt and assemble the final user message.""" self._active_system_prompt = system_prompt try: user_content = self.handle_prompt_file_with_fallback(request).strip() guidance = self._agent_capabilities_guidance() file_section = self._format_file_references(self.get_request_files(request)) sections: list[str] = [] active_prompt = self.get_system_prompt().strip() if include_system_prompt and active_prompt: sections.append(active_prompt) sections.append(guidance) sections.append("=== USER REQUEST ===\n" + user_content) if file_section: sections.append("=== FILE REFERENCES ===\n" + file_section) sections.append("Provide your response below using your own CLI tools as needed:") return "\n\n".join(sections) finally: self._active_system_prompt = "" def _use_external_system_prompt(self, client: ResolvedCLIClient) -> bool: runner_name = (client.runner or client.name).lower() return runner_name == "claude" def _build_success_metadata( self, client: ResolvedCLIClient, role: ResolvedCLIRole, result: AgentOutput, ) -> dict[str, Any]: """Capture execution metadata for successful CLI calls.""" metadata: dict[str, Any] = { "cli_name": client.name, "role": role.name, "command": result.sanitized_command, "duration_seconds": round(result.duration_seconds, 3), "parser": result.parser_name, "return_code": result.returncode, } metadata.update(result.parsed.metadata) if result.stderr.strip(): metadata.setdefault("stderr", result.stderr.strip()) if result.output_file_content and "raw" not in metadata: metadata["raw_output_file"] = result.output_file_content return metadata def _merge_metadata(self, base: dict[str, Any] | None, extra: dict[str, Any]) -> dict[str, Any]: merged = dict(base or {}) merged.update(extra) return merged def _apply_output_limit( self, client: ResolvedCLIClient, content: str, metadata: dict[str, Any], ) -> tuple[str, dict[str, Any]]: if len(content) <= MAX_RESPONSE_CHARS: return content, metadata summary = self._extract_summary(content) if summary: summary_text = summary if len(summary_text) > MAX_RESPONSE_CHARS: logger.debug( "Clink summary from %s exceeded %d chars; truncating summary to fit.", client.name, MAX_RESPONSE_CHARS, ) summary_text = summary_text[:MAX_RESPONSE_CHARS] summary_metadata = self._prune_metadata(metadata, client, reason="summary") summary_metadata.update( { "output_summarized": True, "output_original_length": len(content), "output_summary_length": len(summary_text), "output_limit": MAX_RESPONSE_CHARS, } ) logger.info( "Clink compressed %s output via : original=%d chars, summary=%d chars", client.name, len(content), len(summary_text), ) return summary_text, summary_metadata truncated_metadata = self._prune_metadata(metadata, client, reason="truncated") truncated_metadata.update( { "output_truncated": True, "output_original_length": len(content), "output_limit": MAX_RESPONSE_CHARS, } ) excerpt_limit = min(4000, MAX_RESPONSE_CHARS // 2) excerpt = content[:excerpt_limit] truncated_metadata["output_excerpt_length"] = len(excerpt) logger.warning( "Clink truncated %s output: original=%d chars exceeds limit=%d; excerpt_length=%d", client.name, len(content), MAX_RESPONSE_CHARS, len(excerpt), ) message = ( f"CLI '{client.name}' produced {len(content)} characters, exceeding the configured clink limit " f"({MAX_RESPONSE_CHARS} characters). The full output was suppressed to stay within MCP response caps. " "Please narrow the request (review fewer files, summarize results) or run the CLI directly for the full log.\n\n" f"--- Begin excerpt ({len(excerpt)} of {len(content)} chars) ---\n{excerpt}\n--- End excerpt ---" ) return message, truncated_metadata def _extract_summary(self, content: str) -> str | None: match = SUMMARY_PATTERN.search(content) if not match: return None summary = match.group(1).strip() return summary or None def _prune_metadata( self, metadata: dict[str, Any], client: ResolvedCLIClient, *, reason: str, ) -> dict[str, Any]: cleaned = dict(metadata) events = cleaned.pop("events", None) if events is not None: cleaned[f"events_removed_for_{reason}"] = True logger.debug( "Clink dropped %s events metadata for %s response (%s)", client.name, reason, type(events).__name__, ) return cleaned def _build_error_metadata(self, client: ResolvedCLIClient, exc: CLIAgentError) -> dict[str, Any]: """Assemble metadata for failed CLI calls.""" metadata: dict[str, Any] = { "cli_name": client.name, "return_code": exc.returncode, } if exc.stdout: metadata["stdout"] = exc.stdout.strip() if exc.stderr: metadata["stderr"] = exc.stderr.strip() return metadata def _raise_tool_error(self, message: str, metadata: dict[str, Any] | None = None) -> None: error_output = ToolOutput(status="error", content=message, content_type="text", metadata=metadata) raise ToolExecutionError(error_output.model_dump_json()) def _agent_capabilities_guidance(self) -> str: return ( "You are operating through the Gemini CLI agent. You have access to your full suite of " "CLI capabilities—including launching web searches, reading files, and using any other " "available tools. Gather current information yourself and deliver the final answer without " "asking the PAL MCP host to perform searches or file reads." ) def _format_file_references(self, files: list[str]) -> str: if not files: return "" references: list[str] = [] for file_path in files: try: path = Path(file_path) stat = path.stat() modified = datetime.fromtimestamp(stat.st_mtime, tz=timezone.utc).isoformat() size = stat.st_size references.append(f"- {file_path} (last modified {modified}, {size} bytes)") except OSError: references.append(f"- {file_path} (unavailable)") return "\n".join(references) ================================================ FILE: tools/codereview.py ================================================ """ CodeReview Workflow tool - Systematic code review with step-by-step analysis This tool provides a structured workflow for comprehensive code review and analysis. It guides the CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough code examination, issue identification, and quality assessment before proceeding. The tool supports complex review scenarios including security analysis, performance evaluation, and architectural assessment. Key features: - Step-by-step code review workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic issue tracking with severity classification - Expert analysis integration with external models - Support for focused reviews (security, performance, architecture) - Confidence-based workflow optimization """ import logging from typing import TYPE_CHECKING, Any, Literal, Optional from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import CODEREVIEW_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for code review workflow CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "Review narrative. Step 1: outline the review strategy. Later steps: report findings. MUST cover quality, security, " "performance, and architecture. Reference code via `relevant_files`; avoid dumping large snippets." ), "step_number": "Current review step (starts at 1) – each step should build on the last.", "total_steps": ( "Number of review steps planned. External validation: two steps (analysis + summary). Internal validation: one step. " "Use the same limits when continuing an existing review via continuation_id." ), "next_step_required": ( "True when another review step follows. External validation: step 1 → True, step 2 → False. Internal validation: set False immediately. " "Apply the same rule on continuation flows." ), "findings": "Capture findings (positive and negative) across quality, security, performance, and architecture; update each step.", "files_checked": "Absolute paths of every file reviewed, including those ruled out.", "relevant_files": "Step 1: list all files/dirs under review. Must be absolute full non-abbreviated paths. Final step: narrow to files tied to key findings.", "relevant_context": "Functions or methods central to findings (e.g. 'Class.method' or 'function_name').", "issues_found": "Issues with severity (critical/high/medium/low) and descriptions.", "review_validation_type": "Set 'external' (default) for expert follow-up or 'internal' for local-only review.", "images": "Optional diagram or screenshot paths that clarify review context.", "review_type": "Review focus: full, security, performance, or quick.", "focus_on": "Optional note on areas to emphasise (e.g. 'threading', 'auth flow').", "standards": "Coding standards or style guides to enforce.", "severity_filter": "Lowest severity to include when reporting issues (critical/high/medium/low/all).", } class CodeReviewRequest(WorkflowRequest): """Request model for code review workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) issues_found: list[dict] = Field( default_factory=list, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["issues_found"] ) # Deprecated confidence field kept for backward compatibility only confidence: Optional[str] = Field("low", exclude=True) review_validation_type: Optional[Literal["external", "internal"]] = Field( "external", description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS.get("review_validation_type", "") ) # Optional images for visual context images: Optional[list[str]] = Field(default=None, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Code review-specific fields (only used in step 1 to initialize) review_type: Optional[Literal["full", "security", "performance", "quick"]] = Field( "full", description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["review_type"] ) focus_on: Optional[str] = Field(None, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["focus_on"]) standards: Optional[str] = Field(None, description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["standards"]) severity_filter: Optional[Literal["critical", "high", "medium", "low", "all"]] = Field( "all", description=CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["severity_filter"] ) # Override inherited fields to exclude them from schema (except model which needs to be available) temperature: Optional[float] = Field(default=None, exclude=True) thinking_mode: Optional[str] = Field(default=None, exclude=True) @model_validator(mode="after") def validate_step_one_requirements(self): """Ensure step 1 has required relevant_files field.""" if self.step_number == 1 and not self.relevant_files: raise ValueError("Step 1 requires 'relevant_files' field to specify code files or directories to review") return self class CodeReviewTool(WorkflowTool): """ Code Review workflow tool for step-by-step code review and expert analysis. This tool implements a structured code review workflow that guides users through methodical investigation steps, ensuring thorough code examination, issue identification, and quality assessment before reaching conclusions. It supports complex review scenarios including security audits, performance analysis, architectural review, and maintainability assessment. """ def __init__(self): super().__init__() self.initial_request = None self.review_config = {} def get_name(self) -> str: return "codereview" def get_description(self) -> str: return ( "Performs systematic, step-by-step code review with expert validation. " "Use for comprehensive analysis covering quality, security, performance, and architecture. " "Guides through structured investigation to ensure thoroughness." ) def get_system_prompt(self) -> str: return CODEREVIEW_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Code review requires thorough analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the code review workflow-specific request model.""" return CodeReviewRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with code review-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Code review workflow-specific field overrides codereview_field_overrides = { "step": { "type": "string", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, "review_validation_type": { "type": "string", "enum": ["external", "internal"], "default": "external", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS.get("review_validation_type", ""), }, "issues_found": { "type": "array", "items": {"type": "object"}, "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["issues_found"], }, "images": { "type": "array", "items": {"type": "string"}, "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, # Code review-specific fields (for step 1) "review_type": { "type": "string", "enum": ["full", "security", "performance", "quick"], "default": "full", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["review_type"], }, "focus_on": { "type": "string", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["focus_on"], }, "standards": { "type": "string", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["standards"], }, "severity_filter": { "type": "string", "enum": ["critical", "high", "medium", "low", "all"], "default": "all", "description": CODEREVIEW_WORKFLOW_FIELD_DESCRIPTIONS["severity_filter"], }, } # Use WorkflowSchemaBuilder with code review-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=codereview_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each investigation phase. Now includes request parameter for continuation-aware decisions. """ # Check for continuation - fast track mode if request: continuation_id = self.get_request_continuation_id(request) validation_type = self.get_review_validation_type(request) if continuation_id and validation_type == "external": if step_number == 1: return [ "Quickly review the code files to understand context", "Identify any critical issues that need immediate attention", "Note main architectural patterns and design decisions", "Prepare summary of key findings for expert validation", ] else: return ["Complete review and proceed to expert analysis"] if step_number == 1: # Initial code review investigation tasks return [ "Read and understand the code files specified for review", "Examine the overall structure, architecture, and design patterns used", "Identify the main components, classes, and functions in the codebase", "Understand the business logic and intended functionality", "Look for obvious issues: bugs, security concerns, performance problems", "Note any code smells, anti-patterns, or areas of concern", ] elif step_number == 2: # Deeper investigation for step 2 return [ "Examine specific code sections you've identified as concerning", "Analyze security implications: input validation, authentication, authorization", "Check for performance issues: algorithmic complexity, resource usage, inefficiencies", "Look for architectural problems: tight coupling, missing abstractions, scalability issues", "Identify code quality issues: readability, maintainability, error handling", "Search for over-engineering, unnecessary complexity, or design patterns that could be simplified", ] elif step_number >= 3: # Final verification for later steps return [ "Verify all identified issues have been properly documented with severity levels", "Check for any missed critical security vulnerabilities or performance bottlenecks", "Confirm that architectural concerns and code quality issues are comprehensively captured", "Ensure positive aspects and well-implemented patterns are also noted", "Validate that your assessment aligns with the review type and focus areas specified", "Double-check that findings are actionable and provide clear guidance for improvements", ] else: # General investigation needed return [ "Continue examining the codebase for additional patterns and potential issues", "Gather more evidence using appropriate code analysis techniques", "Test your assumptions about code behavior and design decisions", "Look for patterns that confirm or refute your current assessment", "Focus on areas that haven't been thoroughly examined yet", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Decide when to call external model based on investigation completeness. For continuations with external type, always proceed with expert analysis. """ # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # For continuations with external type, always proceed with expert analysis continuation_id = self.get_request_continuation_id(request) validation_type = self.get_review_validation_type(request) if continuation_id and validation_type == "external": return True # Always perform expert analysis for external continuations # Check if we have meaningful investigation data return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call for final code review validation.""" context_parts = [ f"=== CODE REVIEW REQUEST ===\\n{self.initial_request or 'Code review workflow initiated'}\\n=== END REQUEST ===" ] # Add investigation summary investigation_summary = self._build_code_review_summary(consolidated_findings) context_parts.append( f"\\n=== AGENT'S CODE REVIEW INVESTIGATION ===\\n{investigation_summary}\\n=== END INVESTIGATION ===" ) # Add review configuration context if available if self.review_config: config_text = "\\n".join(f"- {key}: {value}" for key, value in self.review_config.items() if value) context_parts.append(f"\\n=== REVIEW CONFIGURATION ===\\n{config_text}\\n=== END CONFIGURATION ===") # Add relevant code elements if available if consolidated_findings.relevant_context: methods_text = "\\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\\n=== RELEVANT CODE ELEMENTS ===\\n{methods_text}\\n=== END CODE ELEMENTS ===") # Add issues found if available if consolidated_findings.issues_found: issues_text = "\\n".join( f"[{issue.get('severity', 'unknown').upper()}] {issue.get('description', 'No description')}" for issue in consolidated_findings.issues_found ) context_parts.append(f"\\n=== ISSUES IDENTIFIED ===\\n{issues_text}\\n=== END ISSUES ===") # Add assessment evolution if available if consolidated_findings.hypotheses: assessments_text = "\\n".join( f"Step {h['step']} ({h['confidence']} confidence): {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\\n=== ASSESSMENT EVOLUTION ===\\n{assessments_text}\\n=== END ASSESSMENTS ===") # Add images if available if consolidated_findings.images: images_text = "\\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append( f"\\n=== VISUAL REVIEW INFORMATION ===\\n{images_text}\\n=== END VISUAL INFORMATION ===" ) return "\\n".join(context_parts) def _build_code_review_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the code review investigation.""" summary_parts = [ "=== SYSTEMATIC CODE REVIEW INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Code elements analyzed: {len(consolidated_findings.relevant_context)}", f"Issues identified: {len(consolidated_findings.issues_found)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\\n".join(summary_parts) def should_include_files_in_expert_prompt(self) -> bool: """Include files in expert analysis for comprehensive code review.""" return True def should_embed_system_prompt(self) -> bool: """Embed system prompt in expert analysis for proper context.""" return True def get_expert_thinking_mode(self) -> str: """Use high thinking mode for thorough code review analysis.""" return "high" def get_expert_analysis_instruction(self) -> str: """Get specific instruction for code review expert analysis.""" return ( "Please provide comprehensive code review analysis based on the investigation findings. " "Focus on identifying any remaining issues, validating the completeness of the analysis, " "and providing final recommendations for code improvements, following the severity-based " "format specified in the system prompt." ) # Hook method overrides for code review-specific behavior def prepare_step_data(self, request) -> dict: """ Map code review-specific fields for internal processing. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": request.issues_found, "review_validation_type": self.get_review_validation_type(request), "hypothesis": request.findings, # Map findings to hypothesis for compatibility "images": request.images or [], "confidence": "high", # Dummy value for workflow_mixin compatibility } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Code review workflow skips expert analysis only when review_validation_type is "internal". Default is always to use expert analysis (external). For continuations with external type, always perform expert analysis immediately. """ # If it's a continuation and review_validation_type is external, don't skip continuation_id = self.get_request_continuation_id(request) validation_type = self.get_review_validation_type(request) if continuation_id and validation_type != "internal": return False # Always do expert analysis for external continuations # Only skip if explicitly set to internal AND review is complete return validation_type == "internal" and not request.next_step_required def store_initial_issue(self, step_description: str): """Store initial request for expert analysis.""" self.initial_request = step_description # Override inheritance hooks for code review-specific behavior def get_review_validation_type(self, request) -> str: """Get review validation type from request. Hook method for clean inheritance.""" try: return request.review_validation_type or "external" except AttributeError: return "external" # Default to external validation def get_completion_status(self) -> str: """Code review tools use review-specific status.""" return "code_review_complete_ready_for_implementation" def get_completion_data_key(self) -> str: """Code review uses 'complete_code_review' key.""" return "complete_code_review" def get_final_analysis_from_request(self, request): """Code review tools use 'findings' field.""" return request.findings def get_confidence_level(self, request) -> str: """Code review tools use 'certain' for high confidence.""" return "certain" def get_completion_message(self) -> str: """Code review-specific completion message.""" return ( "Code review complete. You have identified all significant issues " "and provided comprehensive analysis. MANDATORY: Present the user with the complete review results " "categorized by severity, and IMMEDIATELY proceed with implementing the highest priority fixes " "or provide specific guidance for improvements. Focus on actionable recommendations." ) def get_skip_reason(self) -> str: """Code review-specific skip reason.""" return "Completed comprehensive code review with internal analysis only (no external model validation)" def get_skip_expert_analysis_status(self) -> str: """Code review-specific expert analysis skip status.""" return "skipped_due_to_internal_analysis_type" def prepare_work_summary(self) -> str: """Code review-specific work summary.""" return self._build_code_review_summary(self.consolidated_findings) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Code review-specific completion message. """ base_message = ( "CODE REVIEW IS COMPLETE. You MUST now summarize and present ALL review findings organized by " "severity (Critical → High → Medium → Low), specific code locations with line numbers, and exact " "recommendations for improvement. Clearly prioritize the top 3 issues that need immediate attention. " "Provide concrete, actionable guidance for each issue—make it easy for a developer to understand " "exactly what needs to be fixed and how to implement the improvements." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Provide specific guidance for handling expert analysis in code reviews. """ return ( "IMPORTANT: Analysis from an assistant model has been provided above. You MUST critically evaluate and validate " "the expert findings rather than accepting them blindly. Cross-reference the expert analysis with " "your own investigation findings, verify that suggested improvements are appropriate for this " "codebase's context and patterns, and ensure recommendations align with the project's standards. " "Present a synthesis that combines your systematic review with validated expert insights, clearly " "distinguishing between findings you've independently confirmed and additional insights from expert analysis." ) def get_step_guidance_message(self, request) -> str: """ Code review-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_code_review_step_guidance(request.step_number, request) return step_guidance["next_steps"] def get_code_review_step_guidance(self, step_number: int, request) -> dict[str, Any]: """ Provide step-specific guidance for code review workflow. Uses get_required_actions to determine what needs to be done, then formats those actions into appropriate guidance messages. """ # Get the required actions from the single source of truth required_actions = self.get_required_actions( step_number, "medium", # Dummy value for backward compatibility request.findings or "", request.total_steps, request, # Pass request for continuation-aware decisions ) # Check if this is a continuation to provide context-aware guidance continuation_id = self.get_request_continuation_id(request) validation_type = self.get_review_validation_type(request) is_external_continuation = continuation_id and validation_type == "external" is_internal_continuation = continuation_id and validation_type == "internal" # Step 1 handling if step_number == 1: if is_external_continuation: # Fast-track for external continuations return { "next_steps": ( "You are on step 1 of MAXIMUM 2 steps for continuation. CRITICAL: Quickly review the code NOW. " "MANDATORY ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + "\\n\\nSet next_step_required=True and step_number=2 for the next call to trigger expert analysis." ) } elif is_internal_continuation: # Internal validation mode next_steps = ( "Continuing previous conversation with internal validation only. The analysis will build " "upon the prior findings without external model validation. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) ) else: # Normal flow for new reviews next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first examine " f"the code files thoroughly using appropriate tools. CRITICAL AWARENESS: You need to:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nOnly call {self.get_name()} again AFTER completing your investigation. " f"When you call {self.get_name()} next time, use step_number: {step_number + 1} " f"and report specific files examined, issues found, and code quality assessments discovered." ) elif step_number == 2: # CRITICAL: Check if violating minimum step requirement if ( request.total_steps >= 3 and request.step_number < request.total_steps and not request.next_step_required ): next_steps = ( f"ERROR: You set total_steps={request.total_steps} but next_step_required=False on step {request.step_number}. " f"This violates the minimum step requirement. You MUST set next_step_required=True until you reach the final step. " f"Call {self.get_name()} again with next_step_required=True and continue your investigation." ) elif is_external_continuation or (not request.next_step_required and validation_type == "external"): # Fast-track completion or about to complete for external validation next_steps = ( "Proceeding immediately to expert analysis. " f"MANDATORY: call {self.get_name()} tool immediately again, and set next_step_required=False to " f"trigger external validation NOW." ) else: # Normal flow - deeper analysis needed next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. You are on step 2 of {request.total_steps} minimum required steps. " f"MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nRemember: You MUST set next_step_required=True until step {request.total_steps}. " + f"Only call {self.get_name()} again with step_number: {step_number + 1} AFTER completing these code review tasks." ) elif step_number >= 3: if not request.next_step_required and validation_type == "external": # About to complete - ready for expert analysis next_steps = ( "Completing review and proceeding to expert analysis. " "Ensure all findings are documented with specific file references and line numbers." ) else: # Later steps - final verification next_steps = ( f"WAIT! Your code review needs final verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nREMEMBER: Ensure you have identified all significant issues across all severity levels and " f"verified the completeness of your review. Document findings with specific file references and " f"line numbers where applicable, then call {self.get_name()} with step_number: {step_number + 1}." ) else: # Fallback for any other case - check minimum step violation first if ( request.total_steps >= 3 and request.step_number < request.total_steps and not request.next_step_required ): next_steps = ( f"ERROR: You set total_steps={request.total_steps} but next_step_required=False on step {request.step_number}. " f"This violates the minimum step requirement. You MUST set next_step_required=True until step {request.total_steps}." ) elif not request.next_step_required and validation_type == "external": next_steps = ( "Completing review. " "Ensure all findings are documented with specific file references and severity levels." ) else: next_steps = ( f"PAUSE REVIEW. Before calling {self.get_name()} step {step_number + 1}, you MUST examine more code thoroughly. " + "Required: " + ", ".join(required_actions[:2]) + ". " + f"Your next {self.get_name()} call (step_number: {step_number + 1}) must include " f"NEW evidence from actual code analysis, not just theories. NO recursive {self.get_name()} calls " f"without investigation work!" ) return {"next_steps": next_steps} def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match code review workflow format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Store review configuration for expert analysis if request.relevant_files: self.review_config = { "relevant_files": request.relevant_files, "review_type": request.review_type, "focus_on": request.focus_on, "standards": request.standards, "severity_filter": request.severity_filter, } # Convert generic status names to code review-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "code_review_in_progress", f"pause_for_{tool_name}": "pause_for_code_review", f"{tool_name}_required": "code_review_required", f"{tool_name}_complete": "code_review_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match code review workflow if f"{tool_name}_status" in response_data: response_data["code_review_status"] = response_data.pop(f"{tool_name}_status") # Add code review-specific status fields response_data["code_review_status"]["issues_by_severity"] = {} for issue in self.consolidated_findings.issues_found: severity = issue.get("severity", "unknown") if severity not in response_data["code_review_status"]["issues_by_severity"]: response_data["code_review_status"]["issues_by_severity"][severity] = 0 response_data["code_review_status"]["issues_by_severity"][severity] += 1 response_data["code_review_status"]["review_validation_type"] = self.get_review_validation_type(request) # Map complete_codereviewworkflow to complete_code_review if f"complete_{tool_name}" in response_data: response_data["complete_code_review"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match code review workflow if f"{tool_name}_complete" in response_data: response_data["code_review_complete"] = response_data.pop(f"{tool_name}_complete") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the code review workflow-specific request model.""" return CodeReviewRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/consensus.py ================================================ """ Consensus tool - Step-by-step multi-model consensus with expert analysis This tool provides a structured workflow for gathering consensus from multiple models. It guides the CLI agent through systematic steps where the CLI agent first provides its own analysis, then consults each requested model one by one, and finally synthesizes all perspectives. Key features: - Step-by-step consensus workflow with progress tracking - The CLI agent's initial neutral analysis followed by model-specific consultations - Context-aware file embedding - Support for stance-based analysis (for/against/neutral) - Final synthesis combining all perspectives """ from __future__ import annotations import json import logging from typing import TYPE_CHECKING, Any from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from mcp.types import TextContent from config import TEMPERATURE_ANALYTICAL from systemprompts import CONSENSUS_PROMPT from tools.shared.base_models import ConsolidatedFindings, WorkflowRequest from utils.conversation_memory import MAX_CONVERSATION_TURNS, create_thread, get_thread from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for consensus workflow CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "Consensus prompt. Step 1: write the exact proposal/question every model will see (use 'Evaluate…', not meta commentary). " "Steps 2+: capture internal notes about the latest model response—these notes are NOT sent to other models." ), "step_number": "Current step index (starts at 1). Step 1 is your analysis; steps 2+ handle each model response.", "total_steps": "Total steps = number of models consulted plus the final synthesis step.", "next_step_required": "True if more model consultations remain; set false when ready to synthesize.", "findings": ( "Step 1: your independent analysis for later synthesis (not shared with other models). Steps 2+: summarize the newest model response." ), "relevant_files": "Optional supporting files that help the consensus analysis. Must be absolute full, non-abbreviated paths.", "models": ( "User-specified list of models to consult (provide at least two entries). " "Each entry may include model, stance (for/against/neutral), and stance_prompt. " "Each (model, stance) pair must be unique, e.g. [{'model':'gpt5','stance':'for'}, {'model':'pro','stance':'against'}]." ), "current_model_index": "0-based index of the next model to consult (managed internally).", "model_responses": "Internal log of responses gathered so far.", "images": "Optional absolute image paths or base64 references that add helpful visual context.", } class ConsensusRequest(WorkflowRequest): """Request model for consensus workflow steps""" # Required fields for each step step: str = Field(..., description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) confidence: str = Field(default="exploring", exclude=True, description="Not used") # Consensus-specific fields (only needed in step 1) models: list[dict] | None = Field(None, description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["models"]) relevant_files: list[str] | None = Field( default_factory=list, description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], ) # Internal tracking fields current_model_index: int | None = Field( 0, description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["current_model_index"], ) model_responses: list[dict] | None = Field( default_factory=list, description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["model_responses"], ) # Optional images for visual debugging images: list[str] | None = Field(default=None, description=CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Override inherited fields to exclude them from schema temperature: float | None = Field(default=None, exclude=True) thinking_mode: str | None = Field(default=None, exclude=True) # Not used in consensus workflow files_checked: list[str] | None = Field(default_factory=list, exclude=True) relevant_context: list[str] | None = Field(default_factory=list, exclude=True) issues_found: list[dict] | None = Field(default_factory=list, exclude=True) hypothesis: str | None = Field(None, exclude=True) @model_validator(mode="after") def validate_step_one_requirements(self): """Ensure step 1 has required models field and unique model+stance combinations.""" if self.step_number == 1: if not self.models: raise ValueError("Step 1 requires 'models' field to specify which models to consult") # Check for unique model + stance combinations seen_combinations = set() for model_config in self.models: model_name = model_config.get("model", "") stance = model_config.get("stance", "neutral") combination = f"{model_name}:{stance}" if combination in seen_combinations: raise ValueError( f"Duplicate model + stance combination found: {model_name} with stance '{stance}'. " f"Each model + stance combination must be unique." ) seen_combinations.add(combination) return self class ConsensusTool(WorkflowTool): """ Consensus workflow tool for step-by-step multi-model consensus gathering. This tool implements a structured consensus workflow where the CLI agent first provides its own neutral analysis, then consults each specified model individually, and finally synthesizes all perspectives into a unified recommendation. """ def __init__(self): super().__init__() self.initial_prompt: str | None = None self.original_proposal: str | None = None # Store the original proposal separately self.models_to_consult: list[dict] = [] self.accumulated_responses: list[dict] = [] self._current_arguments: dict[str, Any] = {} def get_name(self) -> str: return "consensus" def get_description(self) -> str: return ( "Builds multi-model consensus through systematic analysis and structured debate. " "Use for complex decisions, architectural choices, feature proposals, and technology evaluations. " "Consults multiple models with different stances to synthesize comprehensive recommendations." ) def get_system_prompt(self) -> str: # For the CLI agent's initial analysis, use a neutral version of the consensus prompt return CONSENSUS_PROMPT.replace( "{stance_prompt}", """BALANCED PERSPECTIVE Provide objective analysis considering both positive and negative aspects. However, if there is overwhelming evidence that the proposal clearly leans toward being exceptionally good or particularly problematic, you MUST accurately reflect this reality. Being "balanced" means being truthful about the weight of evidence, not artificially creating 50/50 splits when the reality is 90/10. Your analysis should: - Present all significant pros and cons discovered - Weight them according to actual impact and likelihood - If evidence strongly favors one conclusion, clearly state this - Provide proportional coverage based on the strength of arguments - Help the questioner see the true balance of considerations Remember: Artificial balance that misrepresents reality is not helpful. True balance means accurate representation of the evidence, even when it strongly points in one direction.""", ) def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> ToolModelCategory: """Consensus workflow requires extended reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the consensus workflow-specific request model.""" return ConsensusRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema for consensus workflow.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Consensus tool-specific field definitions consensus_field_overrides = { # Override standard workflow fields that need consensus-specific descriptions "step": { "type": "string", "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, # consensus-specific fields (not in base workflow) "models": { "type": "array", "items": { "type": "object", "properties": { "model": {"type": "string"}, "stance": {"type": "string", "enum": ["for", "against", "neutral"], "default": "neutral"}, "stance_prompt": {"type": "string"}, }, "required": ["model"], }, "description": ( "User-specified roster of models to consult (provide at least two entries). " + CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["models"] ), "minItems": 2, }, "current_model_index": { "type": "integer", "minimum": 0, "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["current_model_index"], }, "model_responses": { "type": "array", "items": {"type": "object"}, "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["model_responses"], }, "images": { "type": "array", "items": {"type": "string"}, "description": CONSENSUS_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, } # Provide guidance on available models similar to single-model tools model_description = ( "When the user names a model, you MUST use that exact value or report the " "provider error—never swap in another option. Use the `listmodels` tool for the full roster." ) summaries, total, restricted = self._get_ranked_model_summaries() remainder = max(0, total - len(summaries)) if summaries: label = "Allowed models" if restricted else "Top models" top_line = "; ".join(summaries) if remainder > 0: top_line = f"{label}: {top_line}; +{remainder} more via `listmodels`." else: top_line = f"{label}: {top_line}." model_description = f"{model_description} {top_line}" else: model_description = ( f"{model_description} No models detected—configure provider credentials or use the `listmodels` tool " "to inspect availability." ) restriction_note = self._get_restriction_note() if restriction_note and (remainder > 0 or not summaries): model_description = f"{model_description} {restriction_note}." existing_models_desc = consensus_field_overrides["models"]["description"] consensus_field_overrides["models"]["description"] = f"{existing_models_desc} {model_description}" # Define excluded fields for consensus workflow excluded_workflow_fields = [ "files_checked", # Not used in consensus workflow "relevant_context", # Not used in consensus workflow "issues_found", # Not used in consensus workflow "hypothesis", # Not used in consensus workflow "confidence", # Not used in consensus workflow ] excluded_common_fields = [ "model", # Consensus uses 'models' field instead "temperature", # Not used in consensus workflow "thinking_mode", # Not used in consensus workflow ] requires_model = self.requires_model() model_field_schema = self.get_model_field_schema() if requires_model else None auto_mode = self.is_effective_auto_mode() if requires_model else False return WorkflowSchemaBuilder.build_schema( tool_specific_fields=consensus_field_overrides, model_field_schema=model_field_schema, auto_mode=auto_mode, tool_name=self.get_name(), excluded_workflow_fields=excluded_workflow_fields, excluded_common_fields=excluded_common_fields, require_model=requires_model, ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: # noqa: ARG002 """Define required actions for each consensus phase. Now includes request parameter for continuation-aware decisions. Note: confidence parameter is kept for compatibility with base class but not used. """ if step_number == 1: # CLI Agent's initial analysis return [ "You've provided your initial analysis. The tool will now consult other models.", "Wait for the next step to receive the first model's response.", ] elif step_number < total_steps - 1: # Processing individual model responses return [ "Review the model response provided in this step", "Note key agreements and disagreements with previous analyses", "Wait for the next model's response", ] else: # Ready for final synthesis return [ "All models have been consulted", "Synthesize all perspectives into a comprehensive recommendation", "Identify key points of agreement and disagreement", "Provide clear, actionable guidance based on the consensus", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """Consensus workflow doesn't use traditional expert analysis - it consults models step by step.""" return False def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Not used in consensus workflow.""" return "" def requires_expert_analysis(self) -> bool: """Consensus workflow handles its own model consultations.""" return False def requires_model(self) -> bool: """ Consensus tool doesn't require model resolution at the MCP boundary. Uses it's own set of models Returns: bool: False """ return False # Hook method overrides for consensus-specific behavior def prepare_step_data(self, request) -> dict: """Prepare consensus-specific step data.""" step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": [], # Not used "relevant_files": request.relevant_files or [], "relevant_context": [], # Not used "issues_found": [], # Not used "confidence": "exploring", # Not used, kept for compatibility "hypothesis": None, # Not used "images": request.images or [], # Now used for visual context } return step_data async def handle_work_completion(self, response_data: dict, request, arguments: dict) -> dict: # noqa: ARG002 """Handle consensus workflow completion - no expert analysis, just final synthesis.""" response_data["consensus_complete"] = True response_data["status"] = "consensus_workflow_complete" # Prepare final synthesis data response_data["complete_consensus"] = { "initial_prompt": self.original_proposal if self.original_proposal else self.initial_prompt, "models_consulted": [m["model"] + ":" + m.get("stance", "neutral") for m in self.accumulated_responses], "total_responses": len(self.accumulated_responses), "consensus_confidence": "high", # Consensus complete } response_data["next_steps"] = ( "CONSENSUS GATHERING IS COMPLETE. You MUST now synthesize all perspectives and present:\n" "1. Key points of AGREEMENT across models\n" "2. Key points of DISAGREEMENT and why they differ\n" "3. Your final consolidated recommendation\n" "4. Specific, actionable next steps for implementation\n" "5. Critical risks or concerns that must be addressed" ) return response_data def handle_work_continuation(self, response_data: dict, request) -> dict: """Handle continuation between consensus steps.""" current_idx = request.current_model_index or 0 if request.step_number == 1: # After CLI Agent's initial analysis, prepare to consult first model response_data["status"] = "consulting_models" response_data["next_model"] = self.models_to_consult[0] if self.models_to_consult else None response_data["next_steps"] = ( "Your initial analysis is complete. The tool will now consult the specified models." ) elif current_idx < len(self.models_to_consult): next_model = self.models_to_consult[current_idx] response_data["status"] = "consulting_next_model" response_data["next_model"] = next_model response_data["models_remaining"] = len(self.models_to_consult) - current_idx response_data["next_steps"] = f"Model consultation in progress. Next: {next_model['model']}" else: response_data["status"] = "ready_for_synthesis" response_data["next_steps"] = "All models consulted. Ready for final synthesis." return response_data async def execute_workflow(self, arguments: dict[str, Any]) -> list: """Override execute_workflow to handle model consultations between steps.""" # Store arguments self._current_arguments = arguments # Validate request request = self.get_workflow_request_model()(**arguments) # Resolve existing continuation_id or create a new one on first step continuation_id = request.continuation_id if request.step_number == 1: if not continuation_id: clean_args = {k: v for k, v in arguments.items() if k not in ["_model_context", "_resolved_model_name"]} continuation_id = create_thread(self.get_name(), clean_args) request.continuation_id = continuation_id arguments["continuation_id"] = continuation_id self.work_history = [] self.consolidated_findings = ConsolidatedFindings() # Store the original proposal from step 1 - this is what all models should see self.store_initial_issue(request.step) self.initial_request = request.step self.models_to_consult = request.models or [] self.accumulated_responses = [] # Set total steps: len(models) (each step includes consultation + response) request.total_steps = len(self.models_to_consult) # For all steps (1 through total_steps), consult the corresponding model if request.step_number <= request.total_steps: # Calculate which model to consult for this step model_idx = request.step_number - 1 # 0-based index if model_idx < len(self.models_to_consult): # Track workflow state for conversation memory step_data = self.prepare_step_data(request) self.work_history.append(step_data) self._update_consolidated_findings(step_data) # Consult the model for this step model_response = await self._consult_model(self.models_to_consult[model_idx], request) # Add to accumulated responses self.accumulated_responses.append(model_response) # Include the model response in the step data response_data = { "status": "model_consulted", "step_number": request.step_number, "total_steps": request.total_steps, "model_consulted": model_response["model"], "model_stance": model_response.get("stance", "neutral"), "model_response": model_response, "current_model_index": model_idx + 1, "next_step_required": request.step_number < request.total_steps, } # Add CLAI Agent's analysis to step 1 if request.step_number == 1: response_data["agent_analysis"] = { "initial_analysis": request.step, "findings": request.findings, } response_data["status"] = "analysis_and_first_model_consulted" # Check if this is the final step if request.step_number == request.total_steps: response_data["status"] = "consensus_workflow_complete" response_data["consensus_complete"] = True response_data["complete_consensus"] = { "initial_prompt": self.original_proposal if self.original_proposal else self.initial_prompt, "models_consulted": [ f"{m['model']}:{m.get('stance', 'neutral')}" for m in self.accumulated_responses ], "total_responses": len(self.accumulated_responses), "consensus_confidence": "high", } response_data["next_steps"] = ( "CONSENSUS GATHERING IS COMPLETE. Synthesize all perspectives and present:\n" "1. Key points of AGREEMENT across models\n" "2. Key points of DISAGREEMENT and why they differ\n" "3. Your final consolidated recommendation\n" "4. Specific, actionable next steps for implementation\n" "5. Critical risks or concerns that must be addressed" ) else: response_data["next_steps"] = ( f"Model {model_response['model']} has provided its {model_response.get('stance', 'neutral')} " f"perspective. Please analyze this response and call {self.get_name()} again with:\n" f"- step_number: {request.step_number + 1}\n" f"- findings: Summarize key points from this model's response" ) # Add continuation information and workflow customization response_data = self.customize_workflow_response(response_data, request) # Ensure consensus-specific metadata is attached self._add_workflow_metadata(response_data, arguments) if continuation_id: self.store_conversation_turn(continuation_id, response_data, request) continuation_offer = self._build_continuation_offer(continuation_id) if continuation_offer: response_data["continuation_offer"] = continuation_offer return [TextContent(type="text", text=json.dumps(response_data, indent=2, ensure_ascii=False))] # Otherwise, use standard workflow execution return await super().execute_workflow(arguments) def _build_continuation_offer(self, continuation_id: str) -> dict[str, Any] | None: """Create a continuation offer without exposing prior model responses.""" try: from tools.models import ContinuationOffer thread = get_thread(continuation_id) if thread and thread.turns: remaining_turns = max(0, MAX_CONVERSATION_TURNS - len(thread.turns)) else: remaining_turns = MAX_CONVERSATION_TURNS - 1 # Provide a neutral note specific to consensus workflow note = ( f"Consensus workflow can continue for {remaining_turns} more exchanges." if remaining_turns > 0 else "Consensus workflow continuation limit reached." ) continuation_offer = ContinuationOffer( continuation_id=continuation_id, note=note, remaining_turns=remaining_turns, ) return continuation_offer.model_dump() except Exception: return None async def _consult_model(self, model_config: dict, request) -> dict: """Consult a single model and return its response.""" try: # Import and create ModelContext once at the beginning from utils.model_context import ModelContext # Get the provider for this model model_name = model_config["model"] provider = self.get_model_provider(model_name) # Create model context once and reuse for both file processing and temperature validation model_context = ModelContext(model_name=model_name) # Prepare the prompt with any relevant files # Use continuation_id=None for blinded consensus - each model should only see # original prompt + files, not conversation history or other model responses # CRITICAL: Use the original proposal from step 1, NOT what's in request.step for steps 2+! # Steps 2+ contain summaries/notes that must NEVER be sent to other models prompt = self.original_proposal if self.original_proposal else self.initial_prompt if request.relevant_files: file_content, _ = self._prepare_file_content_for_prompt( request.relevant_files, None, # Use None instead of request.continuation_id for blinded consensus "Context files", model_context=model_context, ) if file_content: prompt = f"{prompt}\n\n=== CONTEXT FILES ===\n{file_content}\n=== END CONTEXT ===" # Get stance-specific system prompt stance = model_config.get("stance", "neutral") stance_prompt = model_config.get("stance_prompt") system_prompt = self._get_stance_enhanced_prompt(stance, stance_prompt) # Validate temperature against model constraints (respects supports_temperature) validated_temperature, temp_warnings = self.validate_and_correct_temperature( self.get_default_temperature(), model_context ) # Log any temperature corrections for warning in temp_warnings: logger.warning(warning) # Call the model with validated temperature response = provider.generate_content( prompt=prompt, model_name=model_name, system_prompt=system_prompt, temperature=validated_temperature, thinking_mode="medium", images=request.images if request.images else None, ) return { "model": model_name, "stance": stance, "status": "success", "verdict": response.content, "metadata": { "provider": provider.get_provider_type().value, "model_name": model_name, }, } except Exception as e: logger.exception("Error consulting model %s", model_config) return { "model": model_config.get("model", "unknown"), "stance": model_config.get("stance", "neutral"), "status": "error", "error": str(e), } def _get_stance_enhanced_prompt(self, stance: str, custom_stance_prompt: str | None = None) -> str: """Get the system prompt with stance injection.""" base_prompt = CONSENSUS_PROMPT if custom_stance_prompt: return base_prompt.replace("{stance_prompt}", custom_stance_prompt) stance_prompts = { "for": """SUPPORTIVE PERSPECTIVE WITH INTEGRITY You are tasked with advocating FOR this proposal, but with CRITICAL GUARDRAILS: MANDATORY ETHICAL CONSTRAINTS: - This is NOT a debate for entertainment. You MUST act in good faith and in the best interest of the questioner - You MUST think deeply about whether supporting this idea is safe, sound, and passes essential requirements - You MUST be direct and unequivocal in saying "this is a bad idea" when it truly is - There must be at least ONE COMPELLING reason to be optimistic, otherwise DO NOT support it WHEN TO REFUSE SUPPORT (MUST OVERRIDE STANCE): - If the idea is fundamentally harmful to users, project, or stakeholders - If implementation would violate security, privacy, or ethical standards - If the proposal is technically infeasible within realistic constraints - If costs/risks dramatically outweigh any potential benefits YOUR SUPPORTIVE ANALYSIS SHOULD: - Identify genuine strengths and opportunities - Propose solutions to overcome legitimate challenges - Highlight synergies with existing systems - Suggest optimizations that enhance value - Present realistic implementation pathways Remember: Being "for" means finding the BEST possible version of the idea IF it has merit, not blindly supporting bad ideas.""", "against": """CRITICAL PERSPECTIVE WITH RESPONSIBILITY You are tasked with critiquing this proposal, but with ESSENTIAL BOUNDARIES: MANDATORY FAIRNESS CONSTRAINTS: - You MUST NOT oppose genuinely excellent, common-sense ideas just to be contrarian - You MUST acknowledge when a proposal is fundamentally sound and well-conceived - You CANNOT give harmful advice or recommend against beneficial changes - If the idea is outstanding, say so clearly while offering constructive refinements WHEN TO MODERATE CRITICISM (MUST OVERRIDE STANCE): - If the proposal addresses critical user needs effectively - If it follows established best practices with good reason - If benefits clearly and substantially outweigh risks - If it's the obvious right solution to the problem YOUR CRITICAL ANALYSIS SHOULD: - Identify legitimate risks and failure modes - Point out overlooked complexities - Suggest more efficient alternatives - Highlight potential negative consequences - Question assumptions that may be flawed Remember: Being "against" means rigorous scrutiny to ensure quality, not undermining good ideas that deserve support.""", "neutral": """BALANCED PERSPECTIVE Provide objective analysis considering both positive and negative aspects. However, if there is overwhelming evidence that the proposal clearly leans toward being exceptionally good or particularly problematic, you MUST accurately reflect this reality. Being "balanced" means being truthful about the weight of evidence, not artificially creating 50/50 splits when the reality is 90/10. Your analysis should: - Present all significant pros and cons discovered - Weight them according to actual impact and likelihood - If evidence strongly favors one conclusion, clearly state this - Provide proportional coverage based on the strength of arguments - Help the questioner see the true balance of considerations Remember: Artificial balance that misrepresents reality is not helpful. True balance means accurate representation of the evidence, even when it strongly points in one direction.""", } stance_prompt = stance_prompts.get(stance, stance_prompts["neutral"]) return base_prompt.replace("{stance_prompt}", stance_prompt) def customize_workflow_response(self, response_data: dict, request) -> dict: """Customize response for consensus workflow.""" # Store model responses in the response for tracking if self.accumulated_responses: response_data["accumulated_responses"] = self.accumulated_responses # Add consensus-specific fields if request.step_number == 1: response_data["consensus_workflow_status"] = "initial_analysis_complete" elif request.step_number < request.total_steps - 1: response_data["consensus_workflow_status"] = "consulting_models" else: response_data["consensus_workflow_status"] = "ready_for_synthesis" # Customize metadata for consensus workflow self._customize_consensus_metadata(response_data, request) return response_data def _customize_consensus_metadata(self, response_data: dict, request) -> None: """ Customize metadata for consensus workflow to accurately reflect multi-model nature. The default workflow metadata shows the model running Agent's analysis steps, but consensus is a multi-model tool that consults different models. We need to provide accurate metadata that reflects this. """ if "metadata" not in response_data: response_data["metadata"] = {} metadata = response_data["metadata"] # Always preserve tool_name metadata["tool_name"] = self.get_name() if request.step_number == request.total_steps: # Final step - show comprehensive consensus metadata models_consulted = [] if self.models_to_consult: models_consulted = [f"{m['model']}:{m.get('stance', 'neutral')}" for m in self.models_to_consult] metadata.update( { "workflow_type": "multi_model_consensus", "models_consulted": models_consulted, "consensus_complete": True, "total_models": len(self.models_to_consult) if self.models_to_consult else 0, } ) # Remove the misleading single model metadata metadata.pop("model_used", None) metadata.pop("provider_used", None) else: # Intermediate steps - show consensus workflow in progress models_to_consult = [] if self.models_to_consult: models_to_consult = [f"{m['model']}:{m.get('stance', 'neutral')}" for m in self.models_to_consult] metadata.update( { "workflow_type": "multi_model_consensus", "models_to_consult": models_to_consult, "consultation_step": request.step_number, "total_consultation_steps": request.total_steps, } ) # Remove the misleading single model metadata that shows Agent's execution model # instead of the models being consulted metadata.pop("model_used", None) metadata.pop("provider_used", None) def _add_workflow_metadata(self, response_data: dict, arguments: dict[str, Any]) -> None: """ Override workflow metadata addition for consensus tool. The consensus tool doesn't use single model metadata because it's a multi-model workflow. Instead, we provide consensus-specific metadata that accurately reflects the models being consulted. """ # Initialize metadata if not present if "metadata" not in response_data: response_data["metadata"] = {} # Add basic tool metadata response_data["metadata"]["tool_name"] = self.get_name() # The consensus-specific metadata is already added by _customize_consensus_metadata # which is called from customize_workflow_response. We don't add the standard # single-model metadata (model_used, provider_used) because it's misleading # for a multi-model consensus workflow. logger.debug( f"[CONSENSUS_METADATA] {self.get_name()}: Using consensus-specific metadata instead of single-model metadata" ) def store_initial_issue(self, step_description: str): """Store initial prompt for model consultations.""" self.original_proposal = step_description self.initial_prompt = step_description # Keep for backward compatibility # Required abstract methods from BaseTool def get_request_model(self): """Return the consensus workflow-specific request model.""" return ConsensusRequest async def prepare_prompt(self, request) -> str: # noqa: ARG002 """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/debug.py ================================================ """ Debug tool - Systematic root cause analysis and debugging assistance This tool provides a structured workflow for investigating complex bugs and issues. It guides you through systematic investigation steps with forced pauses between each step to ensure thorough code examination before proceeding. The tool supports hypothesis evolution and expert analysis integration for comprehensive debugging. Key features: - Step-by-step investigation workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic conversation threading and history preservation - Expert analysis integration with external models - Support for visual debugging with image context - Confidence-based workflow optimization """ import logging from typing import TYPE_CHECKING, Any, Optional from pydantic import Field if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import DEBUG_ISSUE_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions matching original debug tool DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS = { "step": ( "Investigation step. Step 1: State issue+direction. " "Symptoms misleading; 'no bug' valid. Trace dependencies, verify hypotheses. " "Use relevant_files for code; this for text only." ), "step_number": "Current step index (starts at 1). Build upon previous steps.", "total_steps": ( "Estimated total steps needed to complete the investigation. Adjust as new findings emerge. " "IMPORTANT: When continuation_id is provided (continuing a previous conversation), set this to 1 as we're not starting a new multi-step investigation." ), "next_step_required": ( "True if you plan to continue the investigation with another step. False means root cause is known or investigation is complete. " "IMPORTANT: When continuation_id is provided (continuing a previous conversation), set this to False to immediately proceed with expert analysis." ), "findings": ( "Discoveries: clues, code/log evidence, disproven theories. Be specific. " "If no bug found, document clearly as valid." ), "files_checked": "All examined files (absolute paths), including ruled-out ones.", "relevant_files": "Files directly relevant to issue (absolute paths). Cause, trigger, or manifestation locations.", "relevant_context": "Methods/functions central to issue: 'Class.method' or 'function'. Focus on inputs/branching/state.", "hypothesis": ( "Concrete root cause theory from evidence. Can revise. " "Valid: 'No bug found - user misunderstanding' or 'Symptoms unrelated to code' if supported." ), "confidence": ( "Your confidence in the hypothesis: exploring (starting out), low (early idea), medium (some evidence), " "high (strong evidence), very_high (very strong evidence), almost_certain (nearly confirmed), " "certain (100% confidence - root cause and fix are both confirmed locally with no need for external validation). " "WARNING: Do NOT use 'certain' unless the issue can be fully resolved with a fix, use 'very_high' or 'almost_certain' instead when not 100% sure. " "Using 'certain' means you have ABSOLUTE confidence locally and PREVENTS external model validation." ), "images": "Optional screenshots/visuals clarifying issue (absolute paths).", } class DebugInvestigationRequest(WorkflowRequest): """Request model for debug investigation steps matching original debug tool exactly""" # Required fields for each investigation step step: str = Field(..., description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["relevant_context"] ) hypothesis: Optional[str] = Field(None, description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["hypothesis"]) confidence: Optional[str] = Field("low", description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["confidence"]) # Optional images for visual debugging images: Optional[list[str]] = Field(default=None, description=DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["images"]) # Override inherited fields to exclude them from schema (except model which needs to be available) temperature: Optional[float] = Field(default=None, exclude=True) thinking_mode: Optional[str] = Field(default=None, exclude=True) class DebugIssueTool(WorkflowTool): """ Debug tool for systematic root cause analysis and issue investigation. This tool implements a structured debugging workflow that guides users through methodical investigation steps, ensuring thorough code examination and evidence gathering before reaching conclusions. It supports complex debugging scenarios including race conditions, memory leaks, performance issues, and integration problems. """ def __init__(self): super().__init__() self.initial_issue = None def get_name(self) -> str: return "debug" def get_description(self) -> str: return ( "Performs systematic debugging and root cause analysis for any type of issue. " "Use for complex bugs, mysterious errors, performance issues, race conditions, memory leaks, and integration problems. " "Guides through structured investigation with hypothesis testing and expert analysis." ) def get_system_prompt(self) -> str: return DEBUG_ISSUE_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Debug requires deep analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the debug-specific request model.""" return DebugInvestigationRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with debug-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Debug-specific field overrides debug_field_overrides = { "step": { "type": "string", "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["relevant_files"], }, "confidence": { "type": "string", "enum": ["exploring", "low", "medium", "high", "very_high", "almost_certain", "certain"], "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["confidence"], }, "hypothesis": { "type": "string", "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["hypothesis"], }, "images": { "type": "array", "items": {"type": "string"}, "description": DEBUG_INVESTIGATION_FIELD_DESCRIPTIONS["images"], }, } # Use WorkflowSchemaBuilder with debug-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=debug_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each investigation phase.""" if step_number == 1: # Initial investigation tasks return [ "Search for code related to the reported issue or symptoms", "Examine relevant files and understand the current implementation", "Understand the project structure and locate relevant modules", "Identify how the affected functionality is supposed to work", ] elif confidence in ["exploring", "low"]: # Need deeper investigation return [ "Examine the specific files you've identified as relevant", "Trace method calls and data flow through the system", "Check for edge cases, boundary conditions, and assumptions in the code", "Look for related configuration, dependencies, or external factors", ] elif confidence in ["medium", "high", "very_high"]: # Close to root cause - need confirmation return [ "Examine the exact code sections where you believe the issue occurs", "Trace the execution path that leads to the failure", "Verify your hypothesis with concrete code evidence", "Check for any similar patterns elsewhere in the codebase", ] elif confidence == "almost_certain": # Almost certain - final verification before conclusion return [ "Finalize your root cause analysis with specific evidence", "Document the complete chain of causation from symptom to root cause", "Verify the minimal fix approach is correct", "Consider if expert analysis would provide additional insights", ] else: # General investigation needed return [ "Continue examining the code paths identified in your hypothesis", "Gather more evidence using appropriate investigation tools", "Test edge cases and boundary conditions", "Look for patterns that confirm or refute your theory", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Decide when to call external model based on investigation completeness. Don't call expert analysis if the CLI agent has certain confidence - trust their judgment. """ # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # Check if we have meaningful investigation data return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call matching original debug tool format.""" context_parts = [ f"=== ISSUE DESCRIPTION ===\n{self.initial_issue or 'Investigation initiated'}\n=== END DESCRIPTION ===" ] # Add special note if confidence is almost_certain if consolidated_findings.confidence == "almost_certain": context_parts.append( "\n=== IMPORTANT: ALMOST CERTAIN CONFIDENCE ===\n" "The agent has reached 'almost_certain' confidence but has NOT confirmed the bug with 100% certainty. " "Your role is to:\n" "1. Validate the agent's hypothesis and investigation\n" "2. Identify any missing evidence or overlooked aspects\n" "3. Provide additional insights that could confirm or refute the hypothesis\n" "4. Help finalize the root cause analysis with complete certainty\n" "=== END IMPORTANT ===" ) # Add investigation summary investigation_summary = self._build_investigation_summary(consolidated_findings) context_parts.append(f"\n=== AGENT'S INVESTIGATION FINDINGS ===\n{investigation_summary}\n=== END FINDINGS ===") # Add error context if available error_context = self._extract_error_context(consolidated_findings) if error_context: context_parts.append(f"\n=== ERROR CONTEXT/STACK TRACE ===\n{error_context}\n=== END CONTEXT ===") # Add relevant methods/functions if available if consolidated_findings.relevant_context: methods_text = "\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\n=== RELEVANT METHODS/FUNCTIONS ===\n{methods_text}\n=== END METHODS ===") # Add hypothesis evolution if available if consolidated_findings.hypotheses: hypotheses_text = "\n".join( f"Step {h['step']} ({h['confidence']} confidence): {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\n=== HYPOTHESIS EVOLUTION ===\n{hypotheses_text}\n=== END HYPOTHESES ===") # Add images if available if consolidated_findings.images: images_text = "\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append( f"\n=== VISUAL DEBUGGING INFORMATION ===\n{images_text}\n=== END VISUAL INFORMATION ===" ) # Add file content if we have relevant files if consolidated_findings.relevant_files: file_content, _ = self._prepare_file_content_for_prompt( list(consolidated_findings.relevant_files), None, "Essential debugging files" ) if file_content: context_parts.append( f"\n=== ESSENTIAL FILES FOR DEBUGGING ===\n{file_content}\n=== END ESSENTIAL FILES ===" ) return "\n".join(context_parts) def _build_investigation_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the investigation.""" summary_parts = [ "=== SYSTEMATIC INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Methods/functions involved: {len(consolidated_findings.relevant_context)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\n".join(summary_parts) def _extract_error_context(self, consolidated_findings) -> Optional[str]: """Extract error context from investigation findings.""" error_patterns = ["error", "exception", "stack trace", "traceback", "failure"] error_context_parts = [] for finding in consolidated_findings.findings: if any(pattern in finding.lower() for pattern in error_patterns): error_context_parts.append(finding) return "\n".join(error_context_parts) if error_context_parts else None def get_step_guidance(self, step_number: int, confidence: str, request) -> dict[str, Any]: """ Provide step-specific guidance matching original debug tool behavior. This method generates debug-specific guidance that's used by get_step_guidance_message(). """ # Generate the next steps instruction based on required actions required_actions = self.get_required_actions(step_number, confidence, request.findings, request.total_steps) if step_number == 1: next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first investigate " f"the codebase using appropriate tools. CRITICAL AWARENESS: The reported symptoms might be " f"caused by issues elsewhere in the code, not where symptoms appear. Also, after thorough " f"investigation, it's possible NO BUG EXISTS - the issue might be a misunderstanding or " f"user expectation mismatch. Search broadly, examine implementations, understand the logic flow. " f"Only call {self.get_name()} again AFTER gathering concrete evidence. When you call " f"{self.get_name()} next time, " f"use step_number: {step_number + 1} and report specific files examined and findings discovered." ) elif confidence in ["exploring", "low"]: next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. Based on your findings, you've identified potential areas " f"but need concrete evidence. MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nOnly call {self.get_name()} again with step_number: {step_number + 1} AFTER " + "completing these investigations." ) elif confidence in ["medium", "high", "very_high"]: next_steps = ( f"WAIT! Your hypothesis needs verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nREMEMBER: If you cannot find concrete evidence of a bug causing the reported symptoms, " f"'no bug found' is a valid conclusion. Consider suggesting discussion with your thought partner " f"or engineering assistant for clarification. Document findings with specific file:line references, " f"then call {self.get_name()} with step_number: {step_number + 1}." ) elif confidence == "almost_certain": next_steps = ( "ALMOST CERTAIN - Prepare for final analysis. REQUIRED ACTIONS:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + "\n\nIMPORTANT: You're almost certain about the root cause. If you have NOT found the bug with " "100% certainty, consider setting next_step_required=false to invoke expert analysis. The expert " "can validate your hypotheses and provide additional insights. If you ARE 100% certain and have " "identified the exact bug and fix, proceed to confidence='certain'. Otherwise, let expert analysis " "help finalize the investigation." ) else: next_steps = ( f"PAUSE INVESTIGATION. Before calling {self.get_name()} step {step_number + 1}, you MUST examine code. " + "Required: " + ", ".join(required_actions[:2]) + ". " + f"Your next {self.get_name()} call (step_number: {step_number + 1}) must include " f"NEW evidence from actual code examination, not just theories. If no bug evidence " f"is found, suggesting " f"collaboration with thought partner is valuable. NO recursive {self.get_name()} calls " f"without investigation work!" ) return {"next_steps": next_steps} # Hook method overrides for debug-specific behavior def prepare_step_data(self, request) -> dict: """ Prepare debug-specific step data for processing. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": [], # Debug tool doesn't use issues_found field "confidence": request.confidence, "hypothesis": request.hypothesis, "images": request.images or [], } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Debug tool skips expert analysis when agent has "certain" confidence. """ return request.confidence == "certain" and not request.next_step_required # Override inheritance hooks for debug-specific behavior def get_completion_status(self) -> str: """Debug tools use debug-specific status.""" return "certain_confidence_proceed_with_fix" def get_completion_data_key(self) -> str: """Debug uses 'complete_investigation' key.""" return "complete_investigation" def get_final_analysis_from_request(self, request): """Debug tools use 'hypothesis' field.""" return request.hypothesis def get_confidence_level(self, request) -> str: """Debug tools use 'certain' for high confidence.""" return "certain" def get_completion_message(self) -> str: """Debug-specific completion message.""" return ( "Investigation complete with CERTAIN confidence. You have identified the exact " "root cause and a minimal fix. MANDATORY: Present the user with the root cause analysis " "and IMMEDIATELY proceed with implementing the simple fix without requiring further " "consultation. Focus on the precise, minimal change needed." ) def get_skip_reason(self) -> str: """Debug-specific skip reason.""" return "Identified exact root cause with minimal fix requirement locally" def get_request_relevant_context(self, request) -> list: """Get relevant_context for debug tool.""" try: return request.relevant_context or [] except AttributeError: return [] def get_skip_expert_analysis_status(self) -> str: """Debug-specific expert analysis skip status.""" return "skipped_due_to_certain_confidence" def prepare_work_summary(self) -> str: """Debug-specific work summary.""" return self._build_investigation_summary(self.consolidated_findings) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Debug-specific completion message. Args: expert_analysis_used: True if expert analysis was successfully executed """ base_message = ( "INVESTIGATION IS COMPLETE. YOU MUST now summarize and present ALL key findings, confirmed " "hypotheses, and exact recommended fixes. Clearly identify the most likely root cause and " "provide concrete, actionable implementation guidance. Highlight affected code paths and display " "reasoning that led to this conclusion—make it easy for a developer to understand exactly where " "the problem lies. Where necessary, show cause-and-effect / bug-trace call graph." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Get additional guidance for handling expert analysis results in debug context. Returns: Additional guidance text for validating and using expert analysis findings """ return ( "IMPORTANT: Expert debugging analysis has been provided above. You MUST validate " "the expert's root cause analysis and proposed fixes against your own investigation. " "Ensure the expert's findings align with the evidence you've gathered and that the " "recommended solutions address the actual problem, not just symptoms. If the expert " "suggests a different root cause than you identified, carefully consider both perspectives " "and present a balanced assessment to the user." ) def get_step_guidance_message(self, request) -> str: """ Debug-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_step_guidance(request.step_number, request.confidence, request) return step_guidance["next_steps"] def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match original debug tool format. """ # Store initial issue on first step if request.step_number == 1: self.initial_issue = request.step # Convert generic status names to debug-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "investigation_in_progress", f"pause_for_{tool_name}": "pause_for_investigation", f"{tool_name}_required": "investigation_required", f"{tool_name}_complete": "investigation_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match debug tool if f"{tool_name}_status" in response_data: response_data["investigation_status"] = response_data.pop(f"{tool_name}_status") # Add debug-specific status fields response_data["investigation_status"]["hypotheses_formed"] = len(self.consolidated_findings.hypotheses) # Rename complete investigation data if f"complete_{tool_name}" in response_data: response_data["complete_investigation"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match original debug tool if f"{tool_name}_complete" in response_data: response_data["investigation_complete"] = response_data.pop(f"{tool_name}_complete") # Map the required flag to match original debug tool if f"{tool_name}_required" in response_data: response_data["investigation_required"] = response_data.pop(f"{tool_name}_required") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the debug-specific request model.""" return DebugInvestigationRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/docgen.py ================================================ """ Documentation Generation tool - Automated code documentation with complexity analysis This tool provides a structured workflow for adding comprehensive documentation to codebases. It guides you through systematic code analysis to generate modern documentation with: - Function/method parameter documentation - Big O complexity analysis - Call flow and dependency documentation - Inline comments for complex logic - Smart updating of existing documentation Key features: - Step-by-step documentation workflow with progress tracking - Context-aware file embedding (references during analysis, full content for documentation) - Automatic conversation threading and history preservation - Expert analysis integration with external models - Support for multiple programming languages and documentation styles - Configurable documentation features via parameters """ import logging from typing import TYPE_CHECKING, Any, Optional from pydantic import Field if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import DOCGEN_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for documentation generation DOCGEN_FIELD_DESCRIPTIONS = { "step": ( "Step 1 (Discovery): list every file that needs documentation and record the total. Do not write docs yet. " "Steps 2+: document exactly one file per step. Never change code logic; log bugs separately. Keep the counters accurate." ), "step_number": "Current documentation step (starts at 1).", "total_steps": "1 discovery step + one step per file documented (tracks via `total_files_to_document`).", "next_step_required": "True while more files still need documentation; False once everything is complete.", "findings": "Summarize documentation gaps, complexity, call flows, and well-documented areas. Stop and report immediately if you uncover a bug.", "relevant_files": "Absolute paths for the file(s) you are documenting this step—stick to a single file per step.", "relevant_context": "Functions or methods needing documentation (e.g. 'Class.method', 'function_name'), especially complex or user-facing areas.", "num_files_documented": "Count of files finished so far. Increment only when a file is fully documented.", "total_files_to_document": "Total files identified in discovery; completion requires matching this count.", "document_complexity": "Include algorithmic complexity (Big O) analysis when True (default).", "document_flow": "Include call flow/dependency notes when True (default).", "update_existing": "True (default) to polish inaccurate or outdated docs instead of leaving them untouched.", "comments_on_complex_logic": "True (default) to add inline comments around non-obvious logic.", } class DocgenRequest(WorkflowRequest): """Request model for documentation generation steps""" # Required workflow fields step: str = Field(..., description=DOCGEN_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=DOCGEN_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=DOCGEN_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=DOCGEN_FIELD_DESCRIPTIONS["next_step_required"]) # Documentation analysis tracking fields findings: str = Field(..., description=DOCGEN_FIELD_DESCRIPTIONS["findings"]) relevant_files: list[str] = Field(default_factory=list, description=DOCGEN_FIELD_DESCRIPTIONS["relevant_files"]) relevant_context: list[str] = Field(default_factory=list, description=DOCGEN_FIELD_DESCRIPTIONS["relevant_context"]) # Critical completion tracking counters num_files_documented: int = Field(0, description=DOCGEN_FIELD_DESCRIPTIONS["num_files_documented"]) total_files_to_document: int = Field(0, description=DOCGEN_FIELD_DESCRIPTIONS["total_files_to_document"]) # Documentation generation configuration parameters document_complexity: Optional[bool] = Field(True, description=DOCGEN_FIELD_DESCRIPTIONS["document_complexity"]) document_flow: Optional[bool] = Field(True, description=DOCGEN_FIELD_DESCRIPTIONS["document_flow"]) update_existing: Optional[bool] = Field(True, description=DOCGEN_FIELD_DESCRIPTIONS["update_existing"]) comments_on_complex_logic: Optional[bool] = Field( True, description=DOCGEN_FIELD_DESCRIPTIONS["comments_on_complex_logic"] ) class DocgenTool(WorkflowTool): """ Documentation generation tool for automated code documentation with complexity analysis. This tool implements a structured documentation workflow that guides users through methodical code analysis to generate comprehensive documentation including: - Function/method signatures and parameter descriptions - Algorithmic complexity (Big O) analysis - Call flow and dependency documentation - Inline comments for complex logic - Modern documentation style appropriate for the language/platform """ def __init__(self): super().__init__() self.initial_request = None def get_name(self) -> str: return "docgen" def get_description(self) -> str: return ( "Generates comprehensive code documentation with systematic analysis of functions, classes, and complexity. " "Use for documentation generation, code analysis, complexity assessment, and API documentation. " "Analyzes code structure and patterns to create thorough documentation." ) def get_system_prompt(self) -> str: return DOCGEN_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Docgen requires analytical and reasoning capabilities""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def requires_model(self) -> bool: """ Docgen tool doesn't require model resolution at the MCP boundary. The docgen tool is a self-contained workflow tool that guides the CLI agent through systematic documentation generation without calling external AI models. Returns: bool: False - docgen doesn't need external AI model access """ return False def requires_expert_analysis(self) -> bool: """Docgen is self-contained and doesn't need expert analysis.""" return False def get_workflow_request_model(self): """Return the docgen-specific request model.""" return DocgenRequest def get_tool_fields(self) -> dict[str, dict[str, Any]]: """Return the tool-specific fields for docgen.""" return { "document_complexity": { "type": "boolean", "default": True, "description": DOCGEN_FIELD_DESCRIPTIONS["document_complexity"], }, "document_flow": { "type": "boolean", "default": True, "description": DOCGEN_FIELD_DESCRIPTIONS["document_flow"], }, "update_existing": { "type": "boolean", "default": True, "description": DOCGEN_FIELD_DESCRIPTIONS["update_existing"], }, "comments_on_complex_logic": { "type": "boolean", "default": True, "description": DOCGEN_FIELD_DESCRIPTIONS["comments_on_complex_logic"], }, "num_files_documented": { "type": "integer", "default": 0, "minimum": 0, "description": DOCGEN_FIELD_DESCRIPTIONS["num_files_documented"], }, "total_files_to_document": { "type": "integer", "default": 0, "minimum": 0, "description": DOCGEN_FIELD_DESCRIPTIONS["total_files_to_document"], }, } def get_required_fields(self) -> list[str]: """Return additional required fields beyond the standard workflow requirements.""" return [ "document_complexity", "document_flow", "update_existing", "comments_on_complex_logic", "num_files_documented", "total_files_to_document", ] def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with field exclusions.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Exclude workflow fields that documentation generation doesn't need excluded_workflow_fields = [ "confidence", # Documentation doesn't use confidence levels "hypothesis", # Documentation doesn't use hypothesis "files_checked", # Documentation uses doc_files and doc_methods instead for better tracking ] # Exclude common fields that documentation generation doesn't need excluded_common_fields = [ "model", # Documentation doesn't need external model selection "temperature", # Documentation doesn't need temperature control "thinking_mode", # Documentation doesn't need thinking mode "images", # Documentation doesn't use images ] return WorkflowSchemaBuilder.build_schema( tool_specific_fields=self.get_tool_fields(), required_fields=self.get_required_fields(), # Include docgen-specific required fields model_field_schema=None, # Exclude model field - docgen doesn't need external model selection auto_mode=False, # Force non-auto mode to prevent model field addition tool_name=self.get_name(), excluded_workflow_fields=excluded_workflow_fields, excluded_common_fields=excluded_common_fields, ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for comprehensive documentation analysis with step-by-step file focus.""" if step_number == 1: # Initial discovery ONLY - no documentation yet return [ "CRITICAL: DO NOT ALTER ANY CODE LOGIC! Only add documentation (docstrings, comments)", "Discover ALL files in the current directory (not nested) that need documentation", "COUNT the exact number of files that need documentation", "LIST all the files you found that need documentation by name", "IDENTIFY the programming language(s) to use MODERN documentation style (/// for Objective-C, /** */ for Java/JavaScript, etc.)", "DO NOT start documenting any files yet - this is discovery phase only", "Report the total count and file list clearly to the user", "IMMEDIATELY call docgen step 2 after discovery to begin documentation phase", "WHEN CALLING DOCGEN step 2: Set total_files_to_document to the exact count you found", "WHEN CALLING DOCGEN step 2: Set num_files_documented to 0 (haven't started yet)", ] elif step_number == 2: # Start documentation phase with first file return [ "CRITICAL: DO NOT ALTER ANY CODE LOGIC! Only add documentation (docstrings, comments)", "Choose the FIRST file from your discovered list to start documentation", "For the chosen file: identify ALL functions, classes, and methods within it", 'USE MODERN documentation style for the programming language (/// for Objective-C, /** */ for Java/JavaScript, """ for Python, etc.)', "Document ALL functions/methods in the chosen file - don't skip any - DOCUMENTATION ONLY", "When file is 100% documented, increment num_files_documented from 0 to 1", "Note any dependencies this file has (what it imports/calls) and what calls into it", "CRITICAL: If you find ANY bugs/logic errors, STOP documenting and report to user immediately", "Report which specific functions you documented in this step for accountability", "Report progress: num_files_documented (1) out of total_files_to_document", ] elif step_number <= 4: # Continue with focused file-by-file approach return [ "CRITICAL: DO NOT ALTER ANY CODE LOGIC! Only add documentation (docstrings, comments)", "Choose the NEXT undocumented file from your discovered list", "For the chosen file: identify ALL functions, classes, and methods within it", "USE MODERN documentation style for the programming language (NEVER use legacy /* */ style for languages with modern alternatives)", "Document ALL functions/methods in the chosen file - don't skip any - DOCUMENTATION ONLY", "When file is 100% documented, increment num_files_documented by 1", "Verify that EVERY function in the current file has proper documentation (no skipping)", "CRITICAL: If you find ANY bugs/logic errors, STOP documenting and report to user immediately", "Report specific function names you documented for verification", "Report progress: current num_files_documented out of total_files_to_document", ] else: # Continue systematic file-by-file coverage return [ "CRITICAL: DO NOT ALTER ANY CODE LOGIC! Only add documentation (docstrings, comments)", "Check counters: num_files_documented vs total_files_to_document", "If num_files_documented < total_files_to_document: choose NEXT undocumented file", "USE MODERN documentation style appropriate for each programming language (NEVER legacy styles)", "Document every function, method, and class in current file with no exceptions", "When file is 100% documented, increment num_files_documented by 1", "CRITICAL: If you find ANY bugs/logic errors, STOP documenting and report to user immediately", "Report progress: current num_files_documented out of total_files_to_document", "If num_files_documented < total_files_to_document: RESTART docgen with next step", "ONLY set next_step_required=false when num_files_documented equals total_files_to_document", "For nested dependencies: check if functions call into subdirectories and document those too", "CRITICAL: If ANY bugs/logic errors were found, STOP and ask user before proceeding", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """Docgen is self-contained and doesn't need expert analysis.""" return False def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Docgen doesn't use expert analysis.""" return "" def get_step_guidance(self, step_number: int, confidence: str, request) -> dict[str, Any]: """ Provide step-specific guidance for documentation generation workflow. This method generates docgen-specific guidance used by get_step_guidance_message(). """ # Generate the next steps instruction based on required actions # Calculate dynamic total_steps based on files to document total_files_to_document = self.get_request_total_files_to_document(request) calculated_total_steps = 1 + total_files_to_document if total_files_to_document > 0 else request.total_steps required_actions = self.get_required_actions(step_number, confidence, request.findings, calculated_total_steps) if step_number == 1: next_steps = ( f"DISCOVERY PHASE ONLY - DO NOT START DOCUMENTING YET!\n" f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first perform " f"FILE DISCOVERY step by step. DO NOT DOCUMENT ANYTHING YET. " f"MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nCRITICAL: When you call {self.get_name()} step 2, set total_files_to_document to the exact count " f"of files needing documentation and set num_files_documented to 0 (haven't started documenting yet). " f"Your total_steps will be automatically calculated as 1 (discovery) + number of files to document. " f"Step 2 will BEGIN the documentation phase. Report the count clearly and then IMMEDIATELY " f"proceed to call {self.get_name()} step 2 to start documenting the first file." ) elif step_number == 2: next_steps = ( f"DOCUMENTATION PHASE BEGINS! ABSOLUTE RULE: DO NOT ALTER ANY CODE LOGIC! DOCUMENTATION ONLY!\n" f"START FILE-BY-FILE APPROACH! Focus on ONE file until 100% complete. " f"MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nREPORT your progress: which specific functions did you document? Update num_files_documented from 0 to 1 when first file complete. " f"REPORT counters: current num_files_documented out of total_files_to_document. " f"CRITICAL: If you found ANY bugs/logic errors, STOP documenting and ask user what to do before continuing. " f"Do NOT move to a new file until the current one is completely documented. " f"When ready for step {step_number + 1}, report completed work with updated counters." ) elif step_number <= 4: next_steps = ( f"ABSOLUTE RULE: DO NOT ALTER ANY CODE LOGIC! DOCUMENTATION ONLY!\n" f"CONTINUE FILE-BY-FILE APPROACH! Focus on ONE file until 100% complete. " f"MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nREPORT your progress: which specific functions did you document? Update num_files_documented when file complete. " f"REPORT counters: current num_files_documented out of total_files_to_document. " f"CRITICAL: If you found ANY bugs/logic errors, STOP documenting and ask user what to do before continuing. " f"Do NOT move to a new file until the current one is completely documented. " f"When ready for step {step_number + 1}, report completed work with updated counters." ) else: next_steps = ( f"ABSOLUTE RULE: DO NOT ALTER ANY CODE LOGIC! DOCUMENTATION ONLY!\n" f"CRITICAL: Check if MORE FILES need documentation before finishing! " f"REQUIRED ACTIONS before calling {self.get_name()} step {step_number + 1}:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nREPORT which functions you documented and update num_files_documented when file complete. " f"CHECK: If num_files_documented < total_files_to_document, RESTART {self.get_name()} with next step! " f"CRITICAL: Only set next_step_required=false when num_files_documented equals total_files_to_document! " f"REPORT counters: current num_files_documented out of total_files_to_document. " f"CRITICAL: If ANY bugs/logic errors were found during documentation, STOP and ask user before proceeding. " f"NO recursive {self.get_name()} calls without actual documentation work!" ) return {"next_steps": next_steps} # Hook method overrides for docgen-specific behavior async def handle_work_completion(self, response_data: dict, request, arguments: dict) -> dict: """ Override work completion to enforce counter validation. The docgen tool MUST complete ALL files before finishing. If counters don't match, force continuation regardless of next_step_required setting. """ # CRITICAL VALIDATION: Check if all files have been documented using proper inheritance hooks num_files_documented = self.get_request_num_files_documented(request) total_files_to_document = self.get_request_total_files_to_document(request) if num_files_documented < total_files_to_document: # Counters don't match - force continuation! logger.warning( f"Docgen stopping early: {num_files_documented} < {total_files_to_document}. " f"Forcing continuation to document remaining files." ) # Override to continuation mode response_data["status"] = "documentation_analysis_required" response_data[f"pause_for_{self.get_name()}"] = True response_data["next_steps"] = ( f"CRITICAL ERROR: You attempted to finish documentation with only {num_files_documented} " f"out of {total_files_to_document} files documented! You MUST continue documenting " f"the remaining {total_files_to_document - num_files_documented} files. " f"Call {self.get_name()} again with step {request.step_number + 1} and continue documentation " f"of the next undocumented file. DO NOT set next_step_required=false until ALL files are documented!" ) return response_data # If counters match, proceed with normal completion return await super().handle_work_completion(response_data, request, arguments) def prepare_step_data(self, request) -> dict: """ Prepare docgen-specific step data for processing. Calculates total_steps dynamically based on number of files to document: - Step 1: Discovery phase - Steps 2+: One step per file to document """ # Calculate dynamic total_steps based on files to document total_files_to_document = self.get_request_total_files_to_document(request) if total_files_to_document > 0: # Discovery step (1) + one step per file calculated_total_steps = 1 + total_files_to_document else: # Fallback to request total_steps if no file count available calculated_total_steps = request.total_steps step_data = { "step": request.step, "step_number": request.step_number, "total_steps": calculated_total_steps, # Use calculated value "findings": request.findings, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "num_files_documented": request.num_files_documented, "total_files_to_document": request.total_files_to_document, "issues_found": [], # Docgen uses this for documentation gaps "confidence": "medium", # Default confidence for docgen "hypothesis": "systematic_documentation_needed", # Default hypothesis "images": [], # Docgen doesn't typically use images # CRITICAL: Include documentation configuration parameters so the model can see them "document_complexity": request.document_complexity, "document_flow": request.document_flow, "update_existing": request.update_existing, "comments_on_complex_logic": request.comments_on_complex_logic, } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Docgen tool skips expert analysis when the CLI agent has "certain" confidence. """ return request.confidence == "certain" and not request.next_step_required # Override inheritance hooks for docgen-specific behavior def get_completion_status(self) -> str: """Docgen tools use docgen-specific status.""" return "documentation_analysis_complete" def get_completion_data_key(self) -> str: """Docgen uses 'complete_documentation_analysis' key.""" return "complete_documentation_analysis" def get_final_analysis_from_request(self, request): """Docgen tools use 'hypothesis' field for documentation strategy.""" return request.hypothesis def get_confidence_level(self, request) -> str: """Docgen tools use 'certain' for high confidence.""" return request.confidence or "high" def get_completion_message(self) -> str: """Docgen-specific completion message.""" return ( "Documentation analysis complete with high confidence. You have identified the comprehensive " "documentation needs and strategy. MANDATORY: Present the user with the documentation plan " "and IMMEDIATELY proceed with implementing the documentation without requiring further " "consultation. Focus on the precise documentation improvements needed." ) def get_skip_reason(self) -> str: """Docgen-specific skip reason.""" return "Completed comprehensive documentation analysis locally" def get_request_relevant_context(self, request) -> list: """Get relevant_context for docgen tool.""" try: return request.relevant_context or [] except AttributeError: return [] def get_request_num_files_documented(self, request) -> int: """Get num_files_documented from request. Override for custom handling.""" try: return request.num_files_documented or 0 except AttributeError: return 0 def get_request_total_files_to_document(self, request) -> int: """Get total_files_to_document from request. Override for custom handling.""" try: return request.total_files_to_document or 0 except AttributeError: return 0 def get_skip_expert_analysis_status(self) -> str: """Docgen-specific expert analysis skip status.""" return "skipped_due_to_complete_analysis" def prepare_work_summary(self) -> str: """Docgen-specific work summary.""" try: return f"Completed {len(self.work_history)} documentation analysis steps" except AttributeError: return "Completed documentation analysis" def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Docgen-specific completion message. """ return ( "DOCUMENTATION ANALYSIS IS COMPLETE FOR ALL FILES (num_files_documented equals total_files_to_document). " "MANDATORY FINAL VERIFICATION: Before presenting your summary, you MUST perform a final verification scan. " "Read through EVERY file you documented and check EVERY function, method, class, and property to confirm " "it has proper documentation including complexity analysis and call flow information. If ANY items lack " "documentation, document them immediately before finishing. " "THEN present a clear summary showing: 1) Final counters: num_files_documented out of total_files_to_document, " "2) Complete accountability list of ALL files you documented with verification status, " "3) Detailed list of EVERY function/method you documented in each file (proving complete coverage), " "4) Any dependency relationships you discovered between files, 5) Recommended documentation improvements with concrete examples including " "complexity analysis and call flow information. 6) **CRITICAL**: List any bugs or logic issues you found " "during documentation but did NOT fix - present these to the user and ask what they'd like to do about them. " "Make it easy for a developer to see the complete documentation status across the entire codebase with full accountability." ) def get_step_guidance_message(self, request) -> str: """ Docgen-specific step guidance with detailed analysis instructions. """ step_guidance = self.get_step_guidance(request.step_number, request.confidence, request) return step_guidance["next_steps"] def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match docgen tool format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Convert generic status names to docgen-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "documentation_analysis_in_progress", f"pause_for_{tool_name}": "pause_for_documentation_analysis", f"{tool_name}_required": "documentation_analysis_required", f"{tool_name}_complete": "documentation_analysis_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match docgen tool if f"{tool_name}_status" in response_data: response_data["documentation_analysis_status"] = response_data.pop(f"{tool_name}_status") # Add docgen-specific status fields response_data["documentation_analysis_status"]["documentation_strategies"] = len( self.consolidated_findings.hypotheses ) # Rename complete documentation analysis data if f"complete_{tool_name}" in response_data: response_data["complete_documentation_analysis"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match docgen tool if f"{tool_name}_complete" in response_data: response_data["documentation_analysis_complete"] = response_data.pop(f"{tool_name}_complete") # Map the required flag to match docgen tool if f"{tool_name}_required" in response_data: response_data["documentation_analysis_required"] = response_data.pop(f"{tool_name}_required") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the docgen-specific request model.""" return DocgenRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/listmodels.py ================================================ """ List Models Tool - Display all available models organized by provider This tool provides a comprehensive view of all AI models available in the system, organized by their provider (Gemini, OpenAI, X.AI, OpenRouter, Custom). It shows which providers are configured and what models can be used. """ import logging from typing import Any, Optional from mcp.types import TextContent from providers.registries.custom import CustomEndpointModelRegistry from providers.registries.openrouter import OpenRouterModelRegistry from tools.models import ToolModelCategory, ToolOutput from tools.shared.base_models import ToolRequest from tools.shared.base_tool import BaseTool from utils.env import get_env logger = logging.getLogger(__name__) class ListModelsTool(BaseTool): """ Tool for listing all available AI models organized by provider. This tool helps users understand: - Which providers are configured (have API keys) - What models are available from each provider - Model aliases and their full names - Context window sizes and capabilities """ def get_name(self) -> str: return "listmodels" def get_description(self) -> str: return "Shows which AI model providers are configured, available model names, their aliases and capabilities." def get_input_schema(self) -> dict[str, Any]: """Return the JSON schema for the tool's input""" return { "type": "object", "properties": {}, "required": [], "additionalProperties": False, } def get_annotations(self) -> Optional[dict[str, Any]]: """Return tool annotations indicating this is a read-only tool""" return {"readOnlyHint": True} def get_system_prompt(self) -> str: """No AI model needed for this tool""" return "" def get_request_model(self): """Return the Pydantic model for request validation.""" return ToolRequest def requires_model(self) -> bool: return False async def prepare_prompt(self, request: ToolRequest) -> str: """Not used for this utility tool""" return "" def format_response(self, response: str, request: ToolRequest, model_info: Optional[dict] = None) -> str: """Not used for this utility tool""" return response async def execute(self, arguments: dict[str, Any]) -> list[TextContent]: """ List all available models organized by provider. This overrides the base class execute to provide direct output without AI model calls. Args: arguments: Standard tool arguments (none required) Returns: Formatted list of models by provider """ from providers.registry import ModelProviderRegistry from providers.shared import ProviderType from utils.model_restrictions import get_restriction_service output_lines = ["# Available AI Models\n"] restriction_service = get_restriction_service() restricted_models_by_provider: dict[ProviderType, list[str]] = {} if restriction_service: restricted_map = ModelProviderRegistry.get_available_models(respect_restrictions=True) for model_name, provider_type in restricted_map.items(): restricted_models_by_provider.setdefault(provider_type, []).append(model_name) # Map provider types to friendly names and their models provider_info = { ProviderType.GOOGLE: {"name": "Google Gemini", "env_key": "GEMINI_API_KEY"}, ProviderType.OPENAI: {"name": "OpenAI", "env_key": "OPENAI_API_KEY"}, ProviderType.AZURE: {"name": "Azure OpenAI", "env_key": "AZURE_OPENAI_API_KEY"}, ProviderType.XAI: {"name": "X.AI (Grok)", "env_key": "XAI_API_KEY"}, ProviderType.DIAL: {"name": "AI DIAL", "env_key": "DIAL_API_KEY"}, } def format_model_entry(provider, display_name: str) -> list[str]: try: capabilities = provider.get_capabilities(display_name) except ValueError: return [f"- `{display_name}` *(not recognized by provider)*"] canonical = capabilities.model_name if canonical.lower() == display_name.lower(): header = f"- `{canonical}`" else: header = f"- `{display_name}` → `{canonical}`" try: context_value = capabilities.context_window or 0 except AttributeError: context_value = 0 try: context_value = int(context_value) except (TypeError, ValueError): context_value = 0 if context_value >= 1_000_000: context_str = f"{context_value // 1_000_000}M context" elif context_value >= 1_000: context_str = f"{context_value // 1_000}K context" elif context_value > 0: context_str = f"{context_value} context" else: context_str = "unknown context" try: description = capabilities.description or "No description available" except AttributeError: description = "No description available" lines = [header, f" - {context_str}", f" - {description}"] if capabilities.allow_code_generation: lines.append(" - Supports structured code generation") return lines # Check each native provider type for provider_type, info in provider_info.items(): # Check if provider is enabled provider = ModelProviderRegistry.get_provider(provider_type) is_configured = provider is not None output_lines.append(f"## {info['name']} {'✅' if is_configured else '❌'}") if is_configured: output_lines.append("**Status**: Configured and available") has_restrictions = bool(restriction_service and restriction_service.has_restrictions(provider_type)) if has_restrictions: restricted_names = sorted(set(restricted_models_by_provider.get(provider_type, []))) if restricted_names: output_lines.append("\n**Models (policy restricted)**:") for model_name in restricted_names: output_lines.extend(format_model_entry(provider, model_name)) else: output_lines.append("\n*No models are currently allowed by restriction policy.*") else: output_lines.append("\n**Models**:") aliases = [] for model_name, capabilities in provider.get_capabilities_by_rank(): try: description = capabilities.description or "No description available" except AttributeError: description = "No description available" try: context_window = capabilities.context_window or 0 except AttributeError: context_window = 0 if context_window >= 1_000_000: context_str = f"{context_window // 1_000_000}M context" elif context_window >= 1_000: context_str = f"{context_window // 1_000}K context" else: context_str = f"{context_window} context" if context_window > 0 else "unknown context" output_lines.append(f"- `{model_name}` - {context_str}") output_lines.append(f" - {description}") if capabilities.allow_code_generation: output_lines.append(" - Supports structured code generation") for alias in capabilities.aliases or []: if alias != model_name: aliases.append(f"- `{alias}` → `{model_name}`") if aliases: output_lines.append("\n**Aliases**:") output_lines.extend(sorted(aliases)) else: output_lines.append(f"**Status**: Not configured (set {info['env_key']})") output_lines.append("") # Check OpenRouter openrouter_key = get_env("OPENROUTER_API_KEY") is_openrouter_configured = openrouter_key and openrouter_key != "your_openrouter_api_key_here" output_lines.append(f"## OpenRouter {'✅' if is_openrouter_configured else '❌'}") if is_openrouter_configured: output_lines.append("**Status**: Configured and available") output_lines.append("**Description**: Access to multiple cloud AI providers via unified API") try: provider = ModelProviderRegistry.get_provider(ProviderType.OPENROUTER) if provider: registry = OpenRouterModelRegistry() def _format_context(tokens: int) -> str: if not tokens: return "?" if tokens >= 1_000_000: return f"{tokens // 1_000_000}M" if tokens >= 1_000: return f"{tokens // 1_000}K" return str(tokens) has_restrictions = bool( restriction_service and restriction_service.has_restrictions(ProviderType.OPENROUTER) ) if has_restrictions: restricted_names = sorted(set(restricted_models_by_provider.get(ProviderType.OPENROUTER, []))) output_lines.append("\n**Models (policy restricted)**:") if restricted_names: for model_name in restricted_names: try: caps = provider.get_capabilities(model_name) except ValueError: output_lines.append(f"- `{model_name}` *(not recognized by provider)*") continue context_value = int(caps.context_window or 0) context_str = _format_context(context_value) suffix_parts = [f"{context_str} context"] if caps.supports_extended_thinking: suffix_parts.append("thinking") suffix = ", ".join(suffix_parts) arrow = "" if caps.model_name.lower() != model_name.lower(): arrow = f" → `{caps.model_name}`" score = caps.get_effective_capability_rank() output_lines.append(f"- `{model_name}`{arrow} (score {score}, {suffix})") allowed_set = restriction_service.get_allowed_models(ProviderType.OPENROUTER) or set() if allowed_set: output_lines.append( f"\n*OpenRouter models restricted by OPENROUTER_ALLOWED_MODELS: {', '.join(sorted(allowed_set))}*" ) else: output_lines.append("- *No models allowed by current restriction policy.*") else: available_models = provider.list_models(respect_restrictions=True) providers_models: dict[str, list[tuple[int, str, Optional[Any]]]] = {} for model_name in available_models: config = registry.resolve(model_name) provider_name = "other" if config and "/" in config.model_name: provider_name = config.model_name.split("/")[0] elif "/" in model_name: provider_name = model_name.split("/")[0] providers_models.setdefault(provider_name, []) rank = config.get_effective_capability_rank() if config else 0 providers_models[provider_name].append((rank, model_name, config)) output_lines.append("\n**Available Models**:") for provider_name, models in sorted(providers_models.items()): output_lines.append(f"\n*{provider_name.title()}:*") for rank, alias, config in sorted(models, key=lambda item: (-item[0], item[1])): if config: context_str = _format_context(getattr(config, "context_window", 0)) suffix_parts = [f"{context_str} context"] if getattr(config, "supports_extended_thinking", False): suffix_parts.append("thinking") suffix = ", ".join(suffix_parts) arrow = "" if config.model_name.lower() != alias.lower(): arrow = f" → `{config.model_name}`" output_lines.append(f"- `{alias}`{arrow} (score {rank}, {suffix})") else: output_lines.append(f"- `{alias}` (score {rank})") else: output_lines.append("**Error**: Could not load OpenRouter provider") except Exception as e: logger.exception("Error listing OpenRouter models: %s", e) output_lines.append(f"**Error loading models**: {str(e)}") else: output_lines.append("**Status**: Not configured (set OPENROUTER_API_KEY)") output_lines.append("**Note**: Provides access to GPT-5, O3, Mistral, and many more") output_lines.append("") # Check Custom API custom_url = get_env("CUSTOM_API_URL") output_lines.append(f"## Custom/Local API {'✅' if custom_url else '❌'}") if custom_url: output_lines.append("**Status**: Configured and available") output_lines.append(f"**Endpoint**: {custom_url}") output_lines.append("**Description**: Local models via Ollama, vLLM, LM Studio, etc.") try: registry = CustomEndpointModelRegistry() custom_models = [] for alias in registry.list_aliases(): config = registry.resolve(alias) if config: custom_models.append((alias, config)) if custom_models: output_lines.append("\n**Custom Models**:") for alias, config in custom_models: context_str = f"{config.context_window // 1000}K" if config.context_window else "?" output_lines.append(f"- `{alias}` → `{config.model_name}` ({context_str} context)") if config.description: output_lines.append(f" - {config.description}") except Exception as e: output_lines.append(f"**Error loading custom models**: {str(e)}") else: output_lines.append("**Status**: Not configured (set CUSTOM_API_URL)") output_lines.append("**Example**: CUSTOM_API_URL=http://localhost:11434 (for Ollama)") output_lines.append("") # Add summary output_lines.append("## Summary") # Count configured providers configured_count = sum( [ 1 for provider_type, info in provider_info.items() if ModelProviderRegistry.get_provider(provider_type) is not None ] ) if is_openrouter_configured: configured_count += 1 if custom_url: configured_count += 1 output_lines.append(f"**Configured Providers**: {configured_count}") # Get total available models try: from providers.registry import ModelProviderRegistry # Get all available models respecting restrictions available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) total_models = len(available_models) output_lines.append(f"**Total Available Models**: {total_models}") except Exception as e: logger.warning(f"Error getting total available models: {e}") # Add usage tips output_lines.append("\n**Usage Tips**:") output_lines.append("- Use model aliases (e.g., 'flash', 'gpt5', 'opus') for convenience") output_lines.append("- In auto mode, the CLI Agent will select the best model for each task") output_lines.append("- Custom models are only available when CUSTOM_API_URL is set") output_lines.append("- OpenRouter provides access to many cloud models with one API key") # Format output content = "\n".join(output_lines) tool_output = ToolOutput( status="success", content=content, content_type="text", metadata={ "tool_name": self.name, "configured_providers": configured_count, }, ) return [TextContent(type="text", text=tool_output.model_dump_json())] def get_model_category(self) -> ToolModelCategory: """Return the model category for this tool.""" return ToolModelCategory.FAST_RESPONSE # Simple listing, no AI needed ================================================ FILE: tools/models.py ================================================ """ Data models for tool responses and interactions """ from enum import Enum from typing import Any, Literal, Optional from pydantic import BaseModel, Field class ToolModelCategory(Enum): """Categories for tool model selection based on requirements.""" EXTENDED_REASONING = "extended_reasoning" # Requires deep thinking capabilities FAST_RESPONSE = "fast_response" # Speed and cost efficiency preferred BALANCED = "balanced" # Balance of capability and performance class ContinuationOffer(BaseModel): """Offer for CLI agent to continue conversation when Gemini doesn't ask follow-up""" continuation_id: str = Field( ..., description="Thread continuation ID for multi-turn conversations across different tools" ) note: str = Field(..., description="Message explaining continuation opportunity to CLI agent") remaining_turns: int = Field(..., description="Number of conversation turns remaining") class ToolOutput(BaseModel): """Standardized output format for all tools""" status: Literal[ "success", "error", "files_required_to_continue", "full_codereview_required", "focused_review_required", "test_sample_needed", "more_tests_required", "refactor_analysis_complete", "trace_complete", "resend_prompt", "code_too_large", "continuation_available", "no_bug_found", ] = "success" content: Optional[str] = Field(None, description="The main content/response from the tool") content_type: Literal["text", "markdown", "json"] = "text" metadata: Optional[dict[str, Any]] = Field(default_factory=dict) continuation_offer: Optional[ContinuationOffer] = Field( None, description="Optional offer for Agent to continue conversation" ) class FilesNeededRequest(BaseModel): """Request for missing files / code to continue""" status: Literal["files_required_to_continue"] = "files_required_to_continue" mandatory_instructions: str = Field(..., description="Critical instructions for Agent regarding required context") files_needed: Optional[list[str]] = Field( default_factory=list, description="Specific files that are needed for analysis" ) suggested_next_action: Optional[dict[str, Any]] = Field( None, description="Suggested tool call with parameters after getting clarification", ) class FullCodereviewRequired(BaseModel): """Request for full code review when scope is too large for quick review""" status: Literal["full_codereview_required"] = "full_codereview_required" important: Optional[str] = Field(None, description="Important message about escalation") reason: Optional[str] = Field(None, description="Reason why full review is needed") class FocusedReviewRequired(BaseModel): """Request for Agent to provide smaller, focused subsets of code for review""" status: Literal["focused_review_required"] = "focused_review_required" reason: str = Field(..., description="Why the current scope is too large for effective review") suggestion: str = Field( ..., description="Suggested approach for breaking down the review into smaller, focused parts" ) class TestSampleNeeded(BaseModel): """Request for additional test samples to determine testing framework""" status: Literal["test_sample_needed"] = "test_sample_needed" reason: str = Field(..., description="Reason why additional test samples are required") class MoreTestsRequired(BaseModel): """Request for continuation to generate additional tests""" status: Literal["more_tests_required"] = "more_tests_required" pending_tests: str = Field(..., description="List of pending tests to be generated") class RefactorOpportunity(BaseModel): """A single refactoring opportunity with precise targeting information""" id: str = Field(..., description="Unique identifier for this refactoring opportunity") type: Literal["decompose", "codesmells", "modernize", "organization"] = Field( ..., description="Type of refactoring" ) severity: Literal["critical", "high", "medium", "low"] = Field(..., description="Severity level") file: str = Field(..., description="Absolute path to the file") start_line: int = Field(..., description="Starting line number") end_line: int = Field(..., description="Ending line number") context_start_text: str = Field(..., description="Exact text from start line for verification") context_end_text: str = Field(..., description="Exact text from end line for verification") issue: str = Field(..., description="Clear description of what needs refactoring") suggestion: str = Field(..., description="Specific refactoring action to take") rationale: str = Field(..., description="Why this improves the code") code_to_replace: str = Field(..., description="Original code that should be changed") replacement_code_snippet: str = Field(..., description="Refactored version of the code") new_code_snippets: Optional[list[dict]] = Field( default_factory=list, description="Additional code snippets to be added" ) class RefactorAction(BaseModel): """Next action for Agent to implement refactoring""" action_type: Literal["EXTRACT_METHOD", "SPLIT_CLASS", "MODERNIZE_SYNTAX", "REORGANIZE_CODE", "DECOMPOSE_FILE"] = ( Field(..., description="Type of action to perform") ) target_file: str = Field(..., description="Absolute path to target file") source_lines: str = Field(..., description="Line range (e.g., '45-67')") description: str = Field(..., description="Step-by-step action description for CLI Agent") class RefactorAnalysisComplete(BaseModel): """Complete refactor analysis with prioritized opportunities""" status: Literal["refactor_analysis_complete"] = "refactor_analysis_complete" refactor_opportunities: list[RefactorOpportunity] = Field(..., description="List of refactoring opportunities") priority_sequence: list[str] = Field(..., description="Recommended order of refactoring IDs") next_actions: list[RefactorAction] = Field(..., description="Specific actions for the agent to implement") class CodeTooLargeRequest(BaseModel): """Request to reduce file selection due to size constraints""" status: Literal["code_too_large"] = "code_too_large" content: str = Field(..., description="Message explaining the size constraint") content_type: Literal["text"] = "text" metadata: dict[str, Any] = Field(default_factory=dict) class ResendPromptRequest(BaseModel): """Request to resend prompt via file due to size limits""" status: Literal["resend_prompt"] = "resend_prompt" content: str = Field(..., description="Instructions for handling large prompt") content_type: Literal["text"] = "text" metadata: dict[str, Any] = Field(default_factory=dict) class TraceEntryPoint(BaseModel): """Entry point information for trace analysis""" file: str = Field(..., description="Absolute path to the file") class_or_struct: str = Field(..., description="Class or module name") method: str = Field(..., description="Method or function name") signature: str = Field(..., description="Full method signature") parameters: Optional[dict[str, Any]] = Field(default_factory=dict, description="Parameter values used in analysis") class TraceTarget(BaseModel): """Target information for dependency analysis""" file: str = Field(..., description="Absolute path to the file") class_or_struct: str = Field(..., description="Class or module name") method: str = Field(..., description="Method or function name") signature: str = Field(..., description="Full method signature") class CallPathStep(BaseModel): """A single step in the call path trace""" from_info: dict[str, Any] = Field(..., description="Source location information", alias="from") to: dict[str, Any] = Field(..., description="Target location information") reason: str = Field(..., description="Reason for the call or dependency") condition: Optional[str] = Field(None, description="Conditional logic if applicable") ambiguous: bool = Field(False, description="Whether this call is ambiguous") class BranchingPoint(BaseModel): """A branching point in the execution flow""" file: str = Field(..., description="File containing the branching point") method: str = Field(..., description="Method containing the branching point") line: int = Field(..., description="Line number of the branching point") condition: str = Field(..., description="Branching condition") branches: list[str] = Field(..., description="Possible execution branches") ambiguous: bool = Field(False, description="Whether the branching is ambiguous") class SideEffect(BaseModel): """A side effect detected in the trace""" type: str = Field(..., description="Type of side effect") description: str = Field(..., description="Description of the side effect") file: str = Field(..., description="File where the side effect occurs") method: str = Field(..., description="Method where the side effect occurs") line: int = Field(..., description="Line number of the side effect") class UnresolvedDependency(BaseModel): """An unresolved dependency in the trace""" reason: str = Field(..., description="Reason why the dependency is unresolved") affected_file: str = Field(..., description="File affected by the unresolved dependency") line: int = Field(..., description="Line number of the unresolved dependency") class IncomingDependency(BaseModel): """An incoming dependency (what calls this target)""" from_file: str = Field(..., description="Source file of the dependency") from_class: str = Field(..., description="Source class of the dependency") from_method: str = Field(..., description="Source method of the dependency") line: int = Field(..., description="Line number of the dependency") type: str = Field(..., description="Type of dependency") class OutgoingDependency(BaseModel): """An outgoing dependency (what this target calls)""" to_file: str = Field(..., description="Target file of the dependency") to_class: str = Field(..., description="Target class of the dependency") to_method: str = Field(..., description="Target method of the dependency") line: int = Field(..., description="Line number of the dependency") type: str = Field(..., description="Type of dependency") class TypeDependency(BaseModel): """A type-level dependency (inheritance, imports, etc.)""" dependency_type: str = Field(..., description="Type of dependency") source_file: str = Field(..., description="Source file of the dependency") source_entity: str = Field(..., description="Source entity (class, module)") target: str = Field(..., description="Target entity") class StateAccess(BaseModel): """State access information""" file: str = Field(..., description="File where state is accessed") method: str = Field(..., description="Method accessing the state") access_type: str = Field(..., description="Type of access (reads, writes, etc.)") state_entity: str = Field(..., description="State entity being accessed") class TraceComplete(BaseModel): """Complete trace analysis response""" status: Literal["trace_complete"] = "trace_complete" trace_type: Literal["precision", "dependencies"] = Field(..., description="Type of trace performed") # Precision mode fields entry_point: Optional[TraceEntryPoint] = Field(None, description="Entry point for precision trace") call_path: Optional[list[CallPathStep]] = Field(default_factory=list, description="Call path for precision trace") branching_points: Optional[list[BranchingPoint]] = Field(default_factory=list, description="Branching points") side_effects: Optional[list[SideEffect]] = Field(default_factory=list, description="Side effects detected") unresolved: Optional[list[UnresolvedDependency]] = Field( default_factory=list, description="Unresolved dependencies" ) # Dependencies mode fields target: Optional[TraceTarget] = Field(None, description="Target for dependency analysis") incoming_dependencies: Optional[list[IncomingDependency]] = Field( default_factory=list, description="Incoming dependencies" ) outgoing_dependencies: Optional[list[OutgoingDependency]] = Field( default_factory=list, description="Outgoing dependencies" ) type_dependencies: Optional[list[TypeDependency]] = Field(default_factory=list, description="Type dependencies") state_access: Optional[list[StateAccess]] = Field(default_factory=list, description="State access information") class DiagnosticHypothesis(BaseModel): """A debugging hypothesis with context and next steps""" rank: int = Field(..., description="Ranking of this hypothesis (1 = most likely)") confidence: Literal["high", "medium", "low"] = Field(..., description="Confidence level") hypothesis: str = Field(..., description="Description of the potential root cause") reasoning: str = Field(..., description="Why this hypothesis is plausible") next_step: str = Field(..., description="Suggested action to test/validate this hypothesis") class StructuredDebugResponse(BaseModel): """Enhanced debug response with multiple hypotheses""" summary: str = Field(..., description="Brief summary of the issue") hypotheses: list[DiagnosticHypothesis] = Field(..., description="Ranked list of potential causes") immediate_actions: list[str] = Field( default_factory=list, description="Immediate steps to take regardless of root cause", ) additional_context_needed: Optional[list[str]] = Field( default_factory=list, description="Additional files or information that would help with analysis", ) class DebugHypothesis(BaseModel): """A debugging hypothesis with detailed analysis""" name: str = Field(..., description="Name/title of the hypothesis") confidence: Literal["High", "Medium", "Low"] = Field(..., description="Confidence level") root_cause: str = Field(..., description="Technical explanation of the root cause") evidence: str = Field(..., description="Logs or code clues supporting this hypothesis") correlation: str = Field(..., description="How symptoms map to the cause") validation: str = Field(..., description="Quick test to confirm the hypothesis") minimal_fix: str = Field(..., description="Smallest change to resolve the issue") regression_check: str = Field(..., description="Why this fix is safe") file_references: list[str] = Field(default_factory=list, description="File:line format for exact locations") class DebugAnalysisComplete(BaseModel): """Complete debugging analysis with systematic investigation tracking""" status: Literal["analysis_complete"] = "analysis_complete" investigation_id: str = Field(..., description="Auto-generated unique ID for this investigation") summary: str = Field(..., description="Brief description of the problem and its impact") investigation_steps: list[str] = Field(..., description="Steps taken during the investigation") hypotheses: list[DebugHypothesis] = Field(..., description="Ranked hypotheses with detailed analysis") key_findings: list[str] = Field(..., description="Important discoveries made during analysis") immediate_actions: list[str] = Field(..., description="Steps to take regardless of which hypothesis is correct") recommended_tools: list[str] = Field(default_factory=list, description="Additional tools recommended for analysis") prevention_strategy: Optional[str] = Field( None, description="Targeted measures to prevent this exact issue from recurring" ) investigation_summary: str = Field( ..., description="Comprehensive summary of the complete investigation process and conclusions" ) class NoBugFound(BaseModel): """Response when thorough investigation finds no concrete evidence of a bug""" status: Literal["no_bug_found"] = "no_bug_found" summary: str = Field(..., description="Summary of what was thoroughly investigated") investigation_steps: list[str] = Field(..., description="Steps taken during the investigation") areas_examined: list[str] = Field(..., description="Code areas and potential failure points examined") confidence_level: Literal["High", "Medium", "Low"] = Field( ..., description="Confidence level in the no-bug finding" ) alternative_explanations: list[str] = Field( ..., description="Possible alternative explanations for reported symptoms" ) recommended_questions: list[str] = Field(..., description="Questions to clarify the issue with the user") next_steps: list[str] = Field(..., description="Suggested actions to better understand the reported issue") # Registry mapping status strings to their corresponding Pydantic models SPECIAL_STATUS_MODELS = { "files_required_to_continue": FilesNeededRequest, "full_codereview_required": FullCodereviewRequired, "focused_review_required": FocusedReviewRequired, "test_sample_needed": TestSampleNeeded, "more_tests_required": MoreTestsRequired, "refactor_analysis_complete": RefactorAnalysisComplete, "trace_complete": TraceComplete, "resend_prompt": ResendPromptRequest, "code_too_large": CodeTooLargeRequest, "analysis_complete": DebugAnalysisComplete, "no_bug_found": NoBugFound, } ================================================ FILE: tools/planner.py ================================================ """ Interactive Sequential Planner - Break down complex tasks through step-by-step planning This tool enables structured planning through an interactive, step-by-step process that builds plans incrementally with the ability to revise, branch, and adapt as understanding deepens. The planner guides users through sequential thinking with forced pauses between steps to ensure thorough consideration of alternatives, dependencies, and strategic decisions before moving to tactical implementation details. Key features: - Sequential planning with full context awareness - Forced deep reflection for complex plans (≥5 steps) in early stages - Branching capabilities for exploring alternative approaches - Revision capabilities to update earlier decisions - Dynamic step count adjustment as plans evolve - Self-contained completion without external expert analysis Perfect for: complex project planning, system design with unknowns, migration strategies, architectural decisions, and breaking down large problems into manageable steps. """ import logging from typing import TYPE_CHECKING, Any from pydantic import Field, field_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_BALANCED from systemprompts import PLANNER_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions matching original planner tool PLANNER_FIELD_DESCRIPTIONS = { "step": ( "Planning content for this step. Step 1: describe the task, problem and scope. Later steps: capture updates, " "revisions, branches, or open questions that shape the plan." ), "step_number": "Current planning step number (starts at 1).", "total_steps": "Estimated number of planning steps; adjust as the plan evolves.", "next_step_required": "Set true when another planning step will follow after this one.", "is_step_revision": "Set true when you are replacing a previously recorded step.", "revises_step_number": "Step number being replaced when revising.", "is_branch_point": "True when this step creates a new branch to explore an alternative path.", "branch_from_step": "If branching, the step number that this branch starts from.", "branch_id": "Name for this branch (e.g. 'approach-A', 'migration-path').", "more_steps_needed": "True when you now expect to add additional steps beyond the prior estimate.", } class PlannerRequest(WorkflowRequest): """Request model for planner workflow tool matching original planner exactly""" # Required fields for each planning step step: str = Field(..., description=PLANNER_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=PLANNER_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=PLANNER_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=PLANNER_FIELD_DESCRIPTIONS["next_step_required"]) # Optional revision/branching fields (planning-specific) is_step_revision: bool | None = Field(False, description=PLANNER_FIELD_DESCRIPTIONS["is_step_revision"]) revises_step_number: int | None = Field(None, description=PLANNER_FIELD_DESCRIPTIONS["revises_step_number"]) is_branch_point: bool | None = Field(False, description=PLANNER_FIELD_DESCRIPTIONS["is_branch_point"]) branch_from_step: int | None = Field(None, description=PLANNER_FIELD_DESCRIPTIONS["branch_from_step"]) branch_id: str | None = Field(None, description=PLANNER_FIELD_DESCRIPTIONS["branch_id"]) more_steps_needed: bool | None = Field(False, description=PLANNER_FIELD_DESCRIPTIONS["more_steps_needed"]) # Exclude all investigation/analysis fields that aren't relevant to planning findings: str = Field( default="", exclude=True, description="Not used for planning - step content serves as findings" ) files_checked: list[str] = Field(default_factory=list, exclude=True, description="Planning doesn't examine files") relevant_files: list[str] = Field(default_factory=list, exclude=True, description="Planning doesn't use files") relevant_context: list[str] = Field( default_factory=list, exclude=True, description="Planning doesn't track code context" ) issues_found: list[dict] = Field(default_factory=list, exclude=True, description="Planning doesn't find issues") confidence: str = Field(default="planning", exclude=True, description="Planning uses different confidence model") hypothesis: str | None = Field(default=None, exclude=True, description="Planning doesn't use hypothesis") # Exclude other non-planning fields temperature: float | None = Field(default=None, exclude=True) thinking_mode: str | None = Field(default=None, exclude=True) use_assistant_model: bool | None = Field(default=False, exclude=True, description="Planning is self-contained") images: list | None = Field(default=None, exclude=True, description="Planning doesn't use images") @field_validator("step_number") @classmethod def validate_step_number(cls, v): if v < 1: raise ValueError("step_number must be at least 1") return v @field_validator("total_steps") @classmethod def validate_total_steps(cls, v): if v < 1: raise ValueError("total_steps must be at least 1") return v class PlannerTool(WorkflowTool): """ Planner workflow tool for step-by-step planning using the workflow architecture. This tool provides the same planning capabilities as the original planner tool but uses the new workflow architecture for consistency with other workflow tools. It maintains all the original functionality including: - Sequential step-by-step planning - Branching and revision capabilities - Deep thinking pauses for complex plans - Conversation memory integration - Self-contained operation (no expert analysis) """ def __init__(self): super().__init__() self.branches = {} def get_name(self) -> str: return "planner" def get_description(self) -> str: return ( "Breaks down complex tasks through interactive, sequential planning with revision and branching capabilities. " "Use for complex project planning, system design, migration strategies, and architectural decisions. " "Builds plans incrementally with deep reflection for complex scenarios." ) def get_system_prompt(self) -> str: return PLANNER_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_BALANCED def get_model_category(self) -> "ToolModelCategory": """Planner requires deep analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def requires_model(self) -> bool: """ Planner tool doesn't require model resolution at the MCP boundary. The planner is a pure data processing tool that organizes planning steps and provides structured guidance without calling external AI models. Returns: bool: False - planner doesn't need AI model access """ return False def get_workflow_request_model(self): """Return the planner-specific request model.""" return PlannerRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema for planner workflow using override pattern.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Planner tool-specific field definitions planner_field_overrides = { # Override standard workflow fields that need planning-specific descriptions "step": { "type": "string", "description": PLANNER_FIELD_DESCRIPTIONS["step"], # Very planning-specific instructions }, # NEW planning-specific fields (not in base workflow) "is_step_revision": { "type": "boolean", "description": PLANNER_FIELD_DESCRIPTIONS["is_step_revision"], }, "revises_step_number": { "type": "integer", "minimum": 1, "description": PLANNER_FIELD_DESCRIPTIONS["revises_step_number"], }, "is_branch_point": { "type": "boolean", "description": PLANNER_FIELD_DESCRIPTIONS["is_branch_point"], }, "branch_from_step": { "type": "integer", "minimum": 1, "description": PLANNER_FIELD_DESCRIPTIONS["branch_from_step"], }, "branch_id": { "type": "string", "description": PLANNER_FIELD_DESCRIPTIONS["branch_id"], }, "more_steps_needed": { "type": "boolean", "description": PLANNER_FIELD_DESCRIPTIONS["more_steps_needed"], }, } # Define excluded fields for planner workflow excluded_workflow_fields = [ "findings", # Planning uses step content instead "files_checked", # Planning doesn't examine files "relevant_files", # Planning doesn't use files "relevant_context", # Planning doesn't track code context "issues_found", # Planning doesn't find issues "confidence", # Planning uses different confidence model "hypothesis", # Planning doesn't use hypothesis ] excluded_common_fields = [ "temperature", # Planning doesn't need temperature control "thinking_mode", # Planning doesn't need thinking mode "images", # Planning doesn't use images "absolute_file_paths", # Planning doesn't use file attachments ] # Build schema with proper field exclusion (following consensus pattern) return WorkflowSchemaBuilder.build_schema( tool_specific_fields=planner_field_overrides, required_fields=[], # No additional required fields beyond workflow defaults model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), excluded_workflow_fields=excluded_workflow_fields, excluded_common_fields=excluded_common_fields, ) # ================================================================================ # Abstract Methods - Required Implementation from BaseWorkflowMixin # ================================================================================ def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each planning phase.""" if step_number == 1: # Initial planning tasks return [ "Think deeply about the complete scope and complexity of what needs to be planned", "Consider multiple approaches and their trade-offs", "Identify key constraints, dependencies, and potential challenges", "Think about stakeholders, success criteria, and critical requirements", ] elif step_number <= 3 and total_steps >= 5: # Complex plan early stages - force deep thinking if step_number == 2: return [ "Evaluate the approach from step 1 - are there better alternatives?", "Break down the major phases and identify critical decision points", "Consider resource requirements and potential bottlenecks", "Think about how different parts interconnect and affect each other", ] else: # step_number == 3 return [ "Validate that the emerging plan addresses the original requirements", "Identify any gaps or assumptions that need clarification", "Consider how to validate progress and adjust course if needed", "Think about what the first concrete steps should be", ] else: # Later steps or simple plans return [ "Continue developing the plan with concrete, actionable steps", "Consider implementation details and practical considerations", "Think about how to sequence and coordinate different activities", "Prepare for execution planning and resource allocation", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """Planner is self-contained and doesn't need expert analysis.""" return False def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Planner doesn't use expert analysis.""" return "" def requires_expert_analysis(self) -> bool: """Planner is self-contained like the original planner tool.""" return False # ================================================================================ # Workflow Customization - Match Original Planner Behavior # ================================================================================ def prepare_step_data(self, request) -> dict: """ Prepare step data from request with planner-specific fields. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": f"Planning step {request.step_number}: {request.step}", # Use step content as findings "files_checked": [], # Planner doesn't check files "relevant_files": [], # Planner doesn't use files "relevant_context": [], # Planner doesn't track context like debug "issues_found": [], # Planner doesn't track issues "confidence": "planning", # Planning confidence is different from investigation "hypothesis": None, # Planner doesn't use hypothesis "images": [], # Planner doesn't use images # Planner-specific fields "is_step_revision": request.is_step_revision or False, "revises_step_number": request.revises_step_number, "is_branch_point": request.is_branch_point or False, "branch_from_step": request.branch_from_step, "branch_id": request.branch_id, "more_steps_needed": request.more_steps_needed or False, } return step_data def build_base_response(self, request, continuation_id: str = None) -> dict: """ Build the base response structure with planner-specific fields. """ # Use work_history from workflow mixin for consistent step tracking # Add 1 to account for current step being processed current_step_count = len(self.work_history) + 1 response_data = { "status": f"{self.get_name()}_in_progress", "step_number": request.step_number, "total_steps": request.total_steps, "next_step_required": request.next_step_required, "step_content": request.step, f"{self.get_name()}_status": { "files_checked": len(self.consolidated_findings.files_checked), "relevant_files": len(self.consolidated_findings.relevant_files), "relevant_context": len(self.consolidated_findings.relevant_context), "issues_found": len(self.consolidated_findings.issues_found), "images_collected": len(self.consolidated_findings.images), "current_confidence": self.get_request_confidence(request), "step_history_length": current_step_count, # Use work_history + current step }, "metadata": { "branches": list(self.branches.keys()), "step_history_length": current_step_count, # Use work_history + current step "is_step_revision": request.is_step_revision or False, "revises_step_number": request.revises_step_number, "is_branch_point": request.is_branch_point or False, "branch_from_step": request.branch_from_step, "branch_id": request.branch_id, "more_steps_needed": request.more_steps_needed or False, }, } if continuation_id: response_data["continuation_id"] = continuation_id return response_data def handle_work_continuation(self, response_data: dict, request) -> dict: """ Handle work continuation with planner-specific deep thinking pauses. """ response_data["status"] = f"pause_for_{self.get_name()}" response_data[f"{self.get_name()}_required"] = True # Get planner-specific required actions required_actions = self.get_required_actions(request.step_number, "planning", request.step, request.total_steps) response_data["required_actions"] = required_actions # Enhanced deep thinking pauses for complex plans if request.total_steps >= 5 and request.step_number <= 3: response_data["status"] = "pause_for_deep_thinking" response_data["thinking_required"] = True response_data["required_thinking"] = required_actions if request.step_number == 1: response_data["next_steps"] = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. This is a complex plan ({request.total_steps} steps) " f"that requires deep thinking. You MUST first spend time reflecting on the planning challenge:\n\n" f"REQUIRED DEEP THINKING before calling {self.get_name()} step {request.step_number + 1}:\n" f"1. Analyze the FULL SCOPE: What exactly needs to be accomplished?\n" f"2. Consider MULTIPLE APPROACHES: What are 2-3 different ways to tackle this?\n" f"3. Identify CONSTRAINTS & DEPENDENCIES: What limits our options?\n" f"4. Think about SUCCESS CRITERIA: How will we know we've succeeded?\n" f"5. Consider RISKS & MITIGATION: What could go wrong early vs late?\n\n" f"Only call {self.get_name()} again with step_number: {request.step_number + 1} AFTER this deep analysis." ) elif request.step_number == 2: response_data["next_steps"] = ( f"STOP! Complex planning requires reflection between steps. DO NOT call {self.get_name()} immediately.\n\n" f"MANDATORY REFLECTION before {self.get_name()} step {request.step_number + 1}:\n" f"1. EVALUATE YOUR APPROACH: Is the direction from step 1 still the best?\n" f"2. IDENTIFY MAJOR PHASES: What are the 3-5 main chunks of work?\n" f"3. SPOT DEPENDENCIES: What must happen before what?\n" f"4. CONSIDER RESOURCES: What skills, tools, or access do we need?\n" f"5. FIND CRITICAL PATHS: Where could delays hurt the most?\n\n" f"Think deeply about these aspects, then call {self.get_name()} with step_number: {request.step_number + 1}." ) elif request.step_number == 3: response_data["next_steps"] = ( f"PAUSE for final strategic reflection. DO NOT call {self.get_name()} yet.\n\n" f"FINAL DEEP THINKING before {self.get_name()} step {request.step_number + 1}:\n" f"1. VALIDATE COMPLETENESS: Does this plan address all original requirements?\n" f"2. CHECK FOR GAPS: What assumptions need validation? What's unclear?\n" f"3. PLAN FOR ADAPTATION: How will we know if we need to change course?\n" f"4. DEFINE FIRST STEPS: What are the first 2-3 concrete actions?\n" f"5. TRANSITION MINDSET: Ready to shift from strategic to tactical planning?\n\n" f"After this reflection, call {self.get_name()} with step_number: {request.step_number + 1} to continue with tactical details." ) else: # Normal flow for simple plans or later steps remaining_steps = request.total_steps - request.step_number response_data["next_steps"] = ( f"Continue with step {request.step_number + 1}. Approximately {remaining_steps} steps remaining." ) return response_data def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match original planner tool format. """ # No need to append to step_history since workflow mixin already manages work_history # and we calculate step counts from work_history # Handle branching like original planner if request.is_branch_point and request.branch_from_step and request.branch_id: if request.branch_id not in self.branches: self.branches[request.branch_id] = [] step_data = self.prepare_step_data(request) self.branches[request.branch_id].append(step_data) # Ensure metadata exists and preserve existing metadata from build_base_response if "metadata" not in response_data: response_data["metadata"] = {} # Store planner-specific metadata that should persist through workflow metadata addition planner_metadata = { "branches": list(self.branches.keys()), "is_step_revision": request.is_step_revision or False, "revises_step_number": request.revises_step_number, "is_branch_point": request.is_branch_point or False, "branch_from_step": request.branch_from_step, "branch_id": request.branch_id, "more_steps_needed": request.more_steps_needed or False, } # Update metadata while preserving existing values response_data["metadata"].update(planner_metadata) # Add planner-specific output instructions for final steps if not request.next_step_required: response_data["planning_complete"] = True response_data["plan_summary"] = ( f"COMPLETE PLAN: {request.step} (Total {request.total_steps} steps completed)" ) response_data["output"] = { "instructions": "This is a structured planning response. Present the step_content as the main planning analysis. If next_step_required is true, continue with the next step. If planning_complete is true, present the complete plan in a well-structured format with clear sections, headings, numbered steps, and visual elements like ASCII charts for phases/dependencies. Use bullet points, sub-steps, sequences, and visual organization to make complex plans easy to understand and follow. IMPORTANT: Do NOT use emojis - use clear text formatting and ASCII characters only. Do NOT mention time estimates or costs unless explicitly requested.", "format": "step_by_step_planning", "presentation_guidelines": { "completed_plans": "Use clear headings, numbered phases, ASCII diagrams for workflows/dependencies, bullet points for sub-tasks, and visual sequences where helpful. No emojis. No time/cost estimates unless requested.", "step_content": "Present as main analysis with clear structure and actionable insights. No emojis. No time/cost estimates unless requested.", "continuation": "Use continuation_id for related planning sessions or implementation planning", }, } response_data["next_steps"] = ( "Planning complete. Present the complete plan to the user in a well-structured format with clear sections, " "numbered steps, visual elements (ASCII charts/diagrams where helpful), sub-step breakdowns, and implementation guidance. " "Use headings, bullet points, and visual organization to make the plan easy to follow. " "If there are phases, dependencies, or parallel tracks, show these relationships visually. " "IMPORTANT: Do NOT use emojis - use clear text formatting and ASCII characters only. " "Do NOT mention time estimates or costs unless explicitly requested. " "After presenting the plan, offer to either help implement specific parts or use the continuation_id to start related planning sessions." ) # Convert generic status names to planner-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "planning_in_progress", f"pause_for_{tool_name}": "pause_for_planning", f"{tool_name}_required": "planning_required", f"{tool_name}_complete": "planning_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] return response_data # ================================================================================ # Hook Method Overrides for Planner-Specific Behavior # ================================================================================ def get_completion_status(self) -> str: """Planner uses planning-specific status.""" return "planning_complete" def get_completion_data_key(self) -> str: """Planner uses 'complete_planning' key.""" return "complete_planning" def get_completion_message(self) -> str: """Planner-specific completion message.""" return ( "Planning complete. Present the complete plan to the user in a well-structured format " "and offer to help implement specific parts or start related planning sessions." ) def get_skip_reason(self) -> str: """Planner-specific skip reason.""" return "Planner is self-contained and completes planning without external analysis" def get_skip_expert_analysis_status(self) -> str: """Planner-specific expert analysis skip status.""" return "skipped_by_tool_design" def store_initial_issue(self, step_description: str): """Store initial planning description.""" self.initial_planning_description = step_description def get_initial_request(self, fallback_step: str) -> str: """Get initial planning description.""" try: return self.initial_planning_description except AttributeError: return fallback_step # Required abstract methods from BaseTool def get_request_model(self): """Return the planner-specific request model.""" return PlannerRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/precommit.py ================================================ """ Precommit Workflow tool - Step-by-step pre-commit validation with expert analysis This tool provides a structured workflow for comprehensive pre-commit validation. It guides the CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough code examination, git change analysis, and issue detection before proceeding. The tool supports finding updates and expert analysis integration. Key features: - Step-by-step pre-commit investigation workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic git repository discovery and change analysis - Expert analysis integration with external models (default) - Support for multiple repositories and change types - Configurable validation type (external with expert model or internal only) """ import logging from typing import TYPE_CHECKING, Any, Literal, Optional from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import PRECOMMIT_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for precommit workflow PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "Step 1: outline how you'll validate the git changes. Later steps: report findings. Review diffs and impacts, use `relevant_files`, and avoid pasting large snippets." ), "step_number": "Current pre-commit step number (starts at 1).", "total_steps": ( "Planned number of validation steps. External validation: use at most three (analysis → follow-ups → summary). Internal validation: a single step. Honour these limits when resuming via continuation_id." ), "next_step_required": ( "True to continue with another step, False when validation is complete. " "CRITICAL: If total_steps>=3 or when `precommit_type = external`, set to True until the final step. " "When continuation_id is provided: Follow the same validation rules based on precommit_type." ), "findings": "Record git diff insights, risks, missing tests, security concerns, and positives; update previous notes as you go.", "files_checked": "Absolute paths for every file examined, including ruled-out candidates.", "relevant_files": "Absolute paths of files involved in the change or validation (code, configs, tests, docs). Must be absolute full non-abbreviated paths.", "relevant_context": "Key functions/methods touched by the change (e.g. 'Class.method', 'function_name').", "issues_found": "List issues with severity (critical/high/medium/low) plus descriptions (bugs, security, performance, coverage).", "precommit_type": "'external' (default, triggers expert model) or 'internal' (local-only validation).", "images": "Optional absolute paths to screenshots or diagrams that aid validation.", "path": "Absolute path to the repository root. Required in step 1.", "compare_to": "Optional git ref (branch/tag/commit) to diff against; falls back to staged/unstaged changes.", "include_staged": "Whether to inspect staged changes (ignored when `compare_to` is set).", "include_unstaged": "Whether to inspect unstaged changes (ignored when `compare_to` is set).", "focus_on": "Optional emphasis areas such as security, performance, or test coverage.", "severity_filter": "Lowest severity to include when reporting issues.", } class PrecommitRequest(WorkflowRequest): """Request model for precommit workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) issues_found: list[dict] = Field( default_factory=list, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["issues_found"] ) precommit_type: Optional[Literal["external", "internal"]] = Field( "external", description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["precommit_type"] ) # Optional images for visual validation images: Optional[list[str]] = Field(default=None, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Precommit-specific fields (only used in step 1 to initialize) # Required for step 1, validated in model_validator path: Optional[str] = Field(None, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["path"]) compare_to: Optional[str] = Field(None, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["compare_to"]) include_staged: Optional[bool] = Field(True, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["include_staged"]) include_unstaged: Optional[bool] = Field( True, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["include_unstaged"] ) focus_on: Optional[str] = Field(None, description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["focus_on"]) severity_filter: Optional[Literal["critical", "high", "medium", "low", "all"]] = Field( "all", description=PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["severity_filter"] ) # Override inherited fields to exclude them from schema (except model which needs to be available) temperature: Optional[float] = Field(default=None, exclude=True) thinking_mode: Optional[str] = Field(default=None, exclude=True) @model_validator(mode="after") def validate_step_one_requirements(self): """Ensure step 1 has required path field.""" if self.step_number == 1 and not self.path: raise ValueError("Step 1 requires 'path' field to specify git repository location") return self class PrecommitTool(WorkflowTool): """ Precommit workflow tool for step-by-step pre-commit validation and expert analysis. This tool implements a structured pre-commit validation workflow that guides users through methodical investigation steps, ensuring thorough change examination, issue identification, and validation before reaching conclusions. It supports complex validation scenarios including multi-repository analysis, security review, performance validation, and integration testing. """ def __init__(self): super().__init__() self.initial_request = None self.git_config = {} def get_name(self) -> str: return "precommit" def get_description(self) -> str: return ( "Validates git changes and repository state before committing with systematic analysis. " "Use for multi-repository validation, security review, change impact assessment, and completeness verification. " "Guides through structured investigation with expert analysis." ) def get_system_prompt(self) -> str: return PRECOMMIT_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Precommit requires thorough analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the precommit workflow-specific request model.""" return PrecommitRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with precommit-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Precommit workflow-specific field overrides precommit_field_overrides = { "step": { "type": "string", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 3, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, "precommit_type": { "type": "string", "enum": ["external", "internal"], "default": "external", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["precommit_type"], }, "issues_found": { "type": "array", "items": {"type": "object"}, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["issues_found"], }, "images": { "type": "array", "items": {"type": "string"}, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, # Precommit-specific fields (for step 1) "path": { "type": "string", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["path"], }, "compare_to": { "type": "string", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["compare_to"], }, "include_staged": { "type": "boolean", "default": True, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["include_staged"], }, "include_unstaged": { "type": "boolean", "default": True, "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["include_unstaged"], }, "focus_on": { "type": "string", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["focus_on"], }, "severity_filter": { "type": "string", "enum": ["critical", "high", "medium", "low", "all"], "default": "all", "description": PRECOMMIT_WORKFLOW_FIELD_DESCRIPTIONS["severity_filter"], }, } # Use WorkflowSchemaBuilder with precommit-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=precommit_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each investigation phase. Now includes request parameter for continuation-aware decisions. """ # Check for continuation - fast track mode if request: continuation_id = self.get_request_continuation_id(request) precommit_type = self.get_precommit_type(request) if continuation_id and precommit_type == "external": if step_number == 1: return [ "Execute git status to see all changes", "Execute git diff --cached for staged changes (exclude binary files)", "Execute git diff for unstaged changes (exclude binary files)", "List any relevant untracked files as well.", ] else: return ["Complete validation and proceed to expert analysis with changeset file"] # Extract counts for normal flow findings_count = len(findings.split("\n")) if findings else 0 issues_count = self.get_consolidated_issues_count() if step_number == 1: # Initial pre-commit investigation tasks return [ "Search for all git repositories in the specified path using appropriate tools", "Check git status to identify staged, unstaged, and untracked changes as required", "Execute git status to see all changes", "Execute git diff --cached for staged changes (exclude binary files)", "Execute git diff for unstaged changes (exclude binary files)", "List any relevant untracked files as well.", "Understand what functionality was added, modified, or removed", "Identify the scope and intent of the changes being committed", "CRITICAL: You are on step 1 - you MUST set next_step_required=True and continue to at least step 3 minimum", ] elif step_number == 2: # Need deeper investigation actions = [ "Examine the specific files you've identified as changed or relevant", "Analyze the logic and implementation details of modifications", "Check for potential issues: bugs, security risks, performance problems", "Verify that changes align with good coding practices and patterns", "Look for missing tests, documentation, or configuration updates", ] # Add step validation reminder if request and request.total_steps >= 3: actions.append( f"CRITICAL: You are on step 2 of {request.total_steps} minimum steps - you MUST set next_step_required=True unless this is the final step" ) return actions elif step_number >= 2 and (findings_count > 2 or issues_count > 0): # Close to completion - need final verification actions = [ "Verify all identified issues have been properly documented", "Check for any missed dependencies or related files that need review", "Confirm the completeness and correctness of your assessment", "Ensure all security, performance, and quality concerns are captured", "Validate that your findings are comprehensive and actionable", ] # Add step validation reminder if request and request.total_steps >= 3 and step_number < request.total_steps: actions.append( f"CRITICAL: You are on step {step_number} of {request.total_steps} minimum steps - set next_step_required=True to continue" ) elif request and request.total_steps >= 3 and step_number >= request.total_steps: actions.append( f"You are on final step {step_number} - you may now set next_step_required=False to complete" ) return actions else: # General investigation needed actions = [ "Continue examining the changes and their potential impact", "Gather more evidence using appropriate investigation tools", "Test your assumptions about the changes and their effects", "Look for patterns that confirm or refute your current assessment", ] # Add step validation reminder for all other cases if request and request.total_steps >= 3: if step_number < request.total_steps: actions.append( f"CRITICAL: You are on step {step_number} of {request.total_steps} minimum steps - set next_step_required=True to continue" ) else: actions.append( f"You are on final step {step_number} - you may now set next_step_required=False to complete" ) return actions def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Decide when to call external model based on investigation completeness. For continuations with external type, always proceed with expert analysis. """ # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # For continuations with external type, always proceed with expert analysis continuation_id = self.get_request_continuation_id(request) if continuation_id and request.precommit_type == "external": return True # Always perform expert analysis for external continuations # Check if we have meaningful investigation data return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call for final pre-commit validation.""" context_parts = [ f"=== PRE-COMMIT ANALYSIS REQUEST ===\\n{self.initial_request or 'Pre-commit validation initiated'}\\n=== END REQUEST ===" ] # Add investigation summary investigation_summary = self._build_precommit_summary(consolidated_findings) context_parts.append( f"\\n=== AGENT'S PRE-COMMIT INVESTIGATION ===\\n{investigation_summary}\\n=== END INVESTIGATION ===" ) # Add git configuration context if available if self.git_config: config_text = "\\n".join(f"- {key}: {value}" for key, value in self.git_config.items()) context_parts.append(f"\\n=== GIT CONFIGURATION ===\\n{config_text}\\n=== END CONFIGURATION ===") # Add relevant methods/functions if available if consolidated_findings.relevant_context: methods_text = "\\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\\n=== RELEVANT CODE ELEMENTS ===\\n{methods_text}\\n=== END CODE ELEMENTS ===") # Add issues found evolution if available if consolidated_findings.issues_found: issues_text = "\\n".join( f"[{issue.get('severity', 'unknown').upper()}] {issue.get('description', 'No description')}" for issue in consolidated_findings.issues_found ) context_parts.append(f"\\n=== ISSUES IDENTIFIED ===\\n{issues_text}\\n=== END ISSUES ===") # Add assessment evolution if available if consolidated_findings.hypotheses: assessments_text = "\\n".join( f"Step {h['step']}: {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\\n=== ASSESSMENT EVOLUTION ===\\n{assessments_text}\\n=== END ASSESSMENTS ===") # Add images if available if consolidated_findings.images: images_text = "\\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append( f"\\n=== VISUAL VALIDATION INFORMATION ===\\n{images_text}\\n=== END VISUAL INFORMATION ===" ) return "\\n".join(context_parts) def _build_precommit_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the pre-commit investigation.""" summary_parts = [ "=== SYSTEMATIC PRE-COMMIT INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Code elements analyzed: {len(consolidated_findings.relevant_context)}", f"Issues identified: {len(consolidated_findings.issues_found)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\\n".join(summary_parts) def should_include_files_in_expert_prompt(self) -> bool: """Include files in expert analysis for comprehensive validation.""" return True def should_embed_system_prompt(self) -> bool: """Embed system prompt in expert analysis for proper context.""" return True def get_expert_thinking_mode(self) -> str: """Use high thinking mode for thorough pre-commit analysis.""" return "high" def get_expert_analysis_instruction(self) -> str: """Get specific instruction for pre-commit expert analysis.""" return ( "Please provide comprehensive pre-commit validation based on the investigation findings. " "Focus on identifying any remaining issues, validating the completeness of the analysis, " "and providing final recommendations for commit readiness." ) # Hook method overrides for precommit-specific behavior def prepare_step_data(self, request) -> dict: """ Map precommit-specific fields for internal processing. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": request.issues_found, "precommit_type": request.precommit_type, "hypothesis": request.findings, # Map findings to hypothesis for compatibility "images": request.images or [], "confidence": "high", # Dummy value for workflow_mixin compatibility } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Precommit workflow skips expert analysis only when precommit_type is "internal". Default is always to use expert analysis (external). For continuations with external type, always perform expert analysis immediately. """ # If it's a continuation and precommit_type is external, don't skip continuation_id = self.get_request_continuation_id(request) if continuation_id and request.precommit_type != "internal": return False # Always do expert analysis for external continuations return request.precommit_type == "internal" and not request.next_step_required def store_initial_issue(self, step_description: str): """Store initial request for expert analysis.""" self.initial_request = step_description # Override inheritance hooks for precommit-specific behavior def get_completion_status(self) -> str: """Precommit tools use precommit-specific status.""" return "validation_complete_ready_for_commit" def get_completion_data_key(self) -> str: """Precommit uses 'complete_validation' key.""" return "complete_validation" def get_final_analysis_from_request(self, request): """Precommit tools use 'findings' field.""" return request.findings def get_precommit_type(self, request) -> str: """Get precommit type from request. Hook method for clean inheritance.""" try: return request.precommit_type or "external" except AttributeError: return "external" # Default to external validation def get_consolidated_issues_count(self) -> int: """Get count of issues from consolidated findings. Hook method for clean access.""" try: return len(self.consolidated_findings.issues_found) except AttributeError: return 0 def get_completion_message(self) -> str: """Precommit-specific completion message.""" return ( "Pre-commit validation complete. You have identified all issues " "and verified commit readiness. MANDATORY: Present the user with the complete validation results " "and IMMEDIATELY proceed with commit if no critical issues found, or provide specific fix guidance " "if issues need resolution. Focus on actionable next steps." ) def get_skip_reason(self) -> str: """Precommit-specific skip reason.""" return ( "Completed comprehensive pre-commit validation with internal analysis only (no external model validation)" ) def get_skip_expert_analysis_status(self) -> str: """Precommit-specific expert analysis skip status.""" return "skipped_due_to_internal_analysis_type" def prepare_work_summary(self) -> str: """Precommit-specific work summary.""" return self._build_precommit_summary(self.consolidated_findings) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Precommit-specific completion message. Args: expert_analysis_used: True if expert analysis was successfully executed """ base_message = ( "PRE-COMMIT VALIDATION IS COMPLETE. You may delete any `pal_precommit.changeset` created. You MUST now summarize " "and present ALL validation results, identified issues with their severity levels, and exact commit recommendations. " "Clearly state whether the changes are ready for commit or require fixes first. Provide concrete, actionable guidance for " "any issues that need resolution—make it easy for a developer to understand exactly what needs to be " "done before committing." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Get additional guidance for handling expert analysis results in pre-commit context. Returns: Additional guidance text for validating and using expert analysis findings """ return ( "IMPORTANT: Expert analysis has been provided above. You MUST carefully review " "the expert's validation findings and security assessments. Cross-reference the " "expert's analysis with your own investigation to ensure all critical issues are " "addressed. Pay special attention to any security vulnerabilities, performance " "concerns, or architectural issues identified by the expert review." ) def get_step_guidance_message(self, request) -> str: """ Precommit-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_precommit_step_guidance(request.step_number, request) return step_guidance["next_steps"] def get_precommit_step_guidance(self, step_number: int, request) -> dict[str, Any]: """ Provide step-specific guidance for precommit workflow. Uses get_required_actions to determine what needs to be done, then formats those actions into appropriate guidance messages. """ # Get the required actions from the single source of truth required_actions = self.get_required_actions( step_number, request.precommit_type or "external", # Using precommit_type as confidence proxy request.findings or "", request.total_steps, request, # Pass request for continuation-aware decisions ) # Check if this is a continuation to provide context-aware guidance continuation_id = self.get_request_continuation_id(request) is_external_continuation = continuation_id and request.precommit_type == "external" is_internal_continuation = continuation_id and request.precommit_type == "internal" # Format the guidance based on step number and continuation status if step_number == 1: if is_external_continuation: # Fast-track mode for external continuations next_steps = ( "You are on step 1 of MAXIMUM 2 steps. CRITICAL: Gather and save the complete git changeset NOW. " "MANDATORY ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + "\\n\\nMANDATORY: The changeset may be large. You MUST save the required changeset as a 'pal_precommit.changeset' file " "(replacing any existing one) in your work directory and include the FULL absolute path in relevant_files (exclude any " "binary files). ONLY include the code changes, no extra commentary." "Set next_step_required=True and step_number=2 for the next call." ) elif is_internal_continuation: # Internal validation mode next_steps = ( "Continuing previous conversation with internal validation only. The analysis will build " "upon the prior findings without external model validation. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) ) else: # Normal flow for new validations next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first investigate " f"the git repositories and changes using appropriate tools. CRITICAL AWARENESS: You need to:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nOnly call {self.get_name()} again AFTER completing your investigation. " f"When you call {self.get_name()} next time, use step_number: {step_number + 1} " f"and report specific files examined, changes analyzed, and validation findings discovered." ) elif step_number == 2: # CRITICAL: Check if violating minimum step requirement if ( request.total_steps >= 3 and request.step_number < request.total_steps and not request.next_step_required ): next_steps = ( f"ERROR: You set total_steps={request.total_steps} but next_step_required=False on step {request.step_number}. " f"This violates the minimum step requirement. You MUST set next_step_required=True until you reach the final step. " f"Call {self.get_name()} again with next_step_required=True and continue your investigation." ) elif is_external_continuation or (not request.next_step_required and request.precommit_type == "external"): # Fast-track completion or about to complete - ensure changeset is saved next_steps = ( "Proceeding immediately to expert analysis. " f"MANDATORY: call {self.get_name()} tool immediately again, and set next_step_required=False to " f"trigger external validation NOW. " f"MANDATORY: Include the entire changeset! The changeset may be large. You MUST save the required " f"changeset as a 'pal_precommit.changeset' file (replacing any existing one) in your work directory " f"and include the FULL absolute path in relevant_files so the expert can access the complete changeset. " f"ONLY include the code changes, no extra commentary." ) else: # Normal flow - deeper analysis needed next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. You are on step 2 of {request.total_steps} minimum required steps. " f"MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nRemember: You MUST set next_step_required=True until step {request.total_steps}. " + f"Only call {self.get_name()} again with step_number: {step_number + 1} AFTER completing these validations." ) elif step_number >= 3: if not request.next_step_required and request.precommit_type == "external": # About to complete - ensure changeset is saved next_steps = ( "Completing validation and proceeding to expert analysis. " "MANDATORY: Save the complete git changeset as a 'pal_precommit.changeset' file " "in your work directory and include the FULL absolute path in relevant_files." ) else: # Later steps - final verification next_steps = ( f"WAIT! Your validation needs final verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nREMEMBER: Ensure you have identified all potential issues and verified commit readiness. " f"Document findings with specific file references and issue descriptions, then call {self.get_name()} " f"with step_number: {step_number + 1}." ) else: # Fallback for any other case - check minimum step violation first if ( request.total_steps >= 3 and request.step_number < request.total_steps and not request.next_step_required ): next_steps = ( f"ERROR: You set total_steps={request.total_steps} but next_step_required=False on step {request.step_number}. " f"This violates the minimum step requirement. You MUST set next_step_required=True until step {request.total_steps}." ) elif not request.next_step_required and request.precommit_type == "external": next_steps = ( "Completing validation. " "MANDATORY: Save complete git changeset as 'pal_precommit.changeset' file and include path in relevant_files, " "excluding any binary files." ) else: next_steps = ( f"PAUSE VALIDATION. Before calling {self.get_name()} step {step_number + 1}, you MUST examine more code and changes. " + "Required: " + ", ".join(required_actions[:2]) + ". " + f"Your next {self.get_name()} call (step_number: {step_number + 1}) must include " f"NEW evidence from actual change analysis, not just theories. NO recursive {self.get_name()} calls " f"without investigation work!" ) return {"next_steps": next_steps} def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match precommit workflow format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Store git configuration for expert analysis if request.path: self.git_config = { "path": request.path, "compare_to": request.compare_to, "include_staged": request.include_staged, "include_unstaged": request.include_unstaged, "severity_filter": request.severity_filter, } # Convert generic status names to precommit-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "validation_in_progress", f"pause_for_{tool_name}": "pause_for_validation", f"{tool_name}_required": "validation_required", f"{tool_name}_complete": "validation_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match precommit workflow if f"{tool_name}_status" in response_data: response_data["validation_status"] = response_data.pop(f"{tool_name}_status") # Add precommit-specific status fields response_data["validation_status"]["issues_identified"] = len(self.consolidated_findings.issues_found) response_data["validation_status"]["precommit_type"] = request.precommit_type or "external" # Map complete_precommitworkflow to complete_validation if f"complete_{tool_name}" in response_data: response_data["complete_validation"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match precommit workflow if f"{tool_name}_complete" in response_data: response_data["validation_complete"] = response_data.pop(f"{tool_name}_complete") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the precommit workflow-specific request model.""" return PrecommitRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/refactor.py ================================================ """ Refactor tool - Step-by-step refactoring analysis with expert validation This tool provides a structured workflow for comprehensive code refactoring analysis. It guides CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough code examination, refactoring opportunity identification, and quality assessment before proceeding. The tool supports complex refactoring scenarios including code smell detection, decomposition planning, modernization opportunities, and organization improvements. Key features: - Step-by-step refactoring investigation workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic refactoring opportunity tracking with type and severity classification - Expert analysis integration with external models - Support for focused refactoring types (codesmells, decompose, modernize, organization) - Confidence-based workflow optimization with refactor completion tracking """ import logging from typing import TYPE_CHECKING, Any, Literal, Optional from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import REFACTOR_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for refactor tool REFACTOR_FIELD_DESCRIPTIONS = { "step": ( "The refactoring plan. Step 1: State strategy. Later steps: Report findings. " "CRITICAL: Examine code for smells, and opportunities for decomposition, modernization, and organization. " "Use 'relevant_files' for code. FORBIDDEN: Large code snippets." ), "step_number": ( "The index of the current step in the refactoring investigation sequence, beginning at 1. Each step should " "build upon or revise the previous one." ), "total_steps": ( "Your current estimate for how many steps will be needed to complete the refactoring investigation. " "Adjust as new opportunities emerge." ), "next_step_required": ( "Set to true if you plan to continue the investigation with another step. False means you believe the " "refactoring analysis is complete and ready for expert validation." ), "findings": ( "Summary of discoveries from this step, including code smells and opportunities for decomposition, modernization, or organization. " "Document both strengths and weaknesses. In later steps, confirm or update past findings." ), "files_checked": ( "List all files examined (absolute paths). Include even ruled-out files to track exploration path." ), "relevant_files": ( "Subset of files_checked with code requiring refactoring (absolute paths). Include files with " "code smells, decomposition needs, or improvement opportunities." ), "relevant_context": ( "List methods/functions central to refactoring opportunities, in 'ClassName.methodName' or 'functionName' format. " "Prioritize those with code smells or needing improvement." ), "issues_found": ( "Refactoring opportunities as dictionaries with 'severity' (critical/high/medium/low), " "'type' (codesmells/decompose/modernize/organization), and 'description'. " "Include all improvement opportunities found." ), "confidence": ( "Your confidence in refactoring analysis: exploring (starting), incomplete (significant work remaining), " "partial (some opportunities found, more analysis needed), complete (comprehensive analysis finished, " "all major opportunities identified). " "WARNING: Use 'complete' ONLY when fully analyzed and can provide recommendations without expert help. " "'complete' PREVENTS expert validation. Use 'partial' for large files or uncertain analysis." ), "images": ( "Optional list of absolute paths to architecture diagrams, UI mockups, design documents, or visual references " "that help with refactoring context. Only include if they materially assist understanding or assessment." ), "refactor_type": "Type of refactoring analysis to perform (codesmells, decompose, modernize, organization)", "focus_areas": "Specific areas to focus on (e.g., 'performance', 'readability', 'maintainability', 'security')", "style_guide_examples": ( "Optional existing code files to use as style/pattern reference (must be FULL absolute paths to real files / " "folders - DO NOT SHORTEN). These files represent the target coding style and patterns for the project." ), } class RefactorRequest(WorkflowRequest): """Request model for refactor workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=REFACTOR_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=REFACTOR_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=REFACTOR_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=REFACTOR_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=REFACTOR_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field(default_factory=list, description=REFACTOR_FIELD_DESCRIPTIONS["files_checked"]) relevant_files: list[str] = Field(default_factory=list, description=REFACTOR_FIELD_DESCRIPTIONS["relevant_files"]) relevant_context: list[str] = Field( default_factory=list, description=REFACTOR_FIELD_DESCRIPTIONS["relevant_context"] ) issues_found: list[dict] = Field(default_factory=list, description=REFACTOR_FIELD_DESCRIPTIONS["issues_found"]) confidence: Optional[Literal["exploring", "incomplete", "partial", "complete"]] = Field( "incomplete", description=REFACTOR_FIELD_DESCRIPTIONS["confidence"] ) # Optional images for visual context images: Optional[list[str]] = Field(default=None, description=REFACTOR_FIELD_DESCRIPTIONS["images"]) # Refactor-specific fields (only used in step 1 to initialize) refactor_type: Optional[Literal["codesmells", "decompose", "modernize", "organization"]] = Field( "codesmells", description=REFACTOR_FIELD_DESCRIPTIONS["refactor_type"] ) focus_areas: Optional[list[str]] = Field(None, description=REFACTOR_FIELD_DESCRIPTIONS["focus_areas"]) style_guide_examples: Optional[list[str]] = Field( None, description=REFACTOR_FIELD_DESCRIPTIONS["style_guide_examples"] ) # Override inherited fields to exclude them from schema (except model which needs to be available) temperature: Optional[float] = Field(default=None, exclude=True) thinking_mode: Optional[str] = Field(default=None, exclude=True) @model_validator(mode="after") def validate_step_one_requirements(self): """Ensure step 1 has required relevant_files field.""" if self.step_number == 1 and not self.relevant_files: raise ValueError( "Step 1 requires 'relevant_files' field to specify code files or directories to analyze for refactoring" ) return self class RefactorTool(WorkflowTool): """ Refactor tool for step-by-step refactoring analysis and expert validation. This tool implements a structured refactoring workflow that guides users through methodical investigation steps, ensuring thorough code examination, refactoring opportunity identification, and improvement assessment before reaching conclusions. It supports complex refactoring scenarios including code smell detection, decomposition planning, modernization opportunities, and organization improvements. """ def __init__(self): super().__init__() self.initial_request = None self.refactor_config = {} def get_name(self) -> str: return "refactor" def get_description(self) -> str: return ( "Analyzes code for refactoring opportunities with systematic investigation. " "Use for code smell detection, decomposition planning, modernization, and maintainability improvements. " "Guides through structured analysis with expert validation." ) def get_system_prompt(self) -> str: return REFACTOR_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Refactor workflow requires thorough analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the refactor workflow-specific request model.""" return RefactorRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with refactor-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Refactor workflow-specific field overrides refactor_field_overrides = { "step": { "type": "string", "description": REFACTOR_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": REFACTOR_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": REFACTOR_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": REFACTOR_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": REFACTOR_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": REFACTOR_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": REFACTOR_FIELD_DESCRIPTIONS["relevant_files"], }, "confidence": { "type": "string", "enum": ["exploring", "incomplete", "partial", "complete"], "default": "incomplete", "description": REFACTOR_FIELD_DESCRIPTIONS["confidence"], }, "issues_found": { "type": "array", "items": {"type": "object"}, "description": REFACTOR_FIELD_DESCRIPTIONS["issues_found"], }, "images": { "type": "array", "items": {"type": "string"}, "description": REFACTOR_FIELD_DESCRIPTIONS["images"], }, # Refactor-specific fields (for step 1) # Note: Use relevant_files field instead of files for consistency "refactor_type": { "type": "string", "enum": ["codesmells", "decompose", "modernize", "organization"], "default": "codesmells", "description": REFACTOR_FIELD_DESCRIPTIONS["refactor_type"], }, "focus_areas": { "type": "array", "items": {"type": "string"}, "description": REFACTOR_FIELD_DESCRIPTIONS["focus_areas"], }, "style_guide_examples": { "type": "array", "items": {"type": "string"}, "description": REFACTOR_FIELD_DESCRIPTIONS["style_guide_examples"], }, } # Use WorkflowSchemaBuilder with refactor-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=refactor_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each investigation phase.""" if step_number == 1: # Initial refactoring investigation tasks return [ "Read and understand the code files specified for refactoring analysis", "Examine the overall structure, architecture, and design patterns used", "Identify potential code smells: long methods, large classes, duplicate code, complex conditionals", "Look for decomposition opportunities: oversized components that could be broken down", "Check for modernization opportunities: outdated patterns, deprecated features, newer language constructs", "Assess organization: logical grouping, file structure, naming conventions, module boundaries", "Document specific refactoring opportunities with file locations and line numbers", ] elif confidence in ["exploring", "incomplete"]: # Need deeper investigation return [ "Examine specific code sections you've identified as needing refactoring", "Analyze code smells in detail: complexity, coupling, cohesion issues", "Investigate decomposition opportunities: identify natural breaking points for large components", "Look for modernization possibilities: language features, patterns, libraries that could improve the code", "Check organization issues: related functionality that could be better grouped or structured", "Trace dependencies and relationships between components to understand refactoring impact", "Prioritize refactoring opportunities by impact and effort required", ] elif confidence == "partial": # Close to completion - need final verification return [ "Verify all identified refactoring opportunities have been properly documented with locations", "Check for any missed opportunities in areas not yet thoroughly examined", "Confirm that refactoring suggestions align with the specified refactor_type and focus_areas", "Ensure refactoring opportunities are prioritized by severity and impact", "Validate that proposed changes would genuinely improve code quality without breaking functionality", "Double-check that all relevant files and code elements are captured in your analysis", ] else: # General investigation needed return [ "Continue examining the codebase for additional refactoring opportunities", "Gather more evidence using appropriate code analysis techniques", "Test your assumptions about code quality and improvement possibilities", "Look for patterns that confirm or refute your current refactoring assessment", "Focus on areas that haven't been thoroughly examined for refactoring potential", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Decide when to call external model based on investigation completeness. Don't call expert analysis if the CLI agent has certain confidence and complete refactoring - trust their judgment. """ # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # Check if refactoring work is complete if request and request.confidence == "complete": return False # Check if we have meaningful investigation data return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call for final refactoring validation.""" context_parts = [ f"=== REFACTORING ANALYSIS REQUEST ===\\n{self.initial_request or 'Refactoring workflow initiated'}\\n=== END REQUEST ===" ] # Add investigation summary investigation_summary = self._build_refactoring_summary(consolidated_findings) context_parts.append( f"\\n=== AGENT'S REFACTORING INVESTIGATION ===\\n{investigation_summary}\\n=== END INVESTIGATION ===" ) # Add refactor configuration context if available if self.refactor_config: config_text = "\\n".join(f"- {key}: {value}" for key, value in self.refactor_config.items() if value) context_parts.append(f"\\n=== REFACTOR CONFIGURATION ===\\n{config_text}\\n=== END CONFIGURATION ===") # Add relevant code elements if available if consolidated_findings.relevant_context: methods_text = "\\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\\n=== RELEVANT CODE ELEMENTS ===\\n{methods_text}\\n=== END CODE ELEMENTS ===") # Add refactoring opportunities found if available if consolidated_findings.issues_found: opportunities_text = "\\n".join( f"[{issue.get('severity', 'unknown').upper()}] {issue.get('type', 'unknown').upper()}: {issue.get('description', 'No description')}" for issue in consolidated_findings.issues_found ) context_parts.append( f"\\n=== REFACTORING OPPORTUNITIES ===\\n{opportunities_text}\\n=== END OPPORTUNITIES ===" ) # Add assessment evolution if available if consolidated_findings.hypotheses: assessments_text = "\\n".join( f"Step {h['step']} ({h['confidence']} confidence): {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\\n=== ASSESSMENT EVOLUTION ===\\n{assessments_text}\\n=== END ASSESSMENTS ===") # Add images if available if consolidated_findings.images: images_text = "\\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append( f"\\n=== VISUAL REFACTORING INFORMATION ===\\n{images_text}\\n=== END VISUAL INFORMATION ===" ) return "\\n".join(context_parts) def _build_refactoring_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the refactoring investigation.""" summary_parts = [ "=== SYSTEMATIC REFACTORING INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Code elements analyzed: {len(consolidated_findings.relevant_context)}", f"Refactoring opportunities identified: {len(consolidated_findings.issues_found)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\\n".join(summary_parts) def should_include_files_in_expert_prompt(self) -> bool: """Include files in expert analysis for comprehensive refactoring validation.""" return True def should_embed_system_prompt(self) -> bool: """Embed system prompt in expert analysis for proper context.""" return True def get_expert_thinking_mode(self) -> str: """Use high thinking mode for thorough refactoring analysis.""" return "high" def get_expert_analysis_instruction(self) -> str: """Get specific instruction for refactoring expert analysis.""" return ( "Please provide comprehensive refactoring analysis based on the investigation findings. " "Focus on validating the identified opportunities, ensuring completeness of the analysis, " "and providing final recommendations for refactoring implementation, following the structured " "format specified in the system prompt." ) # Hook method overrides for refactor-specific behavior def prepare_step_data(self, request) -> dict: """ Map refactor workflow-specific fields for internal processing. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": request.issues_found, "confidence": request.confidence, "hypothesis": request.findings, # Map findings to hypothesis for compatibility "images": request.images or [], } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Refactor workflow skips expert analysis when the CLI agent has "complete" confidence. """ return request.confidence == "complete" and not request.next_step_required def store_initial_issue(self, step_description: str): """Store initial request for expert analysis.""" self.initial_request = step_description # Inheritance hook methods for refactor-specific behavior # Override inheritance hooks for refactor-specific behavior def get_completion_status(self) -> str: """Refactor tools use refactor-specific status.""" return "refactoring_analysis_complete_ready_for_implementation" def get_completion_data_key(self) -> str: """Refactor uses 'complete_refactoring' key.""" return "complete_refactoring" def get_final_analysis_from_request(self, request): """Refactor tools use 'findings' field.""" return request.findings def get_confidence_level(self, request) -> str: """Refactor tools use 'complete' for high confidence.""" return "complete" def get_completion_message(self) -> str: """Refactor-specific completion message.""" return ( "Refactoring analysis complete with COMPLETE confidence. You have identified all significant " "refactoring opportunities and provided comprehensive analysis. MANDATORY: Present the user with " "the complete refactoring results organized by type and severity, and IMMEDIATELY proceed with " "implementing the highest priority refactoring opportunities or provide specific guidance for " "improvements. Focus on actionable refactoring steps." ) def get_skip_reason(self) -> str: """Refactor-specific skip reason.""" return "Completed comprehensive refactoring analysis with full confidence locally" def get_skip_expert_analysis_status(self) -> str: """Refactor-specific expert analysis skip status.""" return "skipped_due_to_complete_refactoring_confidence" def prepare_work_summary(self) -> str: """Refactor-specific work summary.""" return self._build_refactoring_summary(self.consolidated_findings) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Refactor-specific completion message. Args: expert_analysis_used: True if expert analysis was successfully executed """ base_message = ( "REFACTORING ANALYSIS IS COMPLETE. You MUST now summarize and present ALL refactoring opportunities " "organized by type (codesmells → decompose → modernize → organization) and severity (Critical → High → " "Medium → Low), specific code locations with line numbers, and exact recommendations for improvement. " "Clearly prioritize the top 3 refactoring opportunities that need immediate attention. Provide concrete, " "actionable guidance for each opportunity—make it easy for a developer to understand exactly what needs " "to be refactored and how to implement the improvements." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Get additional guidance for handling expert analysis results in refactor context. Returns: Additional guidance text for validating and using expert analysis findings """ return ( "IMPORTANT: Expert refactoring analysis has been provided above. You MUST review " "the expert's architectural insights and refactoring recommendations. Consider whether " "the expert's suggestions align with the codebase's evolution trajectory and current " "team priorities. Pay special attention to any breaking changes, migration complexity, " "or performance implications highlighted by the expert. Present a balanced view that " "considers both immediate benefits and long-term maintainability." ) def get_step_guidance_message(self, request) -> str: """ Refactor-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_refactor_step_guidance(request.step_number, request.confidence, request) return step_guidance["next_steps"] def get_refactor_step_guidance(self, step_number: int, confidence: str, request) -> dict[str, Any]: """ Provide step-specific guidance for refactor workflow. """ # Generate the next steps instruction based on required actions required_actions = self.get_required_actions(step_number, confidence, request.findings, request.total_steps) if step_number == 1: next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first examine " f"the code files thoroughly for refactoring opportunities using appropriate tools. CRITICAL AWARENESS: " f"You need to identify code smells, decomposition opportunities, modernization possibilities, and " f"organization improvements across the specified refactor_type. Look for complexity issues, outdated " f"patterns, oversized components, and structural problems. Use file reading tools, code analysis, and " f"systematic examination to gather comprehensive refactoring information. Only call {self.get_name()} " f"again AFTER completing your investigation. When you call {self.get_name()} next time, use " f"step_number: {step_number + 1} and report specific files examined, refactoring opportunities found, " f"and improvement assessments discovered." ) elif confidence in ["exploring", "incomplete"]: next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. Based on your findings, you've identified areas that need " f"deeper refactoring analysis. MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nOnly call {self.get_name()} again with step_number: {step_number + 1} AFTER " + "completing these refactoring analysis tasks." ) elif confidence == "partial": next_steps = ( f"WAIT! Your refactoring analysis needs final verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nREMEMBER: Ensure you have identified all significant refactoring opportunities across all types and " f"verified the completeness of your analysis. Document opportunities with specific file references and " f"line numbers where applicable, then call {self.get_name()} with step_number: {step_number + 1}." ) else: next_steps = ( f"PAUSE REFACTORING ANALYSIS. Before calling {self.get_name()} step {step_number + 1}, you MUST examine more code thoroughly. " + "Required: " + ", ".join(required_actions[:2]) + ". " + f"Your next {self.get_name()} call (step_number: {step_number + 1}) must include " f"NEW evidence from actual refactoring analysis, not just theories. NO recursive {self.get_name()} calls " f"without investigation work!" ) return {"next_steps": next_steps} def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match refactor workflow format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Store refactor configuration for expert analysis if request.relevant_files: self.refactor_config = { "relevant_files": request.relevant_files, "refactor_type": request.refactor_type, "focus_areas": request.focus_areas, "style_guide_examples": request.style_guide_examples, } # Convert generic status names to refactor-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "refactoring_analysis_in_progress", f"pause_for_{tool_name}": "pause_for_refactoring_analysis", f"{tool_name}_required": "refactoring_analysis_required", f"{tool_name}_complete": "refactoring_analysis_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match refactor workflow if f"{tool_name}_status" in response_data: response_data["refactoring_status"] = response_data.pop(f"{tool_name}_status") # Add refactor-specific status fields refactor_types = {} for issue in self.consolidated_findings.issues_found: issue_type = issue.get("type", "unknown") if issue_type not in refactor_types: refactor_types[issue_type] = 0 refactor_types[issue_type] += 1 response_data["refactoring_status"]["opportunities_by_type"] = refactor_types response_data["refactoring_status"]["refactor_confidence"] = request.confidence # Map complete_refactor to complete_refactoring if f"complete_{tool_name}" in response_data: response_data["complete_refactoring"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match refactor workflow if f"{tool_name}_complete" in response_data: response_data["refactoring_complete"] = response_data.pop(f"{tool_name}_complete") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the refactor workflow-specific request model.""" return RefactorRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/secaudit.py ================================================ """ SECAUDIT Workflow tool - Comprehensive security audit with systematic investigation This tool provides a structured workflow for comprehensive security assessment and analysis. It guides the CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough security examination, vulnerability identification, and compliance assessment before proceeding. The tool supports complex security scenarios including OWASP Top 10 coverage, compliance framework mapping, and technology-specific security patterns. Key features: - Step-by-step security audit workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic security issue tracking with severity classification - Expert analysis integration with external models - Support for focused security audits (OWASP, compliance, technology-specific) - Confidence-based workflow optimization - Risk-based prioritization and remediation planning """ import logging from typing import TYPE_CHECKING, Any, Literal, Optional from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import SECAUDIT_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for security audit workflow SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "Step 1: outline the audit strategy (OWASP Top 10, auth, validation, etc.). Later steps: report findings. MANDATORY: use `relevant_files` for code references and avoid large snippets." ), "step_number": "Current security-audit step number (starts at 1).", "total_steps": "Expected number of audit steps; adjust as new risks surface.", "next_step_required": "True while additional threat analysis remains; set False once you are ready to hand off for validation.", "findings": "Summarize vulnerabilities, auth issues, validation gaps, compliance notes, and positives; update prior findings as needed.", "files_checked": "Absolute paths for every file inspected, including rejected candidates.", "relevant_files": "Absolute paths for security-relevant files (auth modules, configs, sensitive code).", "relevant_context": "Security-critical classes/methods (e.g. 'AuthService.login', 'encryption_helper').", "issues_found": "Security issues with severity (critical/high/medium/low) and descriptions (vulns, auth flaws, injection, crypto, config).", "confidence": "exploring/low/medium/high/very_high/almost_certain/certain. 'certain' blocks external validation—use only when fully complete.", "images": "Optional absolute paths to diagrams or threat models that inform the audit.", "security_scope": "Security context (web, mobile, API, cloud, etc.) including stack, user types, data sensitivity, and threat landscape.", "threat_level": "Assess the threat level: low (internal/low-risk), medium (customer-facing/business data), high (regulated or sensitive), critical (financial/healthcare/PII).", "compliance_requirements": "Applicable compliance frameworks or standards (SOC2, PCI DSS, HIPAA, GDPR, ISO 27001, NIST, etc.).", "audit_focus": "Primary focus area: owasp, compliance, infrastructure, dependencies, or comprehensive.", "severity_filter": "Minimum severity to include when reporting security issues.", } class SecauditRequest(WorkflowRequest): """Request model for security audit workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) issues_found: list[dict] = Field( default_factory=list, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["issues_found"] ) confidence: Optional[str] = Field("low", description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["confidence"]) # Optional images for visual context images: Optional[list[str]] = Field(default=None, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Security audit-specific fields security_scope: Optional[str] = Field(None, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["security_scope"]) threat_level: Optional[Literal["low", "medium", "high", "critical"]] = Field( "medium", description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["threat_level"] ) compliance_requirements: Optional[list[str]] = Field( default_factory=list, description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["compliance_requirements"] ) audit_focus: Optional[Literal["owasp", "compliance", "infrastructure", "dependencies", "comprehensive"]] = Field( "comprehensive", description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["audit_focus"] ) severity_filter: Optional[Literal["critical", "high", "medium", "low", "all"]] = Field( "all", description=SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["severity_filter"] ) @model_validator(mode="after") def validate_security_audit_request(self): """Validate security audit request parameters""" # Ensure security scope is provided for comprehensive audits if self.step_number == 1 and not self.security_scope: logger.warning("Security scope not provided for security audit - defaulting to general application") # Validate compliance requirements format if self.compliance_requirements: valid_compliance = {"SOC2", "PCI DSS", "HIPAA", "GDPR", "ISO 27001", "NIST", "FedRAMP", "FISMA"} for req in self.compliance_requirements: if req not in valid_compliance: logger.warning(f"Unknown compliance requirement: {req}") return self class SecauditTool(WorkflowTool): """ Comprehensive security audit workflow tool. Provides systematic security assessment through multi-step investigation covering OWASP Top 10, compliance requirements, and technology-specific security patterns. Follows established WorkflowTool patterns while adding security-specific capabilities. """ def __init__(self): super().__init__() self.initial_request = None self.security_config = {} def get_name(self) -> str: """Return the unique name of the tool.""" return "secaudit" def get_description(self) -> str: """Return a description of the tool.""" return ( "Performs comprehensive security audit with systematic vulnerability assessment. " "Use for OWASP Top 10 analysis, compliance evaluation, threat modeling, and security architecture review. " "Guides through structured security investigation with expert validation." ) def get_system_prompt(self) -> str: """Return the system prompt for expert security analysis.""" return SECAUDIT_PROMPT def get_default_temperature(self) -> float: """Return the temperature for security audit analysis""" return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Return the model category for security audit""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self) -> type: """Return the workflow request model class""" return SecauditRequest def get_tool_fields(self) -> dict[str, dict[str, Any]]: """ Get security audit tool field definitions. Returns comprehensive field definitions including security-specific parameters while maintaining compatibility with existing workflow patterns. """ return SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """ Provide step-specific guidance for systematic security analysis. Each step focuses on specific security domains to ensure comprehensive coverage without missing critical security aspects. """ if step_number == 1: return [ "Identify application type, technology stack, and security scope", "Map attack surface, entry points, and data flows", "Determine relevant security standards and compliance requirements", "Establish threat landscape and risk context for the application", ] elif step_number == 2: return [ "Analyze authentication mechanisms and session management", "Check authorization controls, access patterns, and privilege escalation risks", "Assess multi-factor authentication, password policies, and account security", "Review identity and access management implementations", ] elif step_number == 3: return [ "Examine input validation and sanitization mechanisms across all entry points", "Check for injection vulnerabilities (SQL, XSS, Command, LDAP, NoSQL)", "Review data encryption, sensitive data handling, and cryptographic implementations", "Analyze API input validation, rate limiting, and request/response security", ] elif step_number == 4: return [ "Conduct OWASP Top 10 (2021) systematic review across all categories", "Check each OWASP category methodically with specific findings and evidence", "Cross-reference findings with application context and technology stack", "Prioritize vulnerabilities based on exploitability and business impact", ] elif step_number == 5: return [ "Analyze third-party dependencies for known vulnerabilities and outdated versions", "Review configuration security, default settings, and hardening measures", "Check for hardcoded secrets, credentials, and sensitive information exposure", "Assess logging, monitoring, incident response, and security observability", ] elif step_number == 6: return [ "Evaluate compliance requirements and identify gaps in controls", "Assess business impact and risk levels of all identified findings", "Create prioritized remediation roadmap with timeline and effort estimates", "Document comprehensive security posture and recommendations", ] else: return [ "Continue systematic security investigation based on emerging findings", "Deep-dive into specific security concerns identified in previous steps", "Validate security hypotheses and confirm vulnerability assessments", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Determine when to call expert security analysis. Expert analysis is triggered when the security audit has meaningful findings unless the user requested to skip assistant model. """ # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # Check if we have meaningful investigation data return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings) -> str: """ Prepare comprehensive context for expert security model analysis. Provides security-specific context including scope, threat level, compliance requirements, and systematic findings for expert validation. """ context_parts = [ f"=== SECURITY AUDIT REQUEST ===\n{self.initial_request or 'Security audit workflow initiated'}\n=== END REQUEST ===" ] # Add investigation summary investigation_summary = self._build_security_audit_summary(consolidated_findings) context_parts.append( f"\n=== AGENT'S SECURITY INVESTIGATION ===\n{investigation_summary}\n=== END INVESTIGATION ===" ) # Add security configuration context if available if self.security_config: config_text = "\n".join(f"- {key}: {value}" for key, value in self.security_config.items() if value) context_parts.append(f"\n=== SECURITY CONFIGURATION ===\n{config_text}\n=== END CONFIGURATION ===") # Add relevant files if available if consolidated_findings.relevant_files: files_text = "\n".join(f"- {file}" for file in consolidated_findings.relevant_files) context_parts.append(f"\n=== RELEVANT FILES ===\n{files_text}\n=== END FILES ===") # Add relevant security elements if available if consolidated_findings.relevant_context: methods_text = "\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append( f"\n=== SECURITY-CRITICAL CODE ELEMENTS ===\n{methods_text}\n=== END CODE ELEMENTS ===" ) # Add security issues found if available if consolidated_findings.issues_found: issues_text = self._format_security_issues(consolidated_findings.issues_found) context_parts.append(f"\n=== SECURITY ISSUES IDENTIFIED ===\n{issues_text}\n=== END ISSUES ===") # Add assessment evolution if available if consolidated_findings.hypotheses: assessments_text = "\n".join( f"Step {h['step']} ({h['confidence']} confidence): {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\n=== ASSESSMENT EVOLUTION ===\n{assessments_text}\n=== END ASSESSMENTS ===") # Add images if available if consolidated_findings.images: images_text = "\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append( f"\n=== VISUAL SECURITY INFORMATION ===\n{images_text}\n=== END VISUAL INFORMATION ===" ) return "\n".join(context_parts) def _format_security_issues(self, issues_found: list[dict]) -> str: """ Format security issues for expert analysis. Organizes security findings by severity for clear expert review. """ if not issues_found: return "No security issues identified during systematic investigation." # Group issues by severity severity_groups = {"critical": [], "high": [], "medium": [], "low": []} for issue in issues_found: severity = issue.get("severity", "low").lower() description = issue.get("description", "No description provided") if severity in severity_groups: severity_groups[severity].append(description) else: severity_groups["low"].append(f"[{severity.upper()}] {description}") formatted_issues = [] for severity in ["critical", "high", "medium", "low"]: if severity_groups[severity]: formatted_issues.append(f"\n{severity.upper()} SEVERITY:") for issue in severity_groups[severity]: formatted_issues.append(f" • {issue}") return "\n".join(formatted_issues) if formatted_issues else "No security issues identified." def _build_security_audit_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the security audit investigation.""" summary_parts = [ "=== SYSTEMATIC SECURITY AUDIT INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Security-critical elements analyzed: {len(consolidated_findings.relevant_context)}", f"Security issues identified: {len(consolidated_findings.issues_found)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\n".join(summary_parts) def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with security audit-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Security audit workflow-specific field overrides secaudit_field_overrides = { "step": { "type": "string", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, "confidence": { "type": "string", "enum": ["exploring", "low", "medium", "high", "very_high", "almost_certain", "certain"], "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["confidence"], }, "issues_found": { "type": "array", "items": {"type": "object"}, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["issues_found"], }, "images": { "type": "array", "items": {"type": "string"}, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, # Security audit-specific fields (for step 1) "security_scope": { "type": "string", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["security_scope"], }, "threat_level": { "type": "string", "enum": ["low", "medium", "high", "critical"], "default": "medium", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["threat_level"], }, "compliance_requirements": { "type": "array", "items": {"type": "string"}, "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["compliance_requirements"], }, "audit_focus": { "type": "string", "enum": ["owasp", "compliance", "infrastructure", "dependencies", "comprehensive"], "default": "comprehensive", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["audit_focus"], }, "severity_filter": { "type": "string", "enum": ["critical", "high", "medium", "low", "all"], "default": "all", "description": SECAUDIT_WORKFLOW_FIELD_DESCRIPTIONS["severity_filter"], }, } # Use WorkflowSchemaBuilder with security audit-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=secaudit_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) # Hook method overrides for security audit-specific behavior def prepare_step_data(self, request) -> dict: """Map security audit-specific fields for internal processing.""" step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": request.issues_found, "confidence": request.confidence, "hypothesis": request.findings, # Map findings to hypothesis for compatibility "images": request.images or [], } # Store security-specific configuration on first step if request.step_number == 1: self.security_config = { "security_scope": request.security_scope, "threat_level": request.threat_level, "compliance_requirements": request.compliance_requirements, "audit_focus": request.audit_focus, "severity_filter": request.severity_filter, } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """Security audit workflow skips expert analysis when the CLI agent has "certain" confidence.""" return request.confidence == "certain" and not request.next_step_required def store_initial_issue(self, step_description: str): """Store initial request for expert analysis.""" self.initial_request = step_description def should_include_files_in_expert_prompt(self) -> bool: """Include files in expert analysis for comprehensive security audit.""" return True def should_embed_system_prompt(self) -> bool: """Embed system prompt in expert analysis for proper context.""" return True def get_expert_thinking_mode(self) -> str: """Use high thinking mode for thorough security analysis.""" return "high" def get_expert_analysis_instruction(self) -> str: """Get specific instruction for security audit expert analysis.""" return ( "Please provide comprehensive security analysis based on the investigation findings. " "Focus on identifying any remaining vulnerabilities, validating the completeness of the analysis, " "and providing final recommendations for security improvements, following the OWASP-based " "format specified in the system prompt." ) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Security audit-specific completion message. """ base_message = ( "SECURITY AUDIT IS COMPLETE. You MUST now summarize and present ALL security findings organized by " "severity (Critical → High → Medium → Low), specific code locations with line numbers, and exact " "remediation steps for each vulnerability. Clearly prioritize the top 3 security issues that need " "immediate attention. Provide concrete, actionable guidance for each vulnerability—make it easy for " "developers to understand exactly what needs to be fixed and how to implement the security improvements." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Provide specific guidance for handling expert analysis in security audits. """ return ( "IMPORTANT: Analysis from an assistant model has been provided above. You MUST critically evaluate and validate " "the expert security findings rather than accepting them blindly. Cross-reference the expert analysis with " "your own investigation findings, verify that suggested security improvements are appropriate for this " "application's context and threat model, and ensure recommendations align with the project's security requirements. " "Present a synthesis that combines your systematic security review with validated expert insights, clearly " "distinguishing between vulnerabilities you've independently confirmed and additional insights from expert analysis." ) def get_step_guidance_message(self, request) -> str: """ Security audit-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_security_audit_step_guidance(request.step_number, request.confidence, request) return step_guidance["next_steps"] def get_security_audit_step_guidance(self, step_number: int, confidence: str, request) -> dict[str, Any]: """ Provide step-specific guidance for security audit workflow. """ # Generate the next steps instruction based on required actions required_actions = self.get_required_actions(step_number, confidence, request.findings, request.total_steps) if step_number == 1: next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first examine " f"the code files thoroughly using appropriate tools. CRITICAL AWARENESS: You need to understand " f"the security landscape, identify potential vulnerabilities across OWASP Top 10 categories, " f"and look for authentication flaws, injection points, cryptographic issues, and authorization bypasses. " f"Use file reading tools, security analysis, and systematic examination to gather comprehensive information. " f"Only call {self.get_name()} again AFTER completing your security investigation. When you call " f"{self.get_name()} next time, use step_number: {step_number + 1} and report specific " f"files examined, vulnerabilities found, and security assessments discovered." ) elif confidence in ["exploring", "low"]: next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. Based on your findings, you've identified areas that need " f"deeper security analysis. MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nOnly call {self.get_name()} again with step_number: {step_number + 1} AFTER " + "completing these security audit tasks." ) elif confidence in ["medium", "high"]: next_steps = ( f"WAIT! Your security audit needs final verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\n" + "\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nREMEMBER: Ensure you have identified all significant vulnerabilities across all severity levels and " f"verified the completeness of your security review. Document findings with specific file references and " f"line numbers where applicable, then call {self.get_name()} with step_number: {step_number + 1}." ) else: next_steps = ( f"PAUSE SECURITY AUDIT. Before calling {self.get_name()} step {step_number + 1}, you MUST examine more code thoroughly. " + "Required: " + ", ".join(required_actions[:2]) + ". " + f"Your next {self.get_name()} call (step_number: {step_number + 1}) must include " f"NEW evidence from actual security analysis, not just theories. NO recursive {self.get_name()} calls " f"without investigation work!" ) return {"next_steps": next_steps} def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match security audit workflow format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Store security configuration for expert analysis if request.relevant_files: self.security_config = { "relevant_files": request.relevant_files, "security_scope": request.security_scope, "threat_level": request.threat_level, "compliance_requirements": request.compliance_requirements, "audit_focus": request.audit_focus, "severity_filter": request.severity_filter, } # Convert generic status names to security audit-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "security_audit_in_progress", f"pause_for_{tool_name}": "pause_for_security_audit", f"{tool_name}_required": "security_audit_required", f"{tool_name}_complete": "security_audit_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match security audit workflow if f"{tool_name}_status" in response_data: response_data["security_audit_status"] = response_data.pop(f"{tool_name}_status") # Add security audit-specific status fields response_data["security_audit_status"]["vulnerabilities_by_severity"] = {} for issue in self.consolidated_findings.issues_found: severity = issue.get("severity", "unknown") if severity not in response_data["security_audit_status"]["vulnerabilities_by_severity"]: response_data["security_audit_status"]["vulnerabilities_by_severity"][severity] = 0 response_data["security_audit_status"]["vulnerabilities_by_severity"][severity] += 1 response_data["security_audit_status"]["audit_confidence"] = self.get_request_confidence(request) # Map complete_secaudit to complete_security_audit if f"complete_{tool_name}" in response_data: response_data["complete_security_audit"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match security audit workflow if f"{tool_name}_complete" in response_data: response_data["security_audit_complete"] = response_data.pop(f"{tool_name}_complete") return response_data # Override inheritance hooks for security audit-specific behavior def get_completion_status(self) -> str: """Security audit tools use audit-specific status.""" return "security_analysis_complete" def get_completion_data_key(self) -> str: """Security audit uses 'complete_security_audit' key.""" return "complete_security_audit" def get_final_analysis_from_request(self, request): """Security audit tools use 'findings' field.""" return request.findings def get_confidence_level(self, request) -> str: """Security audit tools use 'certain' for high confidence.""" return "certain" def get_completion_message(self) -> str: """Security audit-specific completion message.""" return ( "Security audit complete with CERTAIN confidence. You have identified all significant vulnerabilities " "and provided comprehensive security analysis. MANDATORY: Present the user with the complete security audit results " "categorized by severity, and IMMEDIATELY proceed with implementing the highest priority security fixes " "or provide specific guidance for vulnerability remediation. Focus on actionable security recommendations." ) def get_skip_reason(self) -> str: """Security audit-specific skip reason.""" return "Completed comprehensive security audit with full confidence locally" def get_skip_expert_analysis_status(self) -> str: """Security audit-specific expert analysis skip status.""" return "skipped_due_to_certain_audit_confidence" def prepare_work_summary(self) -> str: """Security audit-specific work summary.""" return self._build_security_audit_summary(self.consolidated_findings) def get_request_model(self): """Return the request model for this tool""" return SecauditRequest async def prepare_prompt(self, request: SecauditRequest) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/shared/__init__.py ================================================ """ Shared infrastructure for PAL MCP tools. This module contains the core base classes and utilities that are shared across all tool types. It provides the foundation for the tool architecture. """ from .base_models import BaseWorkflowRequest, ConsolidatedFindings, ToolRequest, WorkflowRequest from .base_tool import BaseTool from .schema_builders import SchemaBuilder __all__ = [ "BaseTool", "ToolRequest", "BaseWorkflowRequest", "WorkflowRequest", "ConsolidatedFindings", "SchemaBuilder", ] ================================================ FILE: tools/shared/base_models.py ================================================ """ Base models for PAL MCP tools. This module contains the shared Pydantic models used across all tools, extracted to avoid circular imports and promote code reuse. Key Models: - ToolRequest: Base request model for all tools - WorkflowRequest: Extended request model for workflow-based tools - ConsolidatedFindings: Model for tracking workflow progress """ import logging from typing import Optional from pydantic import BaseModel, Field, field_validator logger = logging.getLogger(__name__) # Shared field descriptions to avoid duplication COMMON_FIELD_DESCRIPTIONS = { "model": "Model to run. Supply a name if requested by the user or stay in auto mode. When in auto mode, use `listmodels` tool for model discovery.", "temperature": "0 = deterministic · 1 = creative.", "thinking_mode": "Reasoning depth: minimal, low, medium, high, or max.", "continuation_id": ( "Unique thread continuation ID for multi-turn conversations. Works across different tools. " "ALWAYS reuse the last continuation_id you were given—this preserves full conversation context, " "files, and findings so the agent can resume seamlessly." ), "images": "Optional absolute image paths or base64 blobs for visual context.", "absolute_file_paths": "Full paths to relevant code", } # Workflow-specific field descriptions WORKFLOW_FIELD_DESCRIPTIONS = { "step": "Current work step content and findings from your overall work", "step_number": "Current step number in work sequence (starts at 1)", "total_steps": "Estimated total steps needed to complete work", "next_step_required": "Whether another work step is needed. When false, aim to reduce total_steps to match step_number to avoid mismatch.", "findings": "Important findings, evidence and insights discovered in this step", "files_checked": "List of files examined during this work step", "relevant_files": "Files identified as relevant to issue/goal (FULL absolute paths to real files/folders - DO NOT SHORTEN)", "relevant_context": "Methods/functions identified as involved in the issue", "issues_found": "Issues identified with severity levels during work", "confidence": ( "Confidence level: exploring (just starting), low (early investigation), " "medium (some evidence), high (strong evidence), very_high (comprehensive understanding), " "almost_certain (near complete confidence), certain (100% confidence locally - no external validation needed)" ), "hypothesis": "Current theory about issue/goal based on work", "use_assistant_model": ( "Use assistant model for expert analysis after workflow steps. " "False skips expert analysis, relies solely on your personal investigation. " "Defaults to True for comprehensive validation." ), } class ToolRequest(BaseModel): """ Base request model for all PAL MCP tools. This model defines common fields that all tools accept, including model selection, temperature control, and conversation threading. Tool-specific request models should inherit from this class. """ # Model configuration model: Optional[str] = Field(None, description=COMMON_FIELD_DESCRIPTIONS["model"]) temperature: Optional[float] = Field(None, ge=0.0, le=1.0, description=COMMON_FIELD_DESCRIPTIONS["temperature"]) thinking_mode: Optional[str] = Field(None, description=COMMON_FIELD_DESCRIPTIONS["thinking_mode"]) # Conversation support continuation_id: Optional[str] = Field(None, description=COMMON_FIELD_DESCRIPTIONS["continuation_id"]) # Visual context images: Optional[list[str]] = Field(None, description=COMMON_FIELD_DESCRIPTIONS["images"]) class BaseWorkflowRequest(ToolRequest): """ Minimal base request model for workflow tools. This provides only the essential fields that ALL workflow tools need, allowing for maximum flexibility in tool-specific implementations. """ # Core workflow fields that ALL workflow tools need step: str = Field(..., description=WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., ge=1, description=WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., ge=1, description=WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) class WorkflowRequest(BaseWorkflowRequest): """ Extended request model for workflow-based tools. This model extends ToolRequest with fields specific to the workflow pattern, where tools perform multi-step work with forced pauses between steps. Used by: debug, precommit, codereview, refactor, thinkdeep, analyze """ # Required workflow fields step: str = Field(..., description=WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., ge=1, description=WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., ge=1, description=WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Work tracking fields findings: str = Field(..., description=WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field(default_factory=list, description=WORKFLOW_FIELD_DESCRIPTIONS["files_checked"]) relevant_files: list[str] = Field(default_factory=list, description=WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"]) relevant_context: list[str] = Field( default_factory=list, description=WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) issues_found: list[dict] = Field(default_factory=list, description=WORKFLOW_FIELD_DESCRIPTIONS["issues_found"]) confidence: str = Field("low", description=WORKFLOW_FIELD_DESCRIPTIONS["confidence"]) # Optional workflow fields hypothesis: Optional[str] = Field(None, description=WORKFLOW_FIELD_DESCRIPTIONS["hypothesis"]) use_assistant_model: Optional[bool] = Field(True, description=WORKFLOW_FIELD_DESCRIPTIONS["use_assistant_model"]) @field_validator("files_checked", "relevant_files", "relevant_context", mode="before") @classmethod def convert_string_to_list(cls, v): """Convert string inputs to empty lists to handle malformed inputs gracefully.""" if isinstance(v, str): logger.warning(f"Field received string '{v}' instead of list, converting to empty list") return [] return v class ConsolidatedFindings(BaseModel): """ Model for tracking consolidated findings across workflow steps. This model accumulates findings, files, methods, and issues discovered during multi-step work. It's used by BaseWorkflowMixin to track progress across workflow steps. """ files_checked: set[str] = Field(default_factory=set, description="All files examined across all steps") relevant_files: set[str] = Field( default_factory=set, description="Subset of files_checked identified as relevant for work at hand", ) relevant_context: set[str] = Field( default_factory=set, description="All methods/functions identified during overall work" ) findings: list[str] = Field(default_factory=list, description="Chronological findings from each work step") hypotheses: list[dict] = Field(default_factory=list, description="Evolution of hypotheses across steps") issues_found: list[dict] = Field(default_factory=list, description="All issues with severity levels") images: list[str] = Field(default_factory=list, description="Images collected during work") confidence: str = Field("low", description="Latest confidence level from steps") # Tool-specific field descriptions are now declared in each tool file # This keeps concerns separated and makes each tool self-contained ================================================ FILE: tools/shared/base_tool.py ================================================ """ Core Tool Infrastructure for PAL MCP Tools This module provides the fundamental base class for all tools: - BaseTool: Abstract base class defining the tool interface The BaseTool class defines the core contract that tools must implement and provides common functionality for request validation, error handling, model management, conversation handling, file processing, and response formatting. """ import logging import os from abc import ABC, abstractmethod from typing import TYPE_CHECKING, Any, Optional from mcp.types import TextContent if TYPE_CHECKING: from providers.shared import ModelCapabilities from tools.models import ToolModelCategory from config import MCP_PROMPT_SIZE_LIMIT from providers import ModelProvider, ModelProviderRegistry from utils import estimate_tokens from utils.conversation_memory import ( ConversationTurn, get_conversation_file_list, get_thread, ) from utils.env import get_env from utils.file_utils import read_file_content, read_files # Import models from tools.models for compatibility try: from tools.models import SPECIAL_STATUS_MODELS, ContinuationOffer, ToolOutput except ImportError: # Fallback in case models haven't been set up yet SPECIAL_STATUS_MODELS = {} ContinuationOffer = None ToolOutput = None logger = logging.getLogger(__name__) class BaseTool(ABC): """ Abstract base class for all PAL MCP tools. This class defines the interface that all tools must implement and provides common functionality for request handling, model creation, and response formatting. CONVERSATION-AWARE FILE PROCESSING: This base class implements the sophisticated dual prioritization strategy for conversation-aware file handling across all tools: 1. FILE DEDUPLICATION WITH NEWEST-FIRST PRIORITY: - When same file appears in multiple conversation turns, newest reference wins - Prevents redundant file embedding while preserving most recent file state - Cross-tool file tracking ensures consistent behavior across analyze → codereview → debug 2. CONVERSATION CONTEXT INTEGRATION: - All tools receive enhanced prompts with conversation history via reconstruct_thread_context() - File references from previous turns are preserved and accessible - Cross-tool knowledge transfer maintains full context without manual file re-specification 3. TOKEN-AWARE FILE EMBEDDING: - Respects model-specific token allocation budgets from ModelContext - Prioritizes conversation history, then newest files, then remaining content - Graceful degradation when token limits are approached 4. STATELESS-TO-STATEFUL BRIDGING: - Tools operate on stateless MCP requests but access full conversation state - Conversation memory automatically injected via continuation_id parameter - Enables natural AI-to-AI collaboration across tool boundaries To create a new tool: 1. Create a new class that inherits from BaseTool 2. Implement all abstract methods 3. Define a request model that inherits from ToolRequest 4. Register the tool in server.py's TOOLS dictionary """ # Class-level cache for OpenRouter registry to avoid multiple loads _openrouter_registry_cache = None _custom_registry_cache = None @classmethod def _get_openrouter_registry(cls): """Get cached OpenRouter registry instance, creating if needed.""" # Use BaseTool class directly to ensure cache is shared across all subclasses if BaseTool._openrouter_registry_cache is None: from providers.registries.openrouter import OpenRouterModelRegistry BaseTool._openrouter_registry_cache = OpenRouterModelRegistry() logger.debug("Created cached OpenRouter registry instance") return BaseTool._openrouter_registry_cache @classmethod def _get_custom_registry(cls): """Get cached custom-endpoint registry instance.""" if BaseTool._custom_registry_cache is None: from providers.registries.custom import CustomEndpointModelRegistry BaseTool._custom_registry_cache = CustomEndpointModelRegistry() logger.debug("Created cached Custom registry instance") return BaseTool._custom_registry_cache def __init__(self): # Cache tool metadata at initialization to avoid repeated calls self.name = self.get_name() self.description = self.get_description() self.default_temperature = self.get_default_temperature() # Tool initialization complete @abstractmethod def get_name(self) -> str: """ Return the unique name identifier for this tool. This name is used by MCP clients to invoke the tool and must be unique across all registered tools. Returns: str: The tool's unique name (e.g., "review_code", "analyze") """ pass @abstractmethod def get_description(self) -> str: """ Return a detailed description of what this tool does. This description is shown to MCP clients (like Claude / Codex / Gemini) to help them understand when and how to use the tool. It should be comprehensive and include trigger phrases. Returns: str: Detailed tool description with usage examples """ pass @abstractmethod def get_input_schema(self) -> dict[str, Any]: """ Return the JSON Schema that defines this tool's parameters. This schema is used by MCP clients to validate inputs before sending requests. It should match the tool's request model. Returns: Dict[str, Any]: JSON Schema object defining required and optional parameters """ pass @abstractmethod def get_system_prompt(self) -> str: """ Return the system prompt that configures the AI model's behavior. This prompt sets the context and instructions for how the model should approach the task. It's prepended to the user's request. Returns: str: System prompt with role definition and instructions """ pass def get_capability_system_prompts(self, capabilities: Optional["ModelCapabilities"]) -> list[str]: """Return additional system prompt snippets gated on model capabilities. Subclasses can override this hook to append capability-specific instructions (for example, enabling code-generation exports when a model advertises support). The default implementation returns an empty list so no extra instructions are appended. Args: capabilities: The resolved capabilities for the active model. Returns: List of prompt fragments to append after the base system prompt. """ return [] def _augment_system_prompt_with_capabilities( self, base_prompt: str, capabilities: Optional["ModelCapabilities"] ) -> str: """Merge capability-driven prompt addenda with the base system prompt.""" additions: list[str] = [] if capabilities is not None: additions = [fragment.strip() for fragment in self.get_capability_system_prompts(capabilities) if fragment] if not additions: return base_prompt addition_text = "\n\n".join(additions) if not base_prompt: return addition_text suffix = "" if base_prompt.endswith("\n\n") else "\n\n" return f"{base_prompt}{suffix}{addition_text}" def get_annotations(self) -> Optional[dict[str, Any]]: """ Return optional annotations for this tool. Annotations provide hints about tool behavior without being security-critical. They help MCP clients make better decisions about tool usage. Returns: Optional[dict]: Dictionary with annotation fields like readOnlyHint, destructiveHint, etc. Returns None if no annotations are needed. """ return None def requires_model(self) -> bool: """ Return whether this tool requires AI model access. Tools that override execute() to do pure data processing (like planner) should return False to skip model resolution at the MCP boundary. Returns: bool: True if tool needs AI model access (default), False for data-only tools """ return True def is_effective_auto_mode(self) -> bool: """ Check if we're in effective auto mode for schema generation. This determines whether the model parameter should be required in the tool schema. Used at initialization time when schemas are generated. Returns: bool: True if model parameter should be required in the schema """ from config import DEFAULT_MODEL from providers.registry import ModelProviderRegistry # Case 1: Explicit auto mode if DEFAULT_MODEL.lower() == "auto": return True # Case 2: Model not available (fallback to auto mode) if DEFAULT_MODEL.lower() != "auto": provider = ModelProviderRegistry.get_provider_for_model(DEFAULT_MODEL) if not provider: return True return False def _should_require_model_selection(self, model_name: str) -> bool: """ Check if we should require the CLI to select a model at runtime. This is called during request execution to determine if we need to return an error asking the CLI to provide a model parameter. Args: model_name: The model name from the request or DEFAULT_MODEL Returns: bool: True if we should require model selection """ # Case 1: Model is explicitly "auto" if model_name.lower() == "auto": return True # Case 2: Requested model is not available from providers.registry import ModelProviderRegistry provider = ModelProviderRegistry.get_provider_for_model(model_name) if not provider: logger = logging.getLogger(f"tools.{self.name}") logger.warning(f"Model '{model_name}' is not available with current API keys. Requiring model selection.") return True return False def _get_available_models(self) -> list[str]: """ Get list of models available from enabled providers. Only returns models from providers that have valid API keys configured. This fixes the namespace collision bug where models from disabled providers were shown to the CLI, causing routing conflicts. Returns: List of model names from enabled providers only """ from providers.registry import ModelProviderRegistry # Get models from enabled providers only (those with valid API keys) all_models = ModelProviderRegistry.get_available_model_names() # Add OpenRouter models if OpenRouter is configured openrouter_key = get_env("OPENROUTER_API_KEY") if openrouter_key and openrouter_key != "your_openrouter_api_key_here": try: registry = self._get_openrouter_registry() # Add all aliases from the registry (includes OpenRouter cloud models) for alias in registry.list_aliases(): if alias not in all_models: all_models.append(alias) except Exception as e: import logging logging.debug(f"Failed to add OpenRouter models to enum: {e}") # Add custom models if custom API is configured custom_url = get_env("CUSTOM_API_URL") if custom_url: try: registry = self._get_custom_registry() for alias in registry.list_aliases(): if alias not in all_models: all_models.append(alias) except Exception as e: import logging logging.debug(f"Failed to add custom models to enum: {e}") # Remove duplicates while preserving order seen = set() unique_models = [] for model in all_models: if model not in seen: seen.add(model) unique_models.append(model) return unique_models def _format_available_models_list(self) -> str: """Return a human-friendly list of available models or guidance when none found.""" summaries, total, has_restrictions = self._get_ranked_model_summaries() if not summaries: return ( "No models detected. Configure provider credentials or set DEFAULT_MODEL to a valid option. " "If the user requested a specific model, respond with this notice instead of substituting another model." ) display = "; ".join(summaries) remainder = total - len(summaries) if remainder > 0: display = f"{display}; +{remainder} more (use the `listmodels` tool for the full roster)" return display @staticmethod def _format_context_window(tokens: int) -> Optional[str]: """Convert a raw context window into a short display string.""" if not tokens or tokens <= 0: return None if tokens >= 1_000_000: if tokens % 1_000_000 == 0: return f"{tokens // 1_000_000}M ctx" return f"{tokens / 1_000_000:.1f}M ctx" if tokens >= 1_000: if tokens % 1_000 == 0: return f"{tokens // 1_000}K ctx" return f"{tokens / 1_000:.1f}K ctx" return f"{tokens} ctx" def _collect_ranked_capabilities(self) -> list[tuple[int, str, Any]]: """Gather available model capabilities sorted by capability rank.""" from providers.registry import ModelProviderRegistry ranked: list[tuple[int, str, Any]] = [] available = ModelProviderRegistry.get_available_models(respect_restrictions=True) for model_name, provider_type in available.items(): provider = ModelProviderRegistry.get_provider(provider_type) if not provider: continue try: capabilities = provider.get_capabilities(model_name) except ValueError: continue rank = capabilities.get_effective_capability_rank() ranked.append((rank, model_name, capabilities)) ranked.sort(key=lambda item: (-item[0], item[1])) return ranked @staticmethod def _normalize_model_identifier(name: str) -> str: """Normalize model names for deduplication across providers.""" normalized = name.lower() if ":" in normalized: normalized = normalized.split(":", 1)[0] if "/" in normalized: normalized = normalized.split("/", 1)[-1] return normalized def _get_ranked_model_summaries(self, limit: int = 5) -> tuple[list[str], int, bool]: """Return formatted, ranked model summaries and restriction status.""" ranked = self._collect_ranked_capabilities() # Build allowlist map (provider -> lowercase names) when restrictions are active allowed_map: dict[Any, set[str]] = {} try: from utils.model_restrictions import get_restriction_service restriction_service = get_restriction_service() if restriction_service: from providers.shared import ProviderType for provider_type in ProviderType: allowed = restriction_service.get_allowed_models(provider_type) if allowed: allowed_map[provider_type] = {name.lower() for name in allowed if name} except Exception: allowed_map = {} filtered: list[tuple[int, str, Any]] = [] seen_normalized: set[str] = set() for rank, model_name, capabilities in ranked: canonical_name = getattr(capabilities, "model_name", model_name) canonical_lower = canonical_name.lower() alias_lower = model_name.lower() provider_type = getattr(capabilities, "provider", None) if allowed_map: if provider_type not in allowed_map: continue allowed_set = allowed_map[provider_type] if canonical_lower not in allowed_set and alias_lower not in allowed_set: continue normalized = self._normalize_model_identifier(canonical_name) if normalized in seen_normalized: continue seen_normalized.add(normalized) filtered.append((rank, canonical_name, capabilities)) summaries: list[str] = [] for rank, canonical_name, capabilities in filtered[:limit]: details: list[str] = [] context_str = self._format_context_window(capabilities.context_window) if context_str: details.append(context_str) if capabilities.supports_extended_thinking: details.append("thinking") if capabilities.allow_code_generation: details.append("code-gen") base = f"{canonical_name} (score {rank}" if details: base = f"{base}, {', '.join(details)}" summaries.append(f"{base})") return summaries, len(filtered), bool(allowed_map) def _get_restriction_note(self) -> Optional[str]: """Return a string describing active per-provider allowlists, if any.""" env_labels = { "OPENAI_ALLOWED_MODELS": "OpenAI", "GOOGLE_ALLOWED_MODELS": "Google", "XAI_ALLOWED_MODELS": "X.AI", "OPENROUTER_ALLOWED_MODELS": "OpenRouter", "DIAL_ALLOWED_MODELS": "DIAL", } notes: list[str] = [] for env_var, label in env_labels.items(): raw = get_env(env_var) if not raw: continue models = sorted({token.strip() for token in raw.split(",") if token.strip()}) if not models: continue notes.append(f"{label}: {', '.join(models)}") if not notes: return None return "Policy allows only → " + "; ".join(notes) def _build_model_unavailable_message(self, model_name: str) -> str: """Compose a consistent error message for unavailable model scenarios.""" tool_category = self.get_model_category() suggested_model = ModelProviderRegistry.get_preferred_fallback_model(tool_category) available_models_text = self._format_available_models_list() return ( f"Model '{model_name}' is not available with current API keys. " f"Available models: {available_models_text}. " f"Suggested model for {self.get_name()}: '{suggested_model}' " f"(category: {tool_category.value}). If the user explicitly requested a model, you MUST use that exact name or report this error back—do not substitute another model." ) def _build_auto_mode_required_message(self) -> str: """Compose the auto-mode prompt when an explicit model selection is required.""" tool_category = self.get_model_category() suggested_model = ModelProviderRegistry.get_preferred_fallback_model(tool_category) available_models_text = self._format_available_models_list() return ( "Model parameter is required in auto mode. " f"Available models: {available_models_text}. " f"Suggested model for {self.get_name()}: '{suggested_model}' " f"(category: {tool_category.value}). When the user names a model, relay that exact name—never swap in another option." ) def get_model_field_schema(self) -> dict[str, Any]: """ Generate the model field schema based on auto mode configuration. When auto mode is enabled, the model parameter becomes required and includes detailed descriptions of each model's capabilities. Returns: Dict containing the model field JSON schema """ from config import DEFAULT_MODEL # Use the centralized effective auto mode check if self.is_effective_auto_mode(): description = ( "Currently in auto model selection mode. CRITICAL: When the user names a model, you MUST use that exact name unless the server rejects it. " "If no model is provided, you may use the `listmodels` tool to review options and select an appropriate match." ) summaries, total, restricted = self._get_ranked_model_summaries() remainder = max(0, total - len(summaries)) if summaries: top_line = "; ".join(summaries) if remainder > 0: label = "Allowed models" if restricted else "Top models" top_line = f"{label}: {top_line}; +{remainder} more via `listmodels`." else: label = "Allowed models" if restricted else "Top models" top_line = f"{label}: {top_line}." description = f"{description} {top_line}" restriction_note = self._get_restriction_note() if restriction_note and (remainder > 0 or not summaries): description = f"{description} {restriction_note}." return { "type": "string", "description": description, } description = ( f"The default model is '{DEFAULT_MODEL}'. Override only when the user explicitly requests a different model, and use that exact name. " "If the requested model fails validation, surface the server error instead of substituting another model. When unsure, use the `listmodels` tool for details." ) summaries, total, restricted = self._get_ranked_model_summaries() remainder = max(0, total - len(summaries)) if summaries: top_line = "; ".join(summaries) if remainder > 0: label = "Allowed models" if restricted else "Preferred alternatives" top_line = f"{label}: {top_line}; +{remainder} more via `listmodels`." else: label = "Allowed models" if restricted else "Preferred alternatives" top_line = f"{label}: {top_line}." description = f"{description} {top_line}" restriction_note = self._get_restriction_note() if restriction_note and (remainder > 0 or not summaries): description = f"{description} {restriction_note}." return { "type": "string", "description": description, } def get_default_temperature(self) -> float: """ Return the default temperature setting for this tool. Override this method to set tool-specific temperature defaults. Lower values (0.0-0.3) for analytical tasks, higher (0.7-1.0) for creative tasks. Returns: float: Default temperature between 0.0 and 1.0 """ return 0.5 def wants_line_numbers_by_default(self) -> bool: """ Return whether this tool wants line numbers added to code files by default. By default, ALL tools get line numbers for precise code references. Line numbers are essential for accurate communication about code locations. Returns: bool: True if line numbers should be added by default for this tool """ return True # All tools get line numbers by default for consistency def get_default_thinking_mode(self) -> str: """ Return the default thinking mode for this tool. Thinking mode controls computational budget for reasoning. Override for tools that need more or less reasoning depth. Returns: str: One of "minimal", "low", "medium", "high", "max" """ return "medium" # Default to medium thinking for better reasoning def get_model_category(self) -> "ToolModelCategory": """ Return the model category for this tool. Model category influences which model is selected in auto mode. Override to specify whether your tool needs extended reasoning, fast response, or balanced capabilities. Returns: ToolModelCategory: Category that influences model selection """ from tools.models import ToolModelCategory return ToolModelCategory.BALANCED @abstractmethod def get_request_model(self): """ Return the Pydantic model class used for validating requests. This model should inherit from ToolRequest and define all parameters specific to this tool. Returns: Type[ToolRequest]: The request model class """ pass def validate_file_paths(self, request) -> Optional[str]: """ Validate that all file paths in the request are absolute. This is a critical security function that prevents path traversal attacks and ensures all file access is properly controlled. All file paths must be absolute to avoid ambiguity and security issues. Args: request: The validated request object Returns: Optional[str]: Error message if validation fails, None if all paths are valid """ # Only validate files/paths if they exist in the request file_fields = [ "absolute_file_paths", "file", "path", "directory", "notebooks", "test_examples", "style_guide_examples", "files_checked", "relevant_files", ] for field_name in file_fields: if hasattr(request, field_name): field_value = getattr(request, field_name) if field_value is None: continue # Handle both single paths and lists of paths paths_to_check = field_value if isinstance(field_value, list) else [field_value] for path in paths_to_check: if path and not os.path.isabs(path): return f"All file paths must be FULL absolute paths. Invalid path: '{path}'" return None def _validate_token_limit(self, content: str, content_type: str = "Content") -> None: """ Validate that user-provided content doesn't exceed the MCP prompt size limit. This enforcement is strictly for text crossing the MCP transport boundary (i.e., user input). Internal prompt construction may exceed this size and is governed by model-specific token limits. Args: content: The user-originated content to validate content_type: Description of the content type for error messages Raises: ValueError: If content exceeds the character size limit """ if not content: logger.debug(f"{self.name} tool {content_type.lower()} validation skipped (no content)") return char_count = len(content) if char_count > MCP_PROMPT_SIZE_LIMIT: token_estimate = estimate_tokens(content) error_msg = ( f"{char_count:,} characters (~{token_estimate:,} tokens). " f"Maximum is {MCP_PROMPT_SIZE_LIMIT:,} characters." ) logger.error(f"{self.name} tool {content_type.lower()} validation failed: {error_msg}") raise ValueError(f"{content_type} too large: {error_msg}") token_estimate = estimate_tokens(content) logger.debug( f"{self.name} tool {content_type.lower()} validation passed: " f"{char_count:,} characters (~{token_estimate:,} tokens)" ) def get_model_provider(self, model_name: str) -> ModelProvider: """ Get the appropriate model provider for the given model name. This method performs runtime validation to ensure the requested model is actually available with the current API key configuration. Args: model_name: Name of the model to get provider for Returns: ModelProvider: The provider instance for the model Raises: ValueError: If the model is not available or provider not found """ try: provider = ModelProviderRegistry.get_provider_for_model(model_name) if not provider: logger.error(f"No provider found for model '{model_name}' in {self.name} tool") raise ValueError(self._build_model_unavailable_message(model_name)) return provider except Exception as e: logger.error(f"Failed to get provider for model '{model_name}' in {self.name} tool: {e}") raise # === CONVERSATION AND FILE HANDLING METHODS === def get_conversation_embedded_files(self, continuation_id: Optional[str]) -> list[str]: """ Get list of files already embedded in conversation history. This method returns the list of files that have already been embedded in the conversation history for a given continuation thread. Tools can use this to avoid re-embedding files that are already available in the conversation context. Args: continuation_id: Thread continuation ID, or None for new conversations Returns: list[str]: List of file paths already embedded in conversation history """ if not continuation_id: # New conversation, no files embedded yet return [] thread_context = get_thread(continuation_id) if not thread_context: # Thread not found, no files embedded return [] embedded_files = get_conversation_file_list(thread_context) logger.debug(f"[FILES] {self.name}: Found {len(embedded_files)} embedded files") return embedded_files def filter_new_files(self, requested_files: list[str], continuation_id: Optional[str]) -> list[str]: """ Filter out files that are already embedded in conversation history. This method prevents duplicate file embeddings by filtering out files that have already been embedded in the conversation history. This optimizes token usage while ensuring tools still have logical access to all requested files through conversation history references. Args: requested_files: List of files requested for current tool execution continuation_id: Thread continuation ID, or None for new conversations Returns: list[str]: List of files that need to be embedded (not already in history) """ logger.debug(f"[FILES] {self.name}: Filtering {len(requested_files)} requested files") if not continuation_id: # New conversation, all files are new logger.debug(f"[FILES] {self.name}: New conversation, all {len(requested_files)} files are new") return requested_files try: embedded_files = set(self.get_conversation_embedded_files(continuation_id)) logger.debug(f"[FILES] {self.name}: Found {len(embedded_files)} embedded files in conversation") # Safety check: If no files are marked as embedded but we have a continuation_id, # this might indicate an issue with conversation history. Be conservative. if not embedded_files: logger.debug(f"{self.name} tool: No files found in conversation history for thread {continuation_id}") logger.debug( f"[FILES] {self.name}: No embedded files found, returning all {len(requested_files)} requested files" ) return requested_files # Return only files that haven't been embedded yet new_files = [f for f in requested_files if f not in embedded_files] logger.debug( f"[FILES] {self.name}: After filtering: {len(new_files)} new files, {len(requested_files) - len(new_files)} already embedded" ) logger.debug(f"[FILES] {self.name}: New files to embed: {new_files}") # Log filtering results for debugging if len(new_files) < len(requested_files): skipped = [f for f in requested_files if f in embedded_files] logger.debug( f"{self.name} tool: Filtering {len(skipped)} files already in conversation history: {', '.join(skipped)}" ) logger.debug(f"[FILES] {self.name}: Skipped (already embedded): {skipped}") return new_files except Exception as e: # If there's any issue with conversation history lookup, be conservative # and include all files rather than risk losing access to needed files logger.warning(f"{self.name} tool: Error checking conversation history for {continuation_id}: {e}") logger.warning(f"{self.name} tool: Including all requested files as fallback") logger.debug( f"[FILES] {self.name}: Exception in filter_new_files, returning all {len(requested_files)} files as fallback" ) return requested_files def format_conversation_turn(self, turn: ConversationTurn) -> list[str]: """ Format a conversation turn for display in conversation history. Tools can override this to provide custom formatting for their responses while maintaining the standard structure for cross-tool compatibility. This method is called by build_conversation_history when reconstructing conversation context, allowing each tool to control how its responses appear in subsequent conversation turns. Args: turn: The conversation turn to format (from utils.conversation_memory) Returns: list[str]: Lines of formatted content for this turn Example: Default implementation returns: ["Files used in this turn: file1.py, file2.py", "", "Response content..."] Tools can override to add custom sections, formatting, or metadata display. """ parts = [] # Add files context if present if turn.files: parts.append(f"Files used in this turn: {', '.join(turn.files)}") parts.append("") # Empty line for readability # Add the actual content parts.append(turn.content) return parts def handle_prompt_file(self, files: Optional[list[str]]) -> tuple[Optional[str], Optional[list[str]]]: """ Check for and handle prompt.txt in the absolute file paths list. If prompt.txt is found, reads its content and removes it from the files list. This file is treated specially as the main prompt, not as an embedded file. This mechanism allows us to work around MCP's ~25K token limit by having the CLI save large prompts to a file, effectively using the file transfer mechanism to bypass token constraints while preserving response capacity. Args: files: List of absolute file paths (will be translated for current environment) Returns: tuple: (prompt_content, updated_files_list) """ if not files: return None, files prompt_content = None updated_files = [] for file_path in files: # Check if the filename is exactly "prompt.txt" # This ensures we don't match files like "myprompt.txt" or "prompt.txt.bak" if os.path.basename(file_path) == "prompt.txt": try: # Read prompt.txt content and extract just the text content, _ = read_file_content(file_path) # Extract the content between the file markers if "--- BEGIN FILE:" in content and "--- END FILE:" in content: lines = content.split("\n") in_content = False content_lines = [] for line in lines: if line.startswith("--- BEGIN FILE:"): in_content = True continue elif line.startswith("--- END FILE:"): break elif in_content: content_lines.append(line) prompt_content = "\n".join(content_lines) else: # Fallback: if it's already raw content (from tests or direct input) # and doesn't have error markers, use it directly if not content.startswith("\n--- ERROR"): prompt_content = content else: prompt_content = None except Exception: # If we can't read the file, we'll just skip it # The error will be handled elsewhere pass else: # Keep the original path in the files list (will be translated later by read_files) updated_files.append(file_path) return prompt_content, updated_files if updated_files else None def get_prompt_content_for_size_validation(self, user_content: str) -> str: """ Get the content that should be validated for MCP prompt size limits. This hook method allows tools to specify what content should be checked against the MCP transport size limit. By default, it returns the user content, but can be overridden to exclude conversation history when needed. Args: user_content: The user content that would normally be validated Returns: The content that should actually be validated for size limits """ # Default implementation: validate the full user content return user_content def check_prompt_size(self, text: str) -> Optional[dict[str, Any]]: """ Check if USER INPUT text is too large for MCP transport boundary. IMPORTANT: This method should ONLY be used to validate user input that crosses the CLI ↔ MCP Server transport boundary. It should NOT be used to limit internal MCP Server operations. Args: text: The user input text to check (NOT internal prompt content) Returns: Optional[Dict[str, Any]]: Response asking for file handling if too large, None otherwise """ if text and len(text) > MCP_PROMPT_SIZE_LIMIT: return { "status": "resend_prompt", "content": ( f"MANDATORY ACTION REQUIRED: The prompt is too large for MCP's token limits (>{MCP_PROMPT_SIZE_LIMIT:,} characters). " "YOU MUST IMMEDIATELY save the prompt text to a temporary file named 'prompt.txt' in the working directory. " "DO NOT attempt to shorten or modify the prompt. SAVE IT AS-IS to 'prompt.txt'. " "Then resend the request, passing the absolute file path to 'prompt.txt' as part of the tool call, " "along with any other files you wish to share as context. Leave the prompt text itself empty or very brief in the new request. " "This is the ONLY way to handle large prompts - you MUST follow these exact steps." ), "content_type": "text", "metadata": { "prompt_size": len(text), "limit": MCP_PROMPT_SIZE_LIMIT, "instructions": "MANDATORY: Save prompt to 'prompt.txt' in current folder and provide full path when recalling this tool.", }, } return None def _prepare_file_content_for_prompt( self, request_files: list[str], continuation_id: Optional[str], context_description: str = "New files", max_tokens: Optional[int] = None, reserve_tokens: int = 1_000, remaining_budget: Optional[int] = None, arguments: Optional[dict] = None, model_context: Optional[Any] = None, ) -> tuple[str, list[str]]: """ Centralized file processing implementing dual prioritization strategy. This method is the heart of conversation-aware file processing across all tools. Args: request_files: List of files requested for current tool execution continuation_id: Thread continuation ID, or None for new conversations context_description: Description for token limit validation (e.g. "Code", "New files") max_tokens: Maximum tokens to use (defaults to remaining budget or model-specific content allocation) reserve_tokens: Tokens to reserve for additional prompt content (default 1K) remaining_budget: Remaining token budget after conversation history (from server.py) arguments: Original tool arguments (used to extract _remaining_tokens if available) model_context: Model context object with all model information including token allocation Returns: tuple[str, list[str]]: (formatted_file_content, actually_processed_files) - formatted_file_content: Formatted file content string ready for prompt inclusion - actually_processed_files: List of individual file paths that were actually read and embedded (directories are expanded to individual files) """ if not request_files: return "", [] # Extract remaining budget from arguments if available if remaining_budget is None: # Use provided arguments or fall back to stored arguments from execute() args_to_use = arguments or getattr(self, "_current_arguments", {}) remaining_budget = args_to_use.get("_remaining_tokens") # Use remaining budget if provided, otherwise fall back to max_tokens or model-specific default if remaining_budget is not None: effective_max_tokens = remaining_budget - reserve_tokens elif max_tokens is not None: effective_max_tokens = max_tokens - reserve_tokens else: # Use model_context for token allocation if not model_context: # Try to get from stored attributes as fallback model_context = getattr(self, "_model_context", None) if not model_context: logger.error( f"[FILES] {self.name}: _prepare_file_content_for_prompt called without model_context. " "This indicates an incorrect call sequence in the tool's implementation." ) raise RuntimeError("Model context not provided for file preparation.") # This is now the single source of truth for token allocation. try: token_allocation = model_context.calculate_token_allocation() # Standardize on `file_tokens` for consistency and correctness. effective_max_tokens = token_allocation.file_tokens - reserve_tokens logger.debug( f"[FILES] {self.name}: Using model context for {model_context.model_name}: " f"{token_allocation.file_tokens:,} file tokens from {token_allocation.total_tokens:,} total" ) except Exception as e: logger.error( f"[FILES] {self.name}: Failed to calculate token allocation from model context: {e}", exc_info=True ) # If the context exists but calculation fails, we still need to prevent a crash. # A loud error is logged, and we fall back to a safe default. effective_max_tokens = 100_000 - reserve_tokens # Ensure we have a reasonable minimum budget effective_max_tokens = max(1000, effective_max_tokens) files_to_embed = self.filter_new_files(request_files, continuation_id) logger.debug(f"[FILES] {self.name}: Will embed {len(files_to_embed)} files after filtering") # Log the specific files for debugging/testing if files_to_embed: logger.info( f"[FILE_PROCESSING] {self.name} tool will embed new files: {', '.join([os.path.basename(f) for f in files_to_embed])}" ) else: logger.info( f"[FILE_PROCESSING] {self.name} tool: No new files to embed (all files already in conversation history)" ) content_parts = [] actually_processed_files = [] # Read content of new files only if files_to_embed: logger.debug(f"{self.name} tool embedding {len(files_to_embed)} new files: {', '.join(files_to_embed)}") logger.debug( f"[FILES] {self.name}: Starting file embedding with token budget {effective_max_tokens + reserve_tokens:,}" ) try: # Before calling read_files, expand directories to get individual file paths from utils.file_utils import expand_paths expanded_files = expand_paths(files_to_embed) logger.debug( f"[FILES] {self.name}: Expanded {len(files_to_embed)} paths to {len(expanded_files)} individual files" ) file_content = read_files( files_to_embed, max_tokens=effective_max_tokens + reserve_tokens, reserve_tokens=reserve_tokens, include_line_numbers=self.wants_line_numbers_by_default(), ) # Note: No need to validate against MCP_PROMPT_SIZE_LIMIT here # read_files already handles token-aware truncation based on model's capabilities content_parts.append(file_content) # Track the expanded files as actually processed actually_processed_files.extend(expanded_files) # Estimate tokens for debug logging from utils.token_utils import estimate_tokens content_tokens = estimate_tokens(file_content) logger.debug( f"{self.name} tool successfully embedded {len(files_to_embed)} files ({content_tokens:,} tokens)" ) logger.debug(f"[FILES] {self.name}: Successfully embedded files - {content_tokens:,} tokens used") logger.debug( f"[FILES] {self.name}: Actually processed {len(actually_processed_files)} individual files" ) except Exception as e: logger.error(f"{self.name} tool failed to embed files {files_to_embed}: {type(e).__name__}: {e}") logger.debug(f"[FILES] {self.name}: File embedding failed - {type(e).__name__}: {e}") raise else: logger.debug(f"[FILES] {self.name}: No files to embed after filtering") # Generate note about files already in conversation history if continuation_id and len(files_to_embed) < len(request_files): embedded_files = self.get_conversation_embedded_files(continuation_id) skipped_files = [f for f in request_files if f in embedded_files] if skipped_files: logger.debug( f"{self.name} tool skipping {len(skipped_files)} files already in conversation history: {', '.join(skipped_files)}" ) logger.debug(f"[FILES] {self.name}: Adding note about {len(skipped_files)} skipped files") if content_parts: content_parts.append("\n\n") note_lines = [ "--- NOTE: Additional files referenced in conversation history ---", "The following files are already available in our conversation context:", "\n".join(f" - {f}" for f in skipped_files), "--- END NOTE ---", ] content_parts.append("\n".join(note_lines)) else: logger.debug(f"[FILES] {self.name}: No skipped files to note") result = "".join(content_parts) if content_parts else "" logger.debug( f"[FILES] {self.name}: _prepare_file_content_for_prompt returning {len(result)} chars, {len(actually_processed_files)} processed files" ) return result, actually_processed_files def get_websearch_instruction(self, tool_specific: Optional[str] = None) -> str: """ Generate standardized web search instruction. Args: tool_specific: Optional tool-specific search guidance Returns: str: Web search instruction to append to prompt """ base_instruction = """ WEB SEARCH CAPABILITY: You can request the calling agent to perform web searches to enhance your analysis with current information! IMPORTANT: When you identify areas where web searches would significantly improve your response (such as checking current documentation, finding recent solutions, verifying best practices, or gathering community insights), you MUST explicitly instruct the agent to perform specific web searches and then respond back using the continuation_id from this response to continue the analysis. Use clear, direct language based on the value of the search: For valuable supplementary information: "Please perform a web search on '[specific topic/query]' and then continue this analysis using the continuation_id from this response if you find relevant information." For important missing information: "Please search for '[specific topic/query]' and respond back with the findings using the continuation_id from this response - this information is needed to provide a complete analysis." For critical/essential information: "SEARCH REQUIRED: Please immediately perform a web search on '[specific topic/query]' and respond back with the results using the continuation_id from this response. Cannot provide accurate analysis without this current information." This ensures you get the most current and comprehensive information while maintaining conversation context through the continuation_id.""" if tool_specific: return f"""{base_instruction} {tool_specific} When recommending searches, be specific about what information you need and why it would improve your analysis.""" # Default instruction for all tools return f"""{base_instruction} Consider requesting searches for: - Current documentation and API references - Recent best practices and patterns - Known issues and community solutions - Framework updates and compatibility - Security advisories and patches - Performance benchmarks and optimizations When recommending searches, be specific about what information you need and why it would improve your analysis. Always remember to instruct agent to use the continuation_id from this response when providing search results.""" def get_language_instruction(self) -> str: """ Generate language instruction based on LOCALE configuration. Returns: str: Language instruction to prepend to prompt, or empty string if no locale set """ # Read LOCALE directly from environment to support dynamic changes # Tests can monkeypatch LOCALE via the environment helper (or .env when override is enforced) locale = (get_env("LOCALE", "") or "").strip() if not locale: return "" # Simple language instruction return f"Always respond in {locale}.\n\n" # === ABSTRACT METHODS FOR SIMPLE TOOLS === @abstractmethod async def prepare_prompt(self, request) -> str: """ Prepare the complete prompt for the AI model. This method should construct the full prompt by combining: - System prompt from get_system_prompt() - File content from _prepare_file_content_for_prompt() - Conversation history from reconstruct_thread_context() - User's request and any tool-specific context Args: request: The validated request object Returns: str: Complete prompt ready for the AI model """ pass def format_response(self, response: str, request, model_info: dict = None) -> str: """ Format the AI model's response for the user. This method allows tools to post-process the model's response, adding structure, validation, or additional context. The default implementation returns the response unchanged. Tools can override this method to add custom formatting. Args: response: Raw response from the AI model request: The original request object model_info: Optional model information and metadata Returns: str: Formatted response ready for the user """ return response # === IMPLEMENTATION METHODS === # These will be provided in a full implementation but are inherited from current base.py # for now to maintain compatibility. async def execute(self, arguments: dict[str, Any]) -> list[TextContent]: """Execute the tool - will be inherited from existing base.py for now.""" # This will be implemented by importing from the current base.py # for backward compatibility during the migration raise NotImplementedError("Subclasses must implement execute method") def _should_require_model_selection(self, model_name: str) -> bool: """ Check if we should require the CLI to select a model at runtime. This is called during request execution to determine if we need to return an error asking the CLI to provide a model parameter. Args: model_name: The model name from the request or DEFAULT_MODEL Returns: bool: True if we should require model selection """ # Case 1: Model is explicitly "auto" if model_name.lower() == "auto": return True # Case 2: Requested model is not available from providers.registry import ModelProviderRegistry provider = ModelProviderRegistry.get_provider_for_model(model_name) if not provider: logger.warning(f"Model '{model_name}' is not available with current API keys. Requiring model selection.") return True return False def _get_available_models(self) -> list[str]: """ Get list of models available from enabled providers. Only returns models from providers that have valid API keys configured. This fixes the namespace collision bug where models from disabled providers were shown to the CLI, causing routing conflicts. Returns: List of model names from enabled providers only """ from providers.registry import ModelProviderRegistry # Get models from enabled providers only (those with valid API keys) all_models = ModelProviderRegistry.get_available_model_names() # Add OpenRouter models and their aliases when OpenRouter is configured openrouter_key = get_env("OPENROUTER_API_KEY") if openrouter_key and openrouter_key != "your_openrouter_api_key_here": try: registry = self._get_openrouter_registry() for alias in registry.list_aliases(): if alias not in all_models: all_models.append(alias) except Exception as exc: # pragma: no cover - logged for observability import logging logging.debug(f"Failed to add OpenRouter models to enum: {exc}") # Add custom models (and their aliases) when a custom endpoint is available custom_url = get_env("CUSTOM_API_URL") if custom_url: try: registry = self._get_custom_registry() for alias in registry.list_aliases(): if alias not in all_models: all_models.append(alias) except Exception as exc: # pragma: no cover - logged for observability import logging logging.debug(f"Failed to add custom models to enum: {exc}") # Remove duplicates while preserving insertion order seen: set[str] = set() unique_models: list[str] = [] for model in all_models: if model not in seen: seen.add(model) unique_models.append(model) return unique_models def _resolve_model_context(self, arguments: dict, request) -> tuple[str, Any]: """ Resolve model context and name using centralized logic. This method extracts the model resolution logic from execute() so it can be reused by tools that override execute() (like debug tool) without duplicating code. Args: arguments: Dictionary of arguments from the MCP client request: The validated request object Returns: tuple[str, ModelContext]: (resolved_model_name, model_context) Raises: ValueError: If model resolution fails or model selection is required """ # MODEL RESOLUTION NOW HAPPENS AT MCP BOUNDARY # Extract pre-resolved model context from server.py model_context = arguments.get("_model_context") resolved_model_name = arguments.get("_resolved_model_name") if model_context and resolved_model_name: # Model was already resolved at MCP boundary model_name = resolved_model_name logger.debug(f"Using pre-resolved model '{model_name}' from MCP boundary") else: # Fallback for direct execute calls model_name = getattr(request, "model", None) if not model_name: from config import DEFAULT_MODEL model_name = DEFAULT_MODEL logger.debug(f"Using fallback model resolution for '{model_name}' (test mode)") # For tests: Check if we should require model selection (auto mode) if self._should_require_model_selection(model_name): # Build error message based on why selection is required if model_name.lower() == "auto": error_message = self._build_auto_mode_required_message() else: error_message = self._build_model_unavailable_message(model_name) raise ValueError(error_message) # Create model context for tests from utils.model_context import ModelContext model_context = ModelContext(model_name) return model_name, model_context def validate_and_correct_temperature(self, temperature: float, model_context: Any) -> tuple[float, list[str]]: """ Validate and correct temperature for the specified model. This method ensures that the temperature value is within the valid range for the specific model being used. Different models have different temperature constraints (e.g., o1 models require temperature=1.0, GPT models support 0-2). Args: temperature: Temperature value to validate model_context: Model context object containing model name, provider, and capabilities Returns: Tuple of (corrected_temperature, warning_messages) """ try: # Use model context capabilities directly - clean OOP approach capabilities = model_context.capabilities constraint = capabilities.temperature_constraint warnings = [] if not constraint.validate(temperature): corrected = constraint.get_corrected_value(temperature) warning = ( f"Temperature {temperature} invalid for {model_context.model_name}. " f"{constraint.get_description()}. Using {corrected} instead." ) warnings.append(warning) return corrected, warnings return temperature, warnings except Exception as e: # If validation fails for any reason, use the original temperature # and log a warning (but don't fail the request) logger.warning(f"Temperature validation failed for {model_context.model_name}: {e}") return temperature, [f"Temperature validation failed: {e}"] def _validate_image_limits( self, images: Optional[list[str]], model_context: Optional[Any] = None, continuation_id: Optional[str] = None ) -> Optional[dict]: """ Validate image size and count against model capabilities. This performs strict validation to ensure we don't exceed model-specific image limits. Uses capability-based validation with actual model configuration rather than hard-coded limits. Args: images: List of image paths/data URLs to validate model_context: Model context object containing model name, provider, and capabilities continuation_id: Optional continuation ID for conversation context Returns: Optional[dict]: Error response if validation fails, None if valid """ if not images: return None # Import here to avoid circular imports import base64 from pathlib import Path if not model_context: # Get from tool's stored context as fallback model_context = getattr(self, "_model_context", None) if not model_context: logger.warning("No model context available for image validation") return None try: # Use model context capabilities directly - clean OOP approach capabilities = model_context.capabilities model_name = model_context.model_name except Exception as e: logger.warning(f"Failed to get capabilities from model_context for image validation: {e}") # Generic error response when capabilities cannot be accessed model_name = getattr(model_context, "model_name", "unknown") return { "status": "error", "content": self._build_model_unavailable_message(model_name), "content_type": "text", "metadata": { "error_type": "validation_error", "model_name": model_name, "supports_images": None, # Unknown since model capabilities unavailable "image_count": len(images) if images else 0, }, } # Check if model supports images if not capabilities.supports_images: return { "status": "error", "content": ( f"Image support not available: Model '{model_name}' does not support image processing. " f"Please use a vision-capable model such as 'gemini-2.5-flash', 'o3', " f"or 'claude-opus-4.1' for image analysis tasks." ), "content_type": "text", "metadata": { "error_type": "validation_error", "model_name": model_name, "supports_images": False, "image_count": len(images), }, } # Get model image limits from capabilities max_images = 5 # Default max number of images max_size_mb = capabilities.max_image_size_mb # Check image count if len(images) > max_images: return { "status": "error", "content": ( f"Too many images: Model '{model_name}' supports a maximum of {max_images} images, " f"but {len(images)} were provided. Please reduce the number of images." ), "content_type": "text", "metadata": { "error_type": "validation_error", "model_name": model_name, "image_count": len(images), "max_images": max_images, }, } # Calculate total size of all images total_size_mb = 0.0 for image_path in images: try: if image_path.startswith("data:image/"): # Handle data URL: data:image/png;base64,iVBORw0... _, data = image_path.split(",", 1) # Base64 encoding increases size by ~33%, so decode to get actual size actual_size = len(base64.b64decode(data)) total_size_mb += actual_size / (1024 * 1024) else: # Handle file path path = Path(image_path) if path.exists(): file_size = path.stat().st_size total_size_mb += file_size / (1024 * 1024) else: logger.warning(f"Image file not found: {image_path}") # Assume a reasonable size for missing files to avoid breaking validation total_size_mb += 1.0 # 1MB assumption except Exception as e: logger.warning(f"Failed to get size for image {image_path}: {e}") # Assume a reasonable size for problematic files total_size_mb += 1.0 # 1MB assumption # Apply 40MB cap for custom models if needed effective_limit_mb = max_size_mb try: from providers.shared import ProviderType # ModelCapabilities dataclass has provider field defined if capabilities.provider == ProviderType.CUSTOM: effective_limit_mb = min(max_size_mb, 40.0) except Exception: pass # Validate against size limit if total_size_mb > effective_limit_mb: return { "status": "error", "content": ( f"Image size limit exceeded: Model '{model_name}' supports maximum {effective_limit_mb:.1f}MB " f"for all images combined, but {total_size_mb:.1f}MB was provided. " f"Please reduce image sizes or count and try again." ), "content_type": "text", "metadata": { "error_type": "validation_error", "model_name": model_name, "total_size_mb": round(total_size_mb, 2), "limit_mb": round(effective_limit_mb, 2), "image_count": len(images), "supports_images": True, }, } # All validations passed logger.debug(f"Image validation passed: {len(images)} images, {total_size_mb:.1f}MB total") return None def _parse_response(self, raw_text: str, request, model_info: Optional[dict] = None): """Parse response - will be inherited for now.""" # Implementation inherited from current base.py raise NotImplementedError("Subclasses must implement _parse_response method") ================================================ FILE: tools/shared/exceptions.py ================================================ """ Custom exceptions for PAL MCP tools. These exceptions allow tools to signal protocol-level errors that should be surfaced to MCP clients using the `isError` flag on `CallToolResult`. Raising one of these exceptions ensures the low-level server adapter marks the result as an error while preserving the structured payload we pass through the exception message. """ class ToolExecutionError(RuntimeError): """Raised to indicate a tool-level failure that must set `isError=True`.""" def __init__(self, payload: str): """ Args: payload: Serialized error payload (typically JSON) to return to the client. """ super().__init__(payload) self.payload = payload ================================================ FILE: tools/shared/schema_builders.py ================================================ """ Core schema building functionality for PAL MCP tools. This module provides base schema generation functionality for simple tools. Workflow-specific schema building is located in workflow/schema_builders.py to maintain proper separation of concerns. """ from typing import Any from .base_models import COMMON_FIELD_DESCRIPTIONS class SchemaBuilder: """ Base schema builder for simple MCP tools. This class provides static methods to build consistent schemas for simple tools. Workflow tools use WorkflowSchemaBuilder in workflow/schema_builders.py. """ # Common field schemas that can be reused across all tool types COMMON_FIELD_SCHEMAS = { "temperature": { "type": "number", "description": COMMON_FIELD_DESCRIPTIONS["temperature"], "minimum": 0.0, "maximum": 1.0, }, "thinking_mode": { "type": "string", "enum": ["minimal", "low", "medium", "high", "max"], "description": COMMON_FIELD_DESCRIPTIONS["thinking_mode"], }, "continuation_id": { "type": "string", "description": COMMON_FIELD_DESCRIPTIONS["continuation_id"], }, "images": { "type": "array", "items": {"type": "string"}, "description": COMMON_FIELD_DESCRIPTIONS["images"], }, } # Simple tool-specific field schemas (workflow tools use relevant_files instead) SIMPLE_FIELD_SCHEMAS = { "absolute_file_paths": { "type": "array", "items": {"type": "string"}, "description": COMMON_FIELD_DESCRIPTIONS["absolute_file_paths"], }, } @staticmethod def build_schema( tool_specific_fields: dict[str, dict[str, Any]] = None, required_fields: list[str] = None, model_field_schema: dict[str, Any] = None, auto_mode: bool = False, require_model: bool = False, ) -> dict[str, Any]: """ Build complete schema for simple tools. Args: tool_specific_fields: Additional fields specific to the tool required_fields: List of required field names model_field_schema: Schema for the model field auto_mode: Whether the tool is in auto mode (affects model requirement) Returns: Complete JSON schema for the tool """ properties = {} # Add common fields (temperature, thinking_mode, etc.) properties.update(SchemaBuilder.COMMON_FIELD_SCHEMAS) # Add simple tool-specific fields (files field for simple tools) properties.update(SchemaBuilder.SIMPLE_FIELD_SCHEMAS) # Add model field if provided if model_field_schema: properties["model"] = model_field_schema # Add tool-specific fields if provided if tool_specific_fields: properties.update(tool_specific_fields) # Build required fields list required = list(required_fields) if required_fields else [] if (auto_mode or require_model) and "model" not in required: required.append("model") # Build the complete schema schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": properties, "additionalProperties": False, } if required: schema["required"] = required return schema @staticmethod def get_common_fields() -> dict[str, dict[str, Any]]: """Get the standard field schemas for simple tools.""" return SchemaBuilder.COMMON_FIELD_SCHEMAS.copy() @staticmethod def create_field_schema( field_type: str, description: str, enum_values: list[str] = None, minimum: float = None, maximum: float = None, items_type: str = None, default: Any = None, ) -> dict[str, Any]: """ Helper method to create field schemas with common patterns. Args: field_type: JSON schema type ("string", "number", "array", etc.) description: Human-readable description of the field enum_values: For enum fields, list of allowed values minimum: For numeric fields, minimum value maximum: For numeric fields, maximum value items_type: For array fields, type of array items default: Default value for the field Returns: JSON schema object for the field """ schema = { "type": field_type, "description": description, } if enum_values: schema["enum"] = enum_values if minimum is not None: schema["minimum"] = minimum if maximum is not None: schema["maximum"] = maximum if items_type and field_type == "array": schema["items"] = {"type": items_type} if default is not None: schema["default"] = default return schema ================================================ FILE: tools/simple/__init__.py ================================================ """ Simple tools for PAL MCP. Simple tools follow a basic request → AI model → response pattern. They inherit from SimpleTool which provides streamlined functionality for tools that don't need multi-step workflows. Available simple tools: - chat: General chat and collaborative thinking - consensus: Multi-perspective analysis - listmodels: Model listing and information - testgen: Test generation - tracer: Execution tracing """ from .base import SimpleTool __all__ = ["SimpleTool"] ================================================ FILE: tools/simple/base.py ================================================ """ Base class for simple MCP tools. Simple tools follow a straightforward pattern: 1. Receive request 2. Prepare prompt (with absolute file paths, context, etc.) 3. Call AI model 4. Format and return response They use the shared SchemaBuilder for consistent schema generation and inherit all the conversation, file processing, and model handling capabilities from BaseTool. """ from abc import abstractmethod from typing import Any, Optional from tools.shared.base_models import ToolRequest from tools.shared.base_tool import BaseTool from tools.shared.exceptions import ToolExecutionError from tools.shared.schema_builders import SchemaBuilder class SimpleTool(BaseTool): """ Base class for simple (non-workflow) tools. Simple tools are request/response tools that don't require multi-step workflows. They benefit from: - Automatic schema generation using SchemaBuilder - Inherited conversation handling and file processing - Standardized model integration - Consistent error handling and response formatting To create a simple tool: 1. Inherit from SimpleTool 2. Implement get_tool_fields() to define tool-specific fields 3. Implement prepare_prompt() for prompt preparation 4. Optionally override format_response() for custom formatting 5. Optionally override get_required_fields() for custom requirements Example: class ChatTool(SimpleTool): def get_name(self) -> str: return "chat" def get_tool_fields(self) -> Dict[str, Dict[str, Any]]: return { "prompt": { "type": "string", "description": "Your question or idea...", }, "absolute_file_paths": SimpleTool.FILES_FIELD, } def get_required_fields(self) -> List[str]: return ["prompt"] """ # Common field definitions that simple tools can reuse FILES_FIELD = SchemaBuilder.SIMPLE_FIELD_SCHEMAS["absolute_file_paths"] IMAGES_FIELD = SchemaBuilder.COMMON_FIELD_SCHEMAS["images"] @abstractmethod def get_tool_fields(self) -> dict[str, dict[str, Any]]: """ Return tool-specific field definitions. This method should return a dictionary mapping field names to their JSON schema definitions. Common fields (model, temperature, etc.) are added automatically by the base class. Returns: Dict mapping field names to JSON schema objects Example: return { "prompt": { "type": "string", "description": "The user's question or request", }, "absolute_file_paths": SimpleTool.FILES_FIELD, # Reuse common field "max_tokens": { "type": "integer", "minimum": 1, "description": "Maximum tokens for response", } } """ pass def get_required_fields(self) -> list[str]: """ Return list of required field names. Override this to specify which fields are required for your tool. The model field is automatically added if in auto mode. Returns: List of required field names """ return [] def get_annotations(self) -> Optional[dict[str, Any]]: """ Return tool annotations. Simple tools are read-only by default. All simple tools perform operations without modifying the environment. They may call external AI models for analysis or conversation, but they don't write files or make system changes. Override this method if your simple tool needs different annotations. Returns: Dictionary with readOnlyHint set to True """ return {"readOnlyHint": True} def format_response(self, response: str, request, model_info: Optional[dict] = None) -> str: """ Format the AI response before returning to the client. This is a hook method that subclasses can override to customize response formatting. The default implementation returns the response as-is. Args: response: The raw response from the AI model request: The validated request object model_info: Optional model information dictionary Returns: Formatted response string """ return response def get_input_schema(self) -> dict[str, Any]: """ Generate the complete input schema using SchemaBuilder. This method automatically combines: - Tool-specific fields from get_tool_fields() - Common fields (temperature, thinking_mode, etc.) - Model field with proper auto-mode handling - Required fields from get_required_fields() Tools can override this method for custom schema generation while still benefiting from SimpleTool's convenience methods. Returns: Complete JSON schema for the tool """ required_fields = list(self.get_required_fields()) return SchemaBuilder.build_schema( tool_specific_fields=self.get_tool_fields(), required_fields=required_fields, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), ) def get_request_model(self): """ Return the request model class. Simple tools use the base ToolRequest by default. Override this if your tool needs a custom request model. """ return ToolRequest # Hook methods for safe attribute access without hasattr/getattr def get_request_model_name(self, request) -> Optional[str]: """Get model name from request. Override for custom model name handling.""" try: return request.model except AttributeError: return None def get_request_images(self, request) -> list: """Get images from request. Override for custom image handling.""" try: return request.images if request.images is not None else [] except AttributeError: return [] def get_request_continuation_id(self, request) -> Optional[str]: """Get continuation_id from request. Override for custom continuation handling.""" try: return request.continuation_id except AttributeError: return None def get_request_prompt(self, request) -> str: """Get prompt from request. Override for custom prompt handling.""" try: return request.prompt except AttributeError: return "" def get_request_temperature(self, request) -> Optional[float]: """Get temperature from request. Override for custom temperature handling.""" try: return request.temperature except AttributeError: return None def get_validated_temperature(self, request, model_context: Any) -> tuple[float, list[str]]: """ Get temperature from request and validate it against model constraints. This is a convenience method that combines temperature extraction and validation for simple tools. It ensures temperature is within valid range for the model. Args: request: The request object containing temperature model_context: Model context object containing model info Returns: Tuple of (validated_temperature, warning_messages) """ temperature = self.get_request_temperature(request) if temperature is None: temperature = self.get_default_temperature() return self.validate_and_correct_temperature(temperature, model_context) def get_request_thinking_mode(self, request) -> Optional[str]: """Get thinking_mode from request. Override for custom thinking mode handling.""" try: return request.thinking_mode except AttributeError: return None def get_request_files(self, request) -> list: """Get absolute file paths from request. Override for custom file handling.""" try: files = request.absolute_file_paths except AttributeError: files = None if files is None: return [] return files def get_request_as_dict(self, request) -> dict: """Convert request to dictionary. Override for custom serialization.""" try: # Try Pydantic v2 method first return request.model_dump() except AttributeError: try: # Fall back to Pydantic v1 method return request.dict() except AttributeError: # Last resort - convert to dict manually return {"prompt": self.get_request_prompt(request)} def set_request_files(self, request, files: list) -> None: """Set absolute file paths on request. Override for custom file setting.""" try: request.absolute_file_paths = files except AttributeError: pass def get_actually_processed_files(self) -> list: """Get actually processed files. Override for custom file tracking.""" try: return self._actually_processed_files except AttributeError: return [] async def execute(self, arguments: dict[str, Any]) -> list: """ Execute the simple tool using the comprehensive flow from old base.py. This method replicates the proven execution pattern while using SimpleTool hooks. """ import logging from mcp.types import TextContent from tools.models import ToolOutput logger = logging.getLogger(f"tools.{self.get_name()}") try: # Store arguments for access by helper methods self._current_arguments = arguments logger.info(f"🔧 {self.get_name()} tool called with arguments: {list(arguments.keys())}") # Validate request using the tool's Pydantic model request_model = self.get_request_model() request = request_model(**arguments) logger.debug(f"Request validation successful for {self.get_name()}") # Validate file paths for security # This prevents path traversal attacks and ensures proper access control path_error = self._validate_file_paths(request) if path_error: error_output = ToolOutput( status="error", content=path_error, content_type="text", ) logger.error("Path validation failed for %s: %s", self.get_name(), path_error) raise ToolExecutionError(error_output.model_dump_json()) # Handle model resolution like old base.py model_name = self.get_request_model_name(request) if not model_name: from config import DEFAULT_MODEL model_name = DEFAULT_MODEL # Store the current model name for later use self._current_model_name = model_name # Handle model context from arguments (for in-process testing) if "_model_context" in arguments: self._model_context = arguments["_model_context"] logger.debug(f"{self.get_name()}: Using model context from arguments") else: # Create model context if not provided from utils.model_context import ModelContext self._model_context = ModelContext(model_name) logger.debug(f"{self.get_name()}: Created model context for {model_name}") # Get images if present images = self.get_request_images(request) continuation_id = self.get_request_continuation_id(request) # Handle conversation history and prompt preparation if continuation_id: # Check if conversation history is already embedded field_value = self.get_request_prompt(request) if "=== CONVERSATION HISTORY ===" in field_value: # Use pre-embedded history prompt = field_value logger.debug(f"{self.get_name()}: Using pre-embedded conversation history") else: # No embedded history - reconstruct it (for in-process calls) logger.debug(f"{self.get_name()}: No embedded history found, reconstructing conversation") # Get thread context from utils.conversation_memory import add_turn, build_conversation_history, get_thread thread_context = get_thread(continuation_id) if thread_context: # Add user's new input to conversation user_prompt = self.get_request_prompt(request) user_files = self.get_request_files(request) if user_prompt: add_turn(continuation_id, "user", user_prompt, files=user_files) # Get updated thread context after adding the turn thread_context = get_thread(continuation_id) logger.debug( f"{self.get_name()}: Retrieved updated thread with {len(thread_context.turns)} turns" ) # Build conversation history with updated thread context conversation_history, conversation_tokens = build_conversation_history( thread_context, self._model_context ) # Get the base prompt from the tool base_prompt = await self.prepare_prompt(request) # Combine with conversation history if conversation_history: prompt = f"{conversation_history}\n\n=== NEW USER INPUT ===\n{base_prompt}" else: prompt = base_prompt else: # Thread not found, prepare normally logger.warning(f"Thread {continuation_id} not found, preparing prompt normally") prompt = await self.prepare_prompt(request) else: # New conversation, prepare prompt normally prompt = await self.prepare_prompt(request) # Add follow-up instructions for new conversations from server import get_follow_up_instructions follow_up_instructions = get_follow_up_instructions(0) prompt = f"{prompt}\n\n{follow_up_instructions}" logger.debug( f"Added follow-up instructions for new {self.get_name()} conversation" ) # Validate images if any were provided if images: image_validation_error = self._validate_image_limits( images, model_context=self._model_context, continuation_id=continuation_id ) if image_validation_error: error_output = ToolOutput( status=image_validation_error.get("status", "error"), content=image_validation_error.get("content"), content_type=image_validation_error.get("content_type", "text"), metadata=image_validation_error.get("metadata"), ) payload = error_output.model_dump_json() logger.error("Image validation failed for %s: %s", self.get_name(), payload) raise ToolExecutionError(payload) # Get and validate temperature against model constraints temperature, temp_warnings = self.get_validated_temperature(request, self._model_context) # Log any temperature corrections for warning in temp_warnings: # Get thinking mode with defaults logger.warning(warning) thinking_mode = self.get_request_thinking_mode(request) if thinking_mode is None: thinking_mode = self.get_default_thinking_mode() # Get the provider from model context (clean OOP - no re-fetching) provider = self._model_context.provider capabilities = self._model_context.capabilities # Get system prompt for this tool base_system_prompt = self.get_system_prompt() capability_augmented_prompt = self._augment_system_prompt_with_capabilities( base_system_prompt, capabilities ) language_instruction = self.get_language_instruction() system_prompt = language_instruction + capability_augmented_prompt # Generate AI response using the provider logger.info(f"Sending request to {provider.get_provider_type().value} API for {self.get_name()}") logger.info( f"Using model: {self._model_context.model_name} via {provider.get_provider_type().value} provider" ) # Estimate tokens for logging from utils.token_utils import estimate_tokens estimated_tokens = estimate_tokens(prompt) logger.debug(f"Prompt length: {len(prompt)} characters (~{estimated_tokens:,} tokens)") # Resolve model capabilities for feature gating supports_thinking = capabilities.supports_extended_thinking # Generate content with provider abstraction model_response = provider.generate_content( prompt=prompt, model_name=self._current_model_name, system_prompt=system_prompt, temperature=temperature, thinking_mode=thinking_mode if supports_thinking else None, images=images if images else None, ) logger.info(f"Received response from {provider.get_provider_type().value} API for {self.get_name()}") # Process the model's response if model_response.content: raw_text = model_response.content # Create model info for conversation tracking model_info = { "provider": provider, "model_name": self._current_model_name, "model_response": model_response, } # Parse response using the same logic as old base.py tool_output = self._parse_response(raw_text, request, model_info) logger.info(f"✅ {self.get_name()} tool completed successfully") else: # Handle cases where the model couldn't generate a response metadata = model_response.metadata or {} finish_reason = metadata.get("finish_reason", "Unknown") if metadata.get("is_blocked_by_safety"): # Specific handling for content safety blocks safety_details = metadata.get("safety_feedback") or "details not provided" logger.warning( f"Response blocked by content safety policy for {self.get_name()}. " f"Reason: {finish_reason}, Details: {safety_details}" ) tool_output = ToolOutput( status="error", content="Your request was blocked by the content safety policy. " "Please try modifying your prompt.", content_type="text", ) else: # Handle other empty responses - could be legitimate completion or unclear blocking if finish_reason == "STOP": # Model completed normally but returned empty content - retry with clarification logger.info( f"Model completed with empty response for {self.get_name()}, retrying with clarification" ) # Retry the same request with modified prompt asking for explicit response original_prompt = prompt retry_prompt = f"{original_prompt}\n\nIMPORTANT: Please provide a substantive response. If you cannot respond to the above request, please explain why and suggest alternatives." try: retry_response = provider.generate_content( prompt=retry_prompt, model_name=self._current_model_name, system_prompt=system_prompt, temperature=temperature, thinking_mode=thinking_mode if supports_thinking else None, images=images if images else None, ) if retry_response.content: # Successful retry - use the retry response logger.info(f"Retry successful for {self.get_name()}") raw_text = retry_response.content # Update model info for the successful retry model_info = { "provider": provider, "model_name": self._current_model_name, "model_response": retry_response, } # Parse the retry response tool_output = self._parse_response(raw_text, request, model_info) logger.info(f"✅ {self.get_name()} tool completed successfully after retry") else: # Retry also failed - inspect metadata to find out why retry_metadata = retry_response.metadata or {} if retry_metadata.get("is_blocked_by_safety"): # The retry was blocked by safety filters safety_details = retry_metadata.get("safety_feedback") or "details not provided" logger.warning( f"Retry for {self.get_name()} was blocked by content safety policy. " f"Details: {safety_details}" ) tool_output = ToolOutput( status="error", content="Your request was also blocked by the content safety policy after a retry. " "Please try rephrasing your prompt significantly.", content_type="text", ) else: # Retry failed for other reasons (e.g., another STOP) tool_output = ToolOutput( status="error", content="The model repeatedly returned empty responses. This may indicate content filtering or a model issue.", content_type="text", ) except Exception as retry_error: logger.warning(f"Retry failed for {self.get_name()}: {retry_error}") tool_output = ToolOutput( status="error", content=f"Model returned empty response and retry failed: {str(retry_error)}", content_type="text", ) else: # Non-STOP finish reasons are likely actual errors logger.warning( f"Response blocked or incomplete for {self.get_name()}. Finish reason: {finish_reason}" ) tool_output = ToolOutput( status="error", content=f"Response blocked or incomplete. Finish reason: {finish_reason}", content_type="text", ) # Return the tool output as TextContent, marking protocol errors appropriately payload = tool_output.model_dump_json() if tool_output.status == "error": logger.error("%s reported error status - raising ToolExecutionError", self.get_name()) raise ToolExecutionError(payload) return [TextContent(type="text", text=payload)] except ToolExecutionError: raise except Exception as e: # Special handling for MCP size check errors if str(e).startswith("MCP_SIZE_CHECK:"): # Extract the JSON content after the prefix json_content = str(e)[len("MCP_SIZE_CHECK:") :] raise ToolExecutionError(json_content) logger.error(f"Error in {self.get_name()}: {str(e)}") error_output = ToolOutput( status="error", content=f"Error in {self.get_name()}: {str(e)}", content_type="text", ) raise ToolExecutionError(error_output.model_dump_json()) from e def _parse_response(self, raw_text: str, request, model_info: Optional[dict] = None): """ Parse the raw response and format it using the hook method. This simplified version focuses on the SimpleTool pattern: format the response using the format_response hook, then handle conversation continuation. """ from tools.models import ToolOutput # Format the response using the hook method formatted_response = self.format_response(raw_text, request, model_info) # Handle conversation continuation like old base.py continuation_id = self.get_request_continuation_id(request) if continuation_id: self._record_assistant_turn(continuation_id, raw_text, request, model_info) # Create continuation offer like old base.py continuation_data = self._create_continuation_offer(request, model_info) if continuation_data: return self._create_continuation_offer_response(formatted_response, continuation_data, request, model_info) else: # Build metadata with model and provider info for success response metadata = {} if model_info: model_name = model_info.get("model_name") if model_name: metadata["model_used"] = model_name provider = model_info.get("provider") if provider: # Handle both provider objects and string values if isinstance(provider, str): metadata["provider_used"] = provider else: try: metadata["provider_used"] = provider.get_provider_type().value except AttributeError: # Fallback if provider doesn't have get_provider_type method metadata["provider_used"] = str(provider) return ToolOutput( status="success", content=formatted_response, content_type="text", metadata=metadata if metadata else None, ) def _create_continuation_offer(self, request, model_info: Optional[dict] = None): """Create continuation offer following old base.py pattern""" continuation_id = self.get_request_continuation_id(request) try: from utils.conversation_memory import create_thread, get_thread if continuation_id: # Existing conversation thread_context = get_thread(continuation_id) if thread_context and thread_context.turns: turn_count = len(thread_context.turns) from utils.conversation_memory import MAX_CONVERSATION_TURNS if turn_count >= MAX_CONVERSATION_TURNS - 1: return None # No more turns allowed remaining_turns = MAX_CONVERSATION_TURNS - turn_count - 1 return { "continuation_id": continuation_id, "remaining_turns": remaining_turns, "note": f"You can continue this conversation for {remaining_turns} more exchanges.", } else: # New conversation - create thread and offer continuation # Convert request to dict for initial_context initial_request_dict = self.get_request_as_dict(request) new_thread_id = create_thread(tool_name=self.get_name(), initial_request=initial_request_dict) # Add the initial user turn to the new thread from utils.conversation_memory import MAX_CONVERSATION_TURNS, add_turn user_prompt = self.get_request_prompt(request) user_files = self.get_request_files(request) user_images = self.get_request_images(request) # Add user's initial turn add_turn( new_thread_id, "user", user_prompt, files=user_files, images=user_images, tool_name=self.get_name() ) return { "continuation_id": new_thread_id, "remaining_turns": MAX_CONVERSATION_TURNS - 1, "note": f"You can continue this conversation for {MAX_CONVERSATION_TURNS - 1} more exchanges.", } except Exception: return None def _create_continuation_offer_response( self, content: str, continuation_data: dict, request, model_info: Optional[dict] = None ): """Create response with continuation offer following old base.py pattern""" from tools.models import ContinuationOffer, ToolOutput try: if not self.get_request_continuation_id(request): self._record_assistant_turn( continuation_data["continuation_id"], content, request, model_info, ) continuation_offer = ContinuationOffer( continuation_id=continuation_data["continuation_id"], note=continuation_data["note"], remaining_turns=continuation_data["remaining_turns"], ) # Build metadata with model and provider info metadata = {"tool_name": self.get_name(), "conversation_ready": True} if model_info: model_name = model_info.get("model_name") if model_name: metadata["model_used"] = model_name provider = model_info.get("provider") if provider: # Handle both provider objects and string values if isinstance(provider, str): metadata["provider_used"] = provider else: try: metadata["provider_used"] = provider.get_provider_type().value except AttributeError: # Fallback if provider doesn't have get_provider_type method metadata["provider_used"] = str(provider) return ToolOutput( status="continuation_available", content=content, content_type="text", continuation_offer=continuation_offer, metadata=metadata, ) except Exception: # Fallback to simple success if continuation offer fails return ToolOutput(status="success", content=content, content_type="text") def _record_assistant_turn( self, continuation_id: str, response_text: str, request, model_info: Optional[dict] ) -> None: """Persist an assistant response in conversation memory.""" if not continuation_id: return from utils.conversation_memory import add_turn model_provider = None model_name = None model_metadata = None if model_info: provider = model_info.get("provider") if provider: if isinstance(provider, str): model_provider = provider else: try: model_provider = provider.get_provider_type().value except AttributeError: model_provider = str(provider) model_name = model_info.get("model_name") model_response = model_info.get("model_response") if model_response: model_metadata = {"usage": model_response.usage, "metadata": model_response.metadata} add_turn( continuation_id, "assistant", response_text, files=self.get_request_files(request), images=self.get_request_images(request), tool_name=self.get_name(), model_provider=model_provider, model_name=model_name, model_metadata=model_metadata, ) # Convenience methods for common tool patterns def build_standard_prompt( self, system_prompt: str, user_content: str, request, file_context_title: str = "CONTEXT FILES" ) -> str: """ Build a standard prompt with system prompt, user content, and optional files. This is a convenience method that handles the common pattern of: 1. Adding file content if present 2. Checking token limits 3. Adding web search instructions 4. Combining everything into a well-formatted prompt Args: system_prompt: The system prompt for the tool user_content: The main user request/content request: The validated request object file_context_title: Title for the file context section Returns: Complete formatted prompt ready for the AI model """ # Check size limits against raw user input before enriching with internal context content_to_validate = self.get_prompt_content_for_size_validation(user_content) self._validate_token_limit(content_to_validate, "Content") # Add context files if provided (does not affect MCP boundary enforcement) files = self.get_request_files(request) if files: file_content, processed_files = self._prepare_file_content_for_prompt( files, self.get_request_continuation_id(request), "Context files", model_context=getattr(self, "_model_context", None), ) self._actually_processed_files = processed_files if file_content: user_content = f"{user_content}\n\n=== {file_context_title} ===\n{file_content}\n=== END CONTEXT ====" # Add standardized web search guidance websearch_instruction = self.get_websearch_instruction(self.get_websearch_guidance()) # Combine system prompt with user content full_prompt = f"""{system_prompt}{websearch_instruction} === USER REQUEST === {user_content} === END REQUEST === Please provide a thoughtful, comprehensive response:""" return full_prompt def get_prompt_content_for_size_validation(self, user_content: str) -> str: """ Override to use original user prompt for size validation when conversation history is embedded. When server.py embeds conversation history into the prompt field, it also stores the original user prompt in _original_user_prompt. We use that for size validation to avoid incorrectly triggering size limits due to conversation history. Args: user_content: The user content (may include conversation history) Returns: The original user prompt if available, otherwise the full user content """ # Check if we have the current arguments from execute() method current_args = getattr(self, "_current_arguments", None) if current_args: # If server.py embedded conversation history, it stores original prompt separately original_user_prompt = current_args.get("_original_user_prompt") if original_user_prompt is not None: # Use original user prompt for size validation (excludes conversation history) return original_user_prompt # Fallback to default behavior (validate full user content) return user_content def get_websearch_guidance(self) -> Optional[str]: """ Return tool-specific web search guidance. Override this to provide tool-specific guidance for when web searches would be helpful. Return None to use the default guidance. Returns: Tool-specific web search guidance or None for default """ return None def handle_prompt_file_with_fallback(self, request) -> str: """ Handle prompt.txt files with fallback to request field. This is a convenience method for tools that accept prompts either as a field or as a prompt.txt file. It handles the extraction and validation automatically. Args: request: The validated request object Returns: The effective prompt content Raises: ValueError: If prompt is too large for MCP transport """ # Check for prompt.txt in provided absolute file paths files = self.get_request_files(request) if files: prompt_content, updated_files = self.handle_prompt_file(files) # Update request files list if needed if updated_files is not None: self.set_request_files(request, updated_files) else: prompt_content = None # Use prompt.txt content if available, otherwise use the prompt field user_content = prompt_content if prompt_content else self.get_request_prompt(request) # Check user input size at MCP transport boundary (excluding conversation history) validation_content = self.get_prompt_content_for_size_validation(user_content) size_check = self.check_prompt_size(validation_content) if size_check: from tools.models import ToolOutput raise ValueError(f"MCP_SIZE_CHECK:{ToolOutput(**size_check).model_dump_json()}") return user_content def get_chat_style_websearch_guidance(self) -> str: """ Get Chat tool-style web search guidance. Returns web search guidance that matches the original Chat tool pattern. This is useful for tools that want to maintain the same search behavior. Returns: Web search guidance text """ return """When discussing topics, consider if searches for these would help: - Documentation for any technologies or concepts mentioned - Current best practices and patterns - Recent developments or updates - Community discussions and solutions""" def supports_custom_request_model(self) -> bool: """ Indicate whether this tool supports custom request models. Simple tools support custom request models by default. Tools that override get_request_model() to return something other than ToolRequest should return True here. Returns: True if the tool uses a custom request model """ return self.get_request_model() != ToolRequest def _validate_file_paths(self, request) -> Optional[str]: """ Validate that all file paths in the request are absolute paths. This is a security measure to prevent path traversal attacks and ensure proper access control. All file paths must be absolute (starting with '/'). Args: request: The validated request object Returns: Optional[str]: Error message if validation fails, None if all paths are valid """ import os # Check if request has absolute file paths attribute (legacy tools may still provide 'files') files = self.get_request_files(request) if files: for file_path in files: if not os.path.isabs(file_path): return ( f"Error: All file paths must be FULL absolute paths to real files / folders - DO NOT SHORTEN. " f"Received relative path: {file_path}\n" f"Please provide the full absolute path starting with '/' (must be FULL absolute paths to real files / folders - DO NOT SHORTEN)" ) return None def prepare_chat_style_prompt(self, request, system_prompt: str = None) -> str: """ Prepare a prompt using Chat tool-style patterns. This convenience method replicates the Chat tool's prompt preparation logic: 1. Handle prompt.txt file if present 2. Add file context with specific formatting 3. Add web search guidance 4. Format with system prompt Args: request: The validated request object system_prompt: System prompt to use (uses get_system_prompt() if None) Returns: Complete formatted prompt """ # Use provided system prompt or get from tool if system_prompt is None: system_prompt = self.get_system_prompt() # Get user content (handles prompt.txt files) user_content = self.handle_prompt_file_with_fallback(request) # Build standard prompt with Chat-style web search guidance websearch_guidance = self.get_chat_style_websearch_guidance() # Override the websearch guidance temporarily original_guidance = self.get_websearch_guidance self.get_websearch_guidance = lambda: websearch_guidance try: full_prompt = self.build_standard_prompt(system_prompt, user_content, request, "CONTEXT FILES") finally: # Restore original guidance method self.get_websearch_guidance = original_guidance if system_prompt: marker = "\n\n=== USER REQUEST ===\n" if marker in full_prompt: _, user_section = full_prompt.split(marker, 1) return f"=== USER REQUEST ===\n{user_section}" return full_prompt ================================================ FILE: tools/testgen.py ================================================ """ TestGen Workflow tool - Step-by-step test generation with expert validation This tool provides a structured workflow for comprehensive test generation. It guides the CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough code examination, test planning, and pattern identification before proceeding. The tool supports finding updates and expert analysis integration for comprehensive test suite generation. Key features: - Step-by-step test generation workflow with progress tracking - Context-aware file embedding (references during investigation, full content for analysis) - Automatic test pattern detection and framework identification - Expert analysis integration with external models for additional test suggestions - Support for edge case identification and comprehensive coverage - Confidence-based workflow optimization """ import logging from typing import TYPE_CHECKING, Any, Optional from pydantic import Field, model_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import TESTGEN_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for test generation workflow TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "Test plan for this step. Step 1: outline how you'll analyse structure, business logic, critical paths, and edge cases. Later steps: record findings and new scenarios as they emerge." ), "step_number": "Current test-generation step (starts at 1) — each step should build on prior work.", "total_steps": "Estimated number of steps needed for test planning; adjust as new scenarios appear.", "next_step_required": "True while more investigation or planning remains; set False when test planning is ready for expert validation.", "findings": "Summarise functionality, critical paths, edge cases, boundary conditions, error handling, and existing test patterns. Cover both happy and failure paths.", "files_checked": "Absolute paths of every file examined, including those ruled out.", "relevant_files": "Absolute paths of code that requires new or updated tests (implementation, dependencies, existing test fixtures).", "relevant_context": "Functions/methods needing coverage (e.g. 'Class.method', 'function_name'), with emphasis on critical paths and error-prone code.", "confidence": ( "Indicate your current confidence in the test generation assessment. Use: 'exploring' (starting analysis), " "'low' (early investigation), 'medium' (some patterns identified), 'high' (strong understanding), " "'very_high' (very strong understanding), 'almost_certain' (nearly complete test plan), 'certain' " "(100% confidence - test plan is thoroughly complete and all test scenarios are identified with no need for external model validation). " "Do NOT use 'certain' unless the test generation analysis is comprehensively complete, use 'very_high' or 'almost_certain' instead if not 100% sure. " "Using 'certain' means you have complete confidence locally and prevents external model validation." ), "images": "Optional absolute paths to diagrams or visuals that clarify the system under test.", } class TestGenRequest(WorkflowRequest): """Request model for test generation workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) confidence: Optional[str] = Field("low", description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["confidence"]) # Optional images for visual context images: Optional[list[str]] = Field(default=None, description=TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Override inherited fields to exclude them from schema (except model which needs to be available) temperature: Optional[float] = Field(default=None, exclude=True) thinking_mode: Optional[str] = Field(default=None, exclude=True) @model_validator(mode="after") def validate_step_one_requirements(self): """Ensure step 1 has required relevant_files field.""" if self.step_number == 1 and not self.relevant_files: raise ValueError("Step 1 requires 'relevant_files' field to specify code files to generate tests for") return self class TestGenTool(WorkflowTool): """ Test Generation workflow tool for step-by-step test planning and expert validation. This tool implements a structured test generation workflow that guides users through methodical investigation steps, ensuring thorough code examination, pattern identification, and test scenario planning before reaching conclusions. It supports complex testing scenarios including edge case identification, framework detection, and comprehensive coverage planning. """ __test__ = False # Prevent pytest from collecting this class as a test def __init__(self): super().__init__() self.initial_request = None def get_name(self) -> str: return "testgen" def get_description(self) -> str: return ( "Creates comprehensive test suites with edge case coverage for specific functions, classes, or modules. " "Analyzes code paths, identifies failure modes, and generates framework-specific tests. " "Be specific about scope - target particular components rather than testing everything." ) def get_system_prompt(self) -> str: return TESTGEN_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Test generation requires thorough analysis and reasoning""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the test generation workflow-specific request model.""" return TestGenRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with test generation-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Test generation workflow-specific field overrides testgen_field_overrides = { "step": { "type": "string", "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, "confidence": { "type": "string", "enum": ["exploring", "low", "medium", "high", "very_high", "almost_certain", "certain"], "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["confidence"], }, "images": { "type": "array", "items": {"type": "string"}, "description": TESTGEN_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, } # Use WorkflowSchemaBuilder with test generation-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=testgen_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each investigation phase.""" if step_number == 1: # Initial test generation investigation tasks return [ "Read and understand the code files specified for test generation", "Analyze the overall structure, public APIs, and main functionality", "Identify critical business logic and complex algorithms that need testing", "Look for existing test patterns or examples if provided", "Understand dependencies, external interactions, and integration points", "Note any potential testability issues or areas that might be hard to test", ] elif confidence in ["exploring", "low"]: # Need deeper investigation return [ "Examine specific functions and methods to understand their behavior", "Trace through code paths to identify all possible execution flows", "Identify edge cases, boundary conditions, and error scenarios", "Check for async operations, state management, and side effects", "Look for non-deterministic behavior or external dependencies", "Analyze error handling and exception cases that need testing", ] elif confidence in ["medium", "high"]: # Close to completion - need final verification return [ "Verify all critical paths have been identified for testing", "Confirm edge cases and boundary conditions are comprehensive", "Check that test scenarios cover both success and failure cases", "Ensure async behavior and concurrency issues are addressed", "Validate that the testing strategy aligns with code complexity", "Double-check that findings include actionable test scenarios", ] else: # General investigation needed return [ "Continue examining the codebase for additional test scenarios", "Gather more evidence about code behavior and dependencies", "Test your assumptions about how the code should be tested", "Look for patterns that confirm your testing strategy", "Focus on areas that haven't been thoroughly examined yet", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Decide when to call external model based on investigation completeness. Always call expert analysis for test generation to get additional test ideas. """ # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # Always benefit from expert analysis for comprehensive test coverage return len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 1 def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call for test generation validation.""" context_parts = [ f"=== TEST GENERATION REQUEST ===\n{self.initial_request or 'Test generation workflow initiated'}\n=== END REQUEST ===" ] # Add investigation summary investigation_summary = self._build_test_generation_summary(consolidated_findings) context_parts.append( f"\n=== AGENT'S TEST PLANNING INVESTIGATION ===\n{investigation_summary}\n=== END INVESTIGATION ===" ) # Add relevant code elements if available if consolidated_findings.relevant_context: methods_text = "\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\n=== CODE ELEMENTS TO TEST ===\n{methods_text}\n=== END CODE ELEMENTS ===") # Add images if available if consolidated_findings.images: images_text = "\n".join(f"- {img}" for img in consolidated_findings.images) context_parts.append(f"\n=== VISUAL DOCUMENTATION ===\n{images_text}\n=== END VISUAL DOCUMENTATION ===") return "\n".join(context_parts) def _build_test_generation_summary(self, consolidated_findings) -> str: """Prepare a comprehensive summary of the test generation investigation.""" summary_parts = [ "=== SYSTEMATIC TEST GENERATION INVESTIGATION SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files identified: {len(consolidated_findings.relevant_files)}", f"Code elements to test: {len(consolidated_findings.relevant_context)}", "", "=== INVESTIGATION PROGRESSION ===", ] for finding in consolidated_findings.findings: summary_parts.append(finding) return "\\n".join(summary_parts) def should_include_files_in_expert_prompt(self) -> bool: """Include files in expert analysis for comprehensive test generation.""" return True def should_embed_system_prompt(self) -> bool: """Embed system prompt in expert analysis for proper context.""" return True def get_expert_thinking_mode(self) -> str: """Use high thinking mode for thorough test generation analysis.""" return "high" def get_expert_analysis_instruction(self) -> str: """Get specific instruction for test generation expert analysis.""" return ( "Please provide comprehensive test generation guidance based on the investigation findings. " "Focus on identifying additional test scenarios, edge cases not yet covered, framework-specific " "best practices, and providing concrete test implementation examples following the multi-agent " "workflow specified in the system prompt." ) # Hook method overrides for test generation-specific behavior def prepare_step_data(self, request) -> dict: """ Map test generation-specific fields for internal processing. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "confidence": request.confidence, "images": request.images or [], } return step_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Test generation workflow skips expert analysis when the CLI agent has "certain" confidence. """ return request.confidence == "certain" and not request.next_step_required def store_initial_issue(self, step_description: str): """Store initial request for expert analysis.""" self.initial_request = step_description # Override inheritance hooks for test generation-specific behavior def get_completion_status(self) -> str: """Test generation tools use test-specific status.""" return "test_generation_complete_ready_for_implementation" def get_completion_data_key(self) -> str: """Test generation uses 'complete_test_generation' key.""" return "complete_test_generation" def get_final_analysis_from_request(self, request): """Test generation tools use findings for final analysis.""" return request.findings def get_confidence_level(self, request) -> str: """Test generation tools use 'certain' for high confidence.""" return "certain" def get_completion_message(self) -> str: """Test generation-specific completion message.""" return ( "Test generation analysis complete with CERTAIN confidence. You have identified all test scenarios " "and provided comprehensive coverage strategy. MANDATORY: Present the user with the complete test plan " "and IMMEDIATELY proceed with creating the test files following the identified patterns and framework. " "Focus on implementing concrete, runnable tests with proper assertions." ) def get_skip_reason(self) -> str: """Test generation-specific skip reason.""" return "Completed comprehensive test planning with full confidence locally" def get_skip_expert_analysis_status(self) -> str: """Test generation-specific expert analysis skip status.""" return "skipped_due_to_certain_test_confidence" def prepare_work_summary(self) -> str: """Test generation-specific work summary.""" return self._build_test_generation_summary(self.consolidated_findings) def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Test generation-specific completion message. """ base_message = ( "TEST GENERATION ANALYSIS IS COMPLETE. You MUST now implement ALL identified test scenarios, " "creating comprehensive test files that cover happy paths, edge cases, error conditions, and " "boundary scenarios. Organize tests by functionality, use appropriate assertions, and follow " "the identified framework patterns. Provide concrete, executable test code—make it easy for " "a developer to run the tests and understand what each test validates." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\\n\\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Provide specific guidance for handling expert analysis in test generation. """ return ( "IMPORTANT: Additional test scenarios and edge cases have been provided by the expert analysis above. " "You MUST incorporate these suggestions into your test implementation, ensuring comprehensive coverage. " "Validate that the expert's test ideas are practical and align with the codebase structure. Combine " "your systematic investigation findings with the expert's additional scenarios to create a thorough " "test suite that catches real-world bugs before they reach production." ) def get_step_guidance_message(self, request) -> str: """ Test generation-specific step guidance with detailed investigation instructions. """ step_guidance = self.get_test_generation_step_guidance(request.step_number, request.confidence, request) return step_guidance["next_steps"] def get_test_generation_step_guidance(self, step_number: int, confidence: str, request) -> dict[str, Any]: """ Provide step-specific guidance for test generation workflow. """ # Generate the next steps instruction based on required actions required_actions = self.get_required_actions(step_number, confidence, request.findings, request.total_steps) if step_number == 1: next_steps = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first analyze " f"the code thoroughly using appropriate tools. CRITICAL AWARENESS: You need to understand " f"the code structure, identify testable behaviors, find edge cases and boundary conditions, " f"and determine the appropriate testing strategy. Use file reading tools, code analysis, and " f"systematic examination to gather comprehensive information about what needs to be tested. " f"Only call {self.get_name()} again AFTER completing your investigation. When you call " f"{self.get_name()} next time, use step_number: {step_number + 1} and report specific " f"code paths examined, test scenarios identified, and testing patterns discovered." ) elif confidence in ["exploring", "low"]: next_steps = ( f"STOP! Do NOT call {self.get_name()} again yet. Based on your findings, you've identified areas that need " f"deeper analysis for test generation. MANDATORY ACTIONS before calling {self.get_name()} step {step_number + 1}:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nOnly call {self.get_name()} again with step_number: {step_number + 1} AFTER " + "completing these test planning tasks." ) elif confidence in ["medium", "high"]: next_steps = ( f"WAIT! Your test generation analysis needs final verification. DO NOT call {self.get_name()} immediately. REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nREMEMBER: Ensure you have identified all test scenarios including edge cases and error conditions. " f"Document findings with specific test cases to implement, then call {self.get_name()} " f"with step_number: {step_number + 1}." ) else: next_steps = ( f"PAUSE ANALYSIS. Before calling {self.get_name()} step {step_number + 1}, you MUST examine more code thoroughly. " + "Required: " + ", ".join(required_actions[:2]) + ". " + f"Your next {self.get_name()} call (step_number: {step_number + 1}) must include " f"NEW test scenarios from actual code analysis, not just theories. NO recursive {self.get_name()} calls " f"without investigation work!" ) return {"next_steps": next_steps} def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match test generation workflow format. """ # Store initial request on first step if request.step_number == 1: self.initial_request = request.step # Convert generic status names to test generation-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "test_generation_in_progress", f"pause_for_{tool_name}": "pause_for_test_analysis", f"{tool_name}_required": "test_analysis_required", f"{tool_name}_complete": "test_generation_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] # Rename status field to match test generation workflow if f"{tool_name}_status" in response_data: response_data["test_generation_status"] = response_data.pop(f"{tool_name}_status") # Add test generation-specific status fields response_data["test_generation_status"]["test_scenarios_identified"] = len( self.consolidated_findings.relevant_context ) response_data["test_generation_status"]["analysis_confidence"] = self.get_request_confidence(request) # Map complete_testgen to complete_test_generation if f"complete_{tool_name}" in response_data: response_data["complete_test_generation"] = response_data.pop(f"complete_{tool_name}") # Map the completion flag to match test generation workflow if f"{tool_name}_complete" in response_data: response_data["test_generation_complete"] = response_data.pop(f"{tool_name}_complete") return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the test generation workflow-specific request model.""" return TestGenRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/thinkdeep.py ================================================ """ ThinkDeep Workflow Tool - Extended Reasoning with Systematic Investigation This tool provides step-by-step deep thinking capabilities using a systematic workflow approach. It enables comprehensive analysis of complex problems with expert validation at completion. Key Features: - Systematic step-by-step thinking process - Multi-step analysis with evidence gathering - Confidence-based investigation flow - Expert analysis integration with external models - Support for focused analysis areas (architecture, performance, security, etc.) - Confidence-based workflow optimization """ import logging from typing import TYPE_CHECKING, Any, Optional from pydantic import Field if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_CREATIVE from systemprompts import THINKDEEP_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) class ThinkDeepWorkflowRequest(WorkflowRequest): """Request model for thinkdeep workflow tool with comprehensive investigation capabilities""" # Core workflow parameters step: str = Field(description="Current work step content and findings") step_number: int = Field(description="Current step number (starts at 1)", ge=1) total_steps: int = Field(description="Estimated total steps needed", ge=1) next_step_required: bool = Field(description="Whether another step is needed") findings: str = Field( description="Discoveries: insights, connections, implications, evidence. " "Document contradictions to earlier assumptions. Update past findings." ) # Investigation tracking files_checked: list[str] = Field( default_factory=list, description="All files examined (absolute paths). Include ruled-out files.", ) relevant_files: list[str] = Field( default_factory=list, description="Files relevant to problem/goal (absolute paths). Include root cause, solution, key insights.", ) relevant_context: list[str] = Field( default_factory=list, description="Key concepts/methods: 'concept_name' or 'ClassName.methodName'. Focus on core insights, decision points.", ) hypothesis: Optional[str] = Field( default=None, description="Current theory based on evidence. Revise in later steps.", ) # Analysis metadata issues_found: list[dict] = Field( default_factory=list, description="Issues with dict: 'severity' (critical/high/medium/low), 'description'.", ) confidence: str = Field( default="low", description="exploring/low/medium/high/very_high/almost_certain/certain. CRITICAL: 'certain' PREVENTS external validation.", ) # Expert analysis configuration - keep these fields available for configuring the final assistant model # in expert analysis (commented out exclude=True) temperature: Optional[float] = Field( default=None, description="Creative thinking temp (0-1, default 0.7)", ge=0.0, le=1.0, ) thinking_mode: Optional[str] = Field( default=None, description="Depth: minimal/low/medium/high/max. Default 'high'.", ) # Context files and investigation scope problem_context: Optional[str] = Field( default=None, description="Additional context about problem/goal. Be expressive.", ) focus_areas: Optional[list[str]] = Field( default=None, description="Focus aspects (architecture, performance, security, etc.)", ) class ThinkDeepTool(WorkflowTool): """ ThinkDeep Workflow Tool - Systematic Deep Thinking Analysis Provides comprehensive step-by-step thinking capabilities with expert validation. Uses workflow architecture for systematic investigation and analysis. """ name = "thinkdeep" description = ( "Performs multi-stage investigation and reasoning for complex problem analysis. " "Use for architecture decisions, complex bugs, performance challenges, and security analysis. " "Provides systematic hypothesis testing, evidence-based investigation, and expert validation." ) def __init__(self): """Initialize the ThinkDeep workflow tool""" super().__init__() # Storage for request parameters to use in expert analysis self.stored_request_params = {} def get_name(self) -> str: """Return the tool name""" return self.name def get_description(self) -> str: """Return the tool description""" return self.description def get_model_category(self) -> "ToolModelCategory": """Return the model category for this tool""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def get_workflow_request_model(self): """Return the workflow request model for this tool""" return ThinkDeepWorkflowRequest def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with thinkdeep-specific overrides.""" from .workflow.schema_builders import WorkflowSchemaBuilder # ThinkDeep workflow-specific field overrides thinkdeep_field_overrides = { "problem_context": { "type": "string", "description": "Additional context about problem/goal. Be expressive.", }, "focus_areas": { "type": "array", "items": {"type": "string"}, "description": "Focus aspects (architecture, performance, security, etc.)", }, } # Use WorkflowSchemaBuilder with thinkdeep-specific tool fields return WorkflowSchemaBuilder.build_schema( tool_specific_fields=thinkdeep_field_overrides, model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), ) def get_system_prompt(self) -> str: """Return the system prompt for this workflow tool""" return THINKDEEP_PROMPT def get_default_temperature(self) -> float: """Return default temperature for deep thinking""" return TEMPERATURE_CREATIVE def get_default_thinking_mode(self) -> str: """Return default thinking mode for thinkdeep""" from config import DEFAULT_THINKING_MODE_THINKDEEP return DEFAULT_THINKING_MODE_THINKDEEP def customize_workflow_response(self, response_data: dict, request, **kwargs) -> dict: """ Customize the workflow response for thinkdeep-specific needs """ # Store request parameters for later use in expert analysis self.stored_request_params = {} try: self.stored_request_params["temperature"] = request.temperature except AttributeError: self.stored_request_params["temperature"] = None try: self.stored_request_params["thinking_mode"] = request.thinking_mode except AttributeError: self.stored_request_params["thinking_mode"] = None # Add thinking-specific context to response response_data.update( { "thinking_status": { "current_step": request.step_number, "total_steps": request.total_steps, "files_checked": len(request.files_checked), "relevant_files": len(request.relevant_files), "thinking_confidence": request.confidence, "analysis_focus": request.focus_areas or ["general"], } } ) # Add thinking_complete field for final steps (test expects this) if not request.next_step_required: response_data["thinking_complete"] = True # Add complete_thinking summary (test expects this) response_data["complete_thinking"] = { "steps_completed": len(self.work_history), "final_confidence": request.confidence, "relevant_context": list(self.consolidated_findings.relevant_context), "key_findings": self.consolidated_findings.findings, "issues_identified": self.consolidated_findings.issues_found, "files_analyzed": list(self.consolidated_findings.relevant_files), } # Add thinking-specific completion message based on confidence if request.confidence == "certain": response_data["completion_message"] = ( "Deep thinking analysis is complete with high certainty. " "All aspects have been thoroughly considered and conclusions are definitive." ) elif not request.next_step_required: response_data["completion_message"] = ( "Deep thinking analysis phase complete. Expert validation will provide additional insights and recommendations." ) return response_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ ThinkDeep tool skips expert analysis when the CLI agent has "certain" confidence. """ return request.confidence == "certain" and not request.next_step_required def get_completion_status(self) -> str: """ThinkDeep tools use thinking-specific status.""" return "deep_thinking_complete_ready_for_implementation" def get_completion_data_key(self) -> str: """ThinkDeep uses 'complete_thinking' key.""" return "complete_thinking" def get_final_analysis_from_request(self, request): """ThinkDeep tools use 'findings' field.""" return request.findings def get_skip_expert_analysis_status(self) -> str: """Status when skipping expert analysis for certain confidence.""" return "skipped_due_to_certain_thinking_confidence" def get_skip_reason(self) -> str: """Reason for skipping expert analysis.""" return "Expressed 'certain' confidence in the deep thinking analysis - no additional validation needed" def get_completion_message(self) -> str: """Message for completion without expert analysis.""" return "Deep thinking analysis complete with certain confidence. Proceed with implementation based on the analysis." def customize_expert_analysis_prompt(self, base_prompt: str, request, file_content: str = "") -> str: """ Customize the expert analysis prompt for deep thinking validation """ thinking_context = f""" DEEP THINKING ANALYSIS VALIDATION You are reviewing a comprehensive deep thinking analysis completed through systematic investigation. Your role is to validate the thinking process, identify any gaps, challenge assumptions, and provide additional insights or alternative perspectives. ANALYSIS SCOPE: - Problem Context: {self._get_problem_context(request)} - Focus Areas: {', '.join(self._get_focus_areas(request))} - Investigation Confidence: {request.confidence} - Steps Completed: {request.step_number} of {request.total_steps} THINKING SUMMARY: {request.findings} KEY INSIGHTS AND CONTEXT: {', '.join(request.relevant_context) if request.relevant_context else 'No specific context identified'} VALIDATION OBJECTIVES: 1. Assess the depth and quality of the thinking process 2. Identify any logical gaps, missing considerations, or flawed assumptions 3. Suggest alternative approaches or perspectives not considered 4. Validate the conclusions and recommendations 5. Provide actionable next steps for implementation Be thorough but constructive in your analysis. Challenge the thinking where appropriate, but also acknowledge strong insights and valid conclusions. """ if file_content: thinking_context += f"\n\nFILE CONTEXT:\n{file_content}" return f"{thinking_context}\n\n{base_prompt}" def get_expert_analysis_instructions(self) -> str: """ Return instructions for expert analysis specific to deep thinking validation """ return ( "DEEP THINKING ANALYSIS IS COMPLETE. You MUST now summarize and present ALL thinking insights, " "alternative approaches considered, risks and trade-offs identified, and final recommendations. " "Clearly prioritize the top solutions or next steps that emerged from the analysis. " "Provide concrete, actionable guidance based on the deep thinking—make it easy for the user to " "understand exactly what to do next and how to implement the best solution." ) # Override hook methods to use stored request parameters for expert analysis def get_request_temperature(self, request) -> float: """Use stored temperature from initial request.""" try: stored_params = self.stored_request_params if stored_params and stored_params.get("temperature") is not None: return stored_params["temperature"] except AttributeError: pass return super().get_request_temperature(request) def get_request_thinking_mode(self, request) -> str: """Use stored thinking mode from initial request.""" try: stored_params = self.stored_request_params if stored_params and stored_params.get("thinking_mode") is not None: return stored_params["thinking_mode"] except AttributeError: pass return super().get_request_thinking_mode(request) def _get_problem_context(self, request) -> str: """Get problem context from request. Override for custom context handling.""" try: return request.problem_context or "General analysis" except AttributeError: return "General analysis" def _get_focus_areas(self, request) -> list[str]: """Get focus areas from request. Override for custom focus area handling.""" try: return request.focus_areas or ["comprehensive analysis"] except AttributeError: return ["comprehensive analysis"] def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """ Return required actions for the current thinking step. """ actions = [] if step_number == 1: actions.extend( [ "Begin systematic thinking analysis", "Identify key aspects and assumptions to explore", "Establish initial investigation approach", ] ) elif confidence == "low": actions.extend( [ "Continue gathering evidence and insights", "Test initial hypotheses", "Explore alternative perspectives", ] ) elif confidence == "medium": actions.extend( [ "Deepen analysis of promising approaches", "Validate key assumptions", "Consider implementation challenges", ] ) elif confidence == "high": actions.extend( [ "Refine and validate key findings", "Explore edge cases and limitations", "Document assumptions and trade-offs", ] ) elif confidence == "very_high": actions.extend( [ "Synthesize findings into cohesive recommendations", "Validate conclusions against all evidence", "Prepare comprehensive implementation guidance", ] ) elif confidence == "almost_certain": actions.extend( [ "Finalize recommendations with high confidence", "Document any remaining minor uncertainties", "Prepare for expert analysis or implementation", ] ) else: # certain actions.append("Analysis complete - ready for implementation") return actions def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """ Determine if expert analysis should be called based on confidence and completion. """ if request: try: # Don't call expert analysis if confidence is "certain" if request.confidence == "certain": return False except AttributeError: pass # Call expert analysis if investigation is complete (when next_step_required is False) if request: try: return not request.next_step_required except AttributeError: pass # Fallback: call expert analysis if we have meaningful findings return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings) -> str: """ Prepare context for expert analysis specific to deep thinking. """ context_parts = [] context_parts.append("DEEP THINKING ANALYSIS SUMMARY:") context_parts.append(f"Steps completed: {len(consolidated_findings.findings)}") context_parts.append(f"Final confidence: {consolidated_findings.confidence}") if consolidated_findings.findings: context_parts.append("\nKEY FINDINGS:") for i, finding in enumerate(consolidated_findings.findings, 1): context_parts.append(f"{i}. {finding}") if consolidated_findings.relevant_context: context_parts.append(f"\nRELEVANT CONTEXT:\n{', '.join(consolidated_findings.relevant_context)}") # Get hypothesis from latest hypotheses entry if available if consolidated_findings.hypotheses: latest_hypothesis = consolidated_findings.hypotheses[-1].get("hypothesis", "") if latest_hypothesis: context_parts.append(f"\nFINAL HYPOTHESIS:\n{latest_hypothesis}") if consolidated_findings.issues_found: context_parts.append(f"\nISSUES IDENTIFIED: {len(consolidated_findings.issues_found)} issues") for issue in consolidated_findings.issues_found: context_parts.append( f"- {issue.get('severity', 'unknown')}: {issue.get('description', 'No description')}" ) return "\n".join(context_parts) def get_step_guidance_message(self, request) -> str: """ Generate guidance for the next step in thinking analysis """ if request.next_step_required: next_step_number = request.step_number + 1 if request.confidence == "certain": guidance = ( f"Your thinking analysis confidence is CERTAIN. Consider if you truly need step {next_step_number} " f"or if you should complete the analysis now with expert validation." ) elif request.confidence == "almost_certain": guidance = ( f"Your thinking analysis confidence is ALMOST_CERTAIN. For step {next_step_number}, consider: " f"finalizing recommendations, documenting any minor uncertainties, or preparing for implementation." ) elif request.confidence == "very_high": guidance = ( f"Your thinking analysis confidence is VERY_HIGH. For step {next_step_number}, consider: " f"synthesis of all findings, comprehensive validation, or creating implementation roadmap." ) elif request.confidence == "high": guidance = ( f"Your thinking analysis confidence is HIGH. For step {next_step_number}, consider: " f"exploring edge cases, documenting trade-offs, or stress-testing key assumptions." ) elif request.confidence == "medium": guidance = ( f"Your thinking analysis confidence is MEDIUM. For step {next_step_number}, focus on: " f"deepening insights, exploring alternative approaches, or gathering additional evidence." ) else: # low or exploring guidance = ( f"Your thinking analysis confidence is {request.confidence.upper()}. For step {next_step_number}, " f"continue investigating: gather more evidence, test hypotheses, or explore different angles." ) # Add specific thinking guidance based on progress if request.step_number == 1: guidance += ( " Consider: What are the key assumptions? What evidence supports or contradicts initial theories? " "What alternative approaches exist?" ) elif request.step_number >= request.total_steps // 2: guidance += ( " Consider: Synthesis of findings, validation of conclusions, identification of implementation " "challenges, and preparation for expert analysis." ) return guidance else: return "Thinking analysis is ready for expert validation and final recommendations." def format_final_response(self, assistant_response: str, request, **kwargs) -> dict: """ Format the final response from the assistant for thinking analysis """ response_data = { "thinking_analysis": assistant_response, "analysis_metadata": { "total_steps_completed": request.step_number, "final_confidence": request.confidence, "files_analyzed": len(request.relevant_files), "key_insights": len(request.relevant_context), "issues_identified": len(request.issues_found), }, } # Add completion status if request.confidence == "certain": response_data["completion_status"] = "analysis_complete_with_certainty" else: response_data["completion_status"] = "analysis_complete_pending_validation" return response_data def format_step_response( self, assistant_response: str, request, status: str = "pause_for_thinkdeep", continuation_id: Optional[str] = None, **kwargs, ) -> dict: """ Format intermediate step responses for thinking workflow """ response_data = super().format_step_response(assistant_response, request, status, continuation_id, **kwargs) # Add thinking-specific step guidance step_guidance = self.get_step_guidance_message(request) response_data["thinking_guidance"] = step_guidance # Add analysis progress indicators response_data["analysis_progress"] = { "step_completed": request.step_number, "remaining_steps": max(0, request.total_steps - request.step_number), "confidence_trend": request.confidence, "investigation_depth": "expanding" if request.next_step_required else "finalizing", } return response_data # Required abstract methods from BaseTool def get_request_model(self): """Return the thinkdeep workflow-specific request model.""" return ThinkDeepWorkflowRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/tracer.py ================================================ """ Tracer Workflow tool - Step-by-step code tracing and dependency analysis This tool provides a structured workflow for comprehensive code tracing and analysis. It guides the CLI agent through systematic investigation steps with forced pauses between each step to ensure thorough code examination, dependency mapping, and execution flow analysis before proceeding. The tracer guides users through sequential code analysis with full context awareness and the ability to revise and adapt as understanding deepens. Key features: - Sequential tracing with systematic investigation workflow - Support for precision tracing (execution flow) and dependencies tracing (structural relationships) - Self-contained completion with detailed output formatting instructions - Context-aware analysis that builds understanding step by step - No external expert analysis needed - provides comprehensive guidance internally Perfect for: method/function execution flow analysis, dependency mapping, call chain tracing, structural relationship analysis, architectural understanding, and code comprehension. """ import logging from typing import TYPE_CHECKING, Any, Literal, Optional from pydantic import Field, field_validator if TYPE_CHECKING: from tools.models import ToolModelCategory from config import TEMPERATURE_ANALYTICAL from systemprompts import TRACER_PROMPT from tools.shared.base_models import WorkflowRequest from .workflow.base import WorkflowTool logger = logging.getLogger(__name__) # Tool-specific field descriptions for tracer workflow TRACER_WORKFLOW_FIELD_DESCRIPTIONS = { "step": ( "The plan for the current tracing step. Step 1: State the tracing strategy. Later steps: Report findings and adapt the plan. " "CRITICAL: For 'precision' mode, focus on execution flow and call chains. For 'dependencies' mode, focus on structural relationships. " "If trace_mode is 'ask' in step 1, you MUST prompt the user to choose a mode." ), "step_number": ( "The index of the current step in the tracing sequence, beginning at 1. Each step should build upon or " "revise the previous one." ), "total_steps": ( "Your current estimate for how many steps will be needed to complete the tracing analysis. " "Adjust as new findings emerge." ), "next_step_required": ( "Set to true if you plan to continue the investigation with another step. False means you believe the " "tracing analysis is complete and ready for final output formatting." ), "findings": ( "Summary of discoveries from this step, including execution paths, dependency relationships, call chains, and structural patterns. " "IMPORTANT: Document both direct (immediate calls) and indirect (transitive, side effects) relationships." ), "files_checked": ( "List all files examined (absolute paths). Include even ruled-out files to track exploration path." ), "relevant_files": ( "Subset of files_checked directly relevant to the tracing target (absolute paths). Include implementation files, " "dependencies, or files demonstrating key relationships." ), "relevant_context": ( "List methods/functions central to the tracing analysis, in 'ClassName.methodName' or 'functionName' format. " "Prioritize those in the execution flow or dependency chain." ), "confidence": ( "Your confidence in the tracing analysis. Use: 'exploring', 'low', 'medium', 'high', 'very_high', 'almost_certain', 'certain'. " "CRITICAL: 'certain' implies the analysis is 100% complete locally and PREVENTS external model validation." ), "trace_mode": "Type of tracing: 'ask' (default - prompts user to choose mode), 'precision' (execution flow) or 'dependencies' (structural relationships)", "target_description": ( "Description of what to trace and WHY. Include context about what you're trying to understand or analyze." ), "images": ("Optional paths to architecture diagrams or flow charts that help understand the tracing context."), } class TracerRequest(WorkflowRequest): """Request model for tracer workflow investigation steps""" # Required fields for each investigation step step: str = Field(..., description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["step"]) step_number: int = Field(..., description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["step_number"]) total_steps: int = Field(..., description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["total_steps"]) next_step_required: bool = Field(..., description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"]) # Investigation tracking fields findings: str = Field(..., description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["findings"]) files_checked: list[str] = Field( default_factory=list, description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["files_checked"] ) relevant_files: list[str] = Field( default_factory=list, description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"] ) relevant_context: list[str] = Field( default_factory=list, description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"] ) confidence: Optional[str] = Field("exploring", description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["confidence"]) # Tracer-specific fields (used in step 1 to initialize) trace_mode: Optional[Literal["precision", "dependencies", "ask"]] = Field( "ask", description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["trace_mode"] ) target_description: Optional[str] = Field( None, description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["target_description"] ) images: Optional[list[str]] = Field(default=None, description=TRACER_WORKFLOW_FIELD_DESCRIPTIONS["images"]) # Exclude fields not relevant to tracing workflow issues_found: list[dict] = Field(default_factory=list, exclude=True, description="Tracing doesn't track issues") hypothesis: Optional[str] = Field(default=None, exclude=True, description="Tracing doesn't use hypothesis") # Exclude other non-tracing fields temperature: Optional[float] = Field(default=None, exclude=True) thinking_mode: Optional[str] = Field(default=None, exclude=True) use_assistant_model: Optional[bool] = Field(default=False, exclude=True, description="Tracing is self-contained") @field_validator("step_number") @classmethod def validate_step_number(cls, v): if v < 1: raise ValueError("step_number must be at least 1") return v @field_validator("total_steps") @classmethod def validate_total_steps(cls, v): if v < 1: raise ValueError("total_steps must be at least 1") return v class TracerTool(WorkflowTool): """ Tracer workflow tool for step-by-step code tracing and dependency analysis. This tool implements a structured tracing workflow that guides users through methodical investigation steps, ensuring thorough code examination, dependency mapping, and execution flow analysis before reaching conclusions. It supports both precision tracing (execution flow) and dependencies tracing (structural relationships). """ def __init__(self): super().__init__() self.initial_request = None self.trace_config = {} def get_name(self) -> str: return "tracer" def get_description(self) -> str: return ( "Performs systematic code tracing with modes for execution flow or dependency mapping. " "Use for method execution analysis, call chain tracing, dependency mapping, and architectural understanding. " "Supports precision mode (execution flow) and dependencies mode (structural relationships)." ) def get_system_prompt(self) -> str: return TRACER_PROMPT def get_default_temperature(self) -> float: return TEMPERATURE_ANALYTICAL def get_model_category(self) -> "ToolModelCategory": """Tracer requires analytical reasoning for code analysis""" from tools.models import ToolModelCategory return ToolModelCategory.EXTENDED_REASONING def requires_model(self) -> bool: """ Tracer tool doesn't require model resolution at the MCP boundary. The tracer is a structured workflow tool that organizes tracing steps and provides detailed output formatting guidance without calling external AI models. Returns: bool: False - tracer doesn't need AI model access """ return False def get_workflow_request_model(self): """Return the tracer-specific request model.""" return TracerRequest def get_tool_fields(self) -> dict[str, dict[str, Any]]: """Return tracing-specific field definitions beyond the standard workflow fields.""" return { # Tracer-specific fields "trace_mode": { "type": "string", "enum": ["precision", "dependencies", "ask"], "description": TRACER_WORKFLOW_FIELD_DESCRIPTIONS["trace_mode"], }, "target_description": { "type": "string", "description": TRACER_WORKFLOW_FIELD_DESCRIPTIONS["target_description"], }, "images": { "type": "array", "items": {"type": "string"}, "description": TRACER_WORKFLOW_FIELD_DESCRIPTIONS["images"], }, } def get_input_schema(self) -> dict[str, Any]: """Generate input schema using WorkflowSchemaBuilder with field exclusion.""" from .workflow.schema_builders import WorkflowSchemaBuilder # Exclude investigation-specific fields that tracing doesn't need excluded_workflow_fields = [ "issues_found", # Tracing doesn't track issues "hypothesis", # Tracing doesn't use hypothesis ] # Exclude common fields that tracing doesn't need excluded_common_fields = [ "temperature", # Tracing doesn't need temperature control "thinking_mode", # Tracing doesn't need thinking mode "absolute_file_paths", # Tracing uses relevant_files instead ] return WorkflowSchemaBuilder.build_schema( tool_specific_fields=self.get_tool_fields(), required_fields=["target_description", "trace_mode"], # Step 1 requires these model_field_schema=self.get_model_field_schema(), auto_mode=self.is_effective_auto_mode(), tool_name=self.get_name(), excluded_workflow_fields=excluded_workflow_fields, excluded_common_fields=excluded_common_fields, ) # ================================================================================ # Abstract Methods - Required Implementation from BaseWorkflowMixin # ================================================================================ def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each tracing phase.""" if step_number == 1: # Check if we're in ask mode and need to prompt for mode selection if self.get_trace_mode() == "ask": return [ "MUST ask user to choose between precision or dependencies mode", "Explain precision mode: traces execution flow, call chains, and usage patterns (best for methods/functions)", "Explain dependencies mode: maps structural relationships and bidirectional dependencies (best for classes/modules)", "Wait for user's mode selection before proceeding with investigation", ] # Initial tracing investigation tasks (when mode is already selected) return [ "Search for and locate the target method/function/class/module in the codebase", "Read and understand the implementation of the target code", "Identify the file location, complete signature, and basic structure", "Begin mapping immediate relationships (what it calls, what calls it)", "Understand the context and purpose of the target code", ] elif confidence in ["exploring", "low"]: # Need deeper investigation return [ "Trace deeper into the execution flow or dependency relationships", "Examine how the target code is used throughout the codebase", "Map additional layers of dependencies or call chains", "Look for conditional execution paths, error handling, and edge cases", "Understand the broader architectural context and patterns", ] elif confidence in ["medium", "high"]: # Close to completion - need final verification return [ "Verify completeness of the traced relationships and execution paths", "Check for any missed dependencies, usage patterns, or execution branches", "Confirm understanding of side effects, state changes, and external interactions", "Validate that the tracing covers all significant code relationships", "Prepare comprehensive findings for final output formatting", ] else: # General investigation needed return [ "Continue systematic tracing of code relationships and execution paths", "Gather more evidence using appropriate code analysis techniques", "Test assumptions about code behavior and dependency relationships", "Look for patterns that enhance understanding of the code structure", "Focus on areas that haven't been thoroughly traced yet", ] def should_call_expert_analysis(self, consolidated_findings, request=None) -> bool: """Tracer is self-contained and doesn't need expert analysis.""" return False def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Tracer doesn't use expert analysis.""" return "" def requires_expert_analysis(self) -> bool: """Tracer is self-contained like the planner tool.""" return False # ================================================================================ # Workflow Customization - Match Planner Behavior # ================================================================================ def prepare_step_data(self, request) -> dict: """ Prepare step data from request with tracer-specific fields. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": request.files_checked, "relevant_files": request.relevant_files, "relevant_context": request.relevant_context, "issues_found": [], # Tracer doesn't track issues "confidence": request.confidence or "exploring", "hypothesis": None, # Tracer doesn't use hypothesis "images": request.images or [], # Tracer-specific fields "trace_mode": request.trace_mode, "target_description": request.target_description, } return step_data def build_base_response(self, request, continuation_id: str = None) -> dict: """ Build the base response structure with tracer-specific fields. """ # Use work_history from workflow mixin for consistent step tracking current_step_count = len(self.work_history) + 1 response_data = { "status": f"{self.get_name()}_in_progress", "step_number": request.step_number, "total_steps": request.total_steps, "next_step_required": request.next_step_required, "step_content": request.step, f"{self.get_name()}_status": { "files_checked": len(self.consolidated_findings.files_checked), "relevant_files": len(self.consolidated_findings.relevant_files), "relevant_context": len(self.consolidated_findings.relevant_context), "issues_found": len(self.consolidated_findings.issues_found), "images_collected": len(self.consolidated_findings.images), "current_confidence": self.get_request_confidence(request), "step_history_length": current_step_count, }, "metadata": { "trace_mode": self.trace_config.get("trace_mode", "unknown"), "target_description": self.trace_config.get("target_description", ""), "step_history_length": current_step_count, }, } if continuation_id: response_data["continuation_id"] = continuation_id return response_data def handle_work_continuation(self, response_data: dict, request) -> dict: """ Handle work continuation with tracer-specific guidance. """ response_data["status"] = f"pause_for_{self.get_name()}" response_data[f"{self.get_name()}_required"] = True # Get tracer-specific required actions required_actions = self.get_required_actions( request.step_number, request.confidence or "exploring", request.findings, request.total_steps ) response_data["required_actions"] = required_actions # Generate step-specific guidance if request.step_number == 1: # Check if we're in ask mode and need to prompt for mode selection if self.get_trace_mode() == "ask": response_data["next_steps"] = ( f"STOP! You MUST ask the user to choose a tracing mode before proceeding. " f"Present these options clearly:\\n\\n" f"**PRECISION MODE**: Traces execution flow, call chains, and usage patterns. " f"Best for understanding how a specific method or function works, what it calls, " f"and how data flows through the execution path.\\n\\n" f"**DEPENDENCIES MODE**: Maps structural relationships and bidirectional dependencies. " f"Best for understanding how a class or module relates to other components, " f"what depends on it, and what it depends on.\\n\\n" f"After the user selects a mode, call {self.get_name()} again with step_number: 1 " f"but with the chosen trace_mode (either 'precision' or 'dependencies')." ) else: response_data["next_steps"] = ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. You MUST first investigate " f"the codebase to understand the target code. CRITICAL AWARENESS: You need to find and understand " f"the target method/function/class/module, examine its implementation, and begin mapping its " f"relationships. Use file reading tools, code search, and systematic examination to gather " f"comprehensive information about the target. Only call {self.get_name()} again AFTER completing " f"your investigation. When you call {self.get_name()} next time, use step_number: {request.step_number + 1} " f"and report specific files examined, code structure discovered, and initial relationship findings." ) elif request.confidence in ["exploring", "low"]: next_step = request.step_number + 1 response_data["next_steps"] = ( f"STOP! Do NOT call {self.get_name()} again yet. Based on your findings, you've identified areas that need " f"deeper tracing analysis. MANDATORY ACTIONS before calling {self.get_name()} step {next_step}:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nOnly call {self.get_name()} again with step_number: {next_step} AFTER " + "completing these tracing investigations." ) elif request.confidence in ["medium", "high"]: next_step = request.step_number + 1 response_data["next_steps"] = ( f"WAIT! Your tracing analysis needs final verification. DO NOT call {self.get_name()} immediately. " f"REQUIRED ACTIONS:\\n" + "\\n".join(f"{i+1}. {action}" for i, action in enumerate(required_actions)) + f"\\n\\nREMEMBER: Ensure you have traced all significant relationships and execution paths. " f"Document findings with specific file references and method signatures, then call {self.get_name()} " f"with step_number: {next_step}." ) else: # General investigation needed next_step = request.step_number + 1 remaining_steps = request.total_steps - request.step_number response_data["next_steps"] = ( f"Continue systematic tracing with step {next_step}. Approximately {remaining_steps} steps remaining. " f"Focus on deepening your understanding of the code relationships and execution patterns." ) return response_data def customize_workflow_response(self, response_data: dict, request) -> dict: """ Customize response to match tracer tool format with output instructions. """ # Store trace configuration on first step if request.step_number == 1: self.initial_request = request.step self.trace_config = { "trace_mode": request.trace_mode, "target_description": request.target_description, } # Update metadata with trace configuration if "metadata" in response_data: response_data["metadata"]["trace_mode"] = request.trace_mode or "unknown" response_data["metadata"]["target_description"] = request.target_description or "" # If in ask mode, mark this as mode selection phase if request.trace_mode == "ask": response_data["mode_selection_required"] = True response_data["status"] = "mode_selection_required" # Add tracer-specific output instructions for final steps if not request.next_step_required: response_data["tracing_complete"] = True response_data["trace_summary"] = f"TRACING COMPLETE: {request.step}" # Get mode-specific output instructions trace_mode = self.trace_config.get("trace_mode", "precision") rendering_instructions = self._get_rendering_instructions(trace_mode) response_data["output"] = { "instructions": ( "This is a structured tracing analysis response. Present the comprehensive tracing findings " "using the specific rendering format for the trace mode. Follow the exact formatting guidelines " "provided in rendering_instructions. Include all discovered relationships, execution paths, " "and dependencies with precise file references and line numbers." ), "format": f"{trace_mode}_trace_analysis", "rendering_instructions": rendering_instructions, "presentation_guidelines": { "completed_trace": ( "Use the exact rendering format specified for the trace mode. Include comprehensive " "diagrams, tables, and structured analysis. Reference specific file paths and line numbers. " "Follow formatting rules precisely." ), "step_content": "Present as main analysis with clear structure and actionable insights.", "continuation": "Use continuation_id for related tracing sessions or follow-up analysis", }, } response_data["next_steps"] = ( f"Tracing analysis complete. Present the comprehensive {trace_mode} trace analysis to the user " f"using the exact rendering format specified in the output instructions. Follow the formatting " f"guidelines precisely, including diagrams, tables, and file references. After presenting the " f"analysis, offer to help with related tracing tasks or use the continuation_id for follow-up analysis." ) # Convert generic status names to tracer-specific ones tool_name = self.get_name() status_mapping = { f"{tool_name}_in_progress": "tracing_in_progress", f"pause_for_{tool_name}": "pause_for_tracing", f"{tool_name}_required": "tracing_required", f"{tool_name}_complete": "tracing_complete", } if response_data["status"] in status_mapping: response_data["status"] = status_mapping[response_data["status"]] return response_data def _get_rendering_instructions(self, trace_mode: str) -> str: """ Get mode-specific rendering instructions for the CLI agent. Args: trace_mode: Either "precision" or "dependencies" Returns: str: Complete rendering instructions for the specified mode """ if trace_mode == "precision": return self._get_precision_rendering_instructions() else: # dependencies mode return self._get_dependencies_rendering_instructions() def _get_precision_rendering_instructions(self) -> str: """Get rendering instructions for precision trace mode.""" return """ ## MANDATORY RENDERING INSTRUCTIONS FOR PRECISION TRACE You MUST render the trace analysis using ONLY the Vertical Indented Flow Style: ### CALL FLOW DIAGRAM - Vertical Indented Style **EXACT FORMAT TO FOLLOW:** ``` [ClassName::MethodName] (file: /complete/file/path.ext, line: ##) ↓ [AnotherClass::calledMethod] (file: /path/to/file.ext, line: ##) ↓ [ThirdClass::nestedMethod] (file: /path/file.ext, line: ##) ↓ [DeeperClass::innerCall] (file: /path/inner.ext, line: ##) ? if some_condition ↓ [ServiceClass::processData] (file: /services/service.ext, line: ##) ↓ [RepositoryClass::saveData] (file: /data/repo.ext, line: ##) ↓ [ClientClass::sendRequest] (file: /clients/client.ext, line: ##) ↓ [EmailService::sendEmail] (file: /email/service.ext, line: ##) ⚠️ ambiguous branch → [SMSService::sendSMS] (file: /sms/service.ext, line: ##) ⚠️ ambiguous branch ``` **CRITICAL FORMATTING RULES:** 1. **Method Names**: Use the actual naming convention of the project language you're analyzing. Automatically detect and adapt to the project's conventions (camelCase, snake_case, PascalCase, etc.) based on the codebase structure and file extensions. 2. **Vertical Flow Arrows**: - Use `↓` for standard sequential calls (vertical flow) - Use `→` for parallel/alternative calls (horizontal branch) - NEVER use other arrow types 3. **Indentation Logic**: - Start at column 0 for entry point - Indent 2 spaces for each nesting level - Maintain consistent indentation for same call depth - Sibling calls at same level should have same indentation 4. **Conditional Calls**: - Add `? if condition_description` after method for conditional execution - Use actual condition names from code when possible 5. **Ambiguous Branches**: - Mark with `⚠️ ambiguous branch` when execution path is uncertain - Use `→` to show alternative paths at same indentation level 6. **File Path Format**: - Use complete relative paths from project root - Include actual file extensions from the project - Show exact line numbers where method is defined ### ADDITIONAL ANALYSIS VIEWS **1. BRANCHING & SIDE EFFECT TABLE** | Location | Condition | Branches | Uncertain | |----------|-----------|----------|-----------| | CompleteFileName.ext:## | if actual_condition_from_code | method1(), method2(), else skip | No | | AnotherFile.ext:## | if boolean_check | callMethod(), else return | No | | ThirdFile.ext:## | if validation_passes | processData(), else throw | Yes | **2. SIDE EFFECTS** ``` Side Effects: - [database] Specific database operation description (CompleteFileName.ext:##) - [network] Specific network call description (CompleteFileName.ext:##) - [filesystem] Specific file operation description (CompleteFileName.ext:##) - [state] State changes or property modifications (CompleteFileName.ext:##) - [memory] Memory allocation or cache operations (CompleteFileName.ext:##) ``` **3. USAGE POINTS** ``` Usage Points: 1. FileName.ext:## - Context description of where/why it's called 2. AnotherFile.ext:## - Context description of usage scenario 3. ThirdFile.ext:## - Context description of calling pattern 4. FourthFile.ext:## - Context description of integration point ``` **4. ENTRY POINTS** ``` Entry Points: - ClassName::methodName (context: where this flow typically starts) - AnotherClass::entryMethod (context: alternative entry scenario) - ThirdClass::triggerMethod (context: event-driven entry point) ``` **ABSOLUTE REQUIREMENTS:** - Use ONLY the vertical indented style for the call flow diagram - Present ALL FOUR additional analysis views (Branching Table, Side Effects, Usage Points, Entry Points) - Adapt method naming to match the project's programming language conventions - Use exact file paths and line numbers from the actual codebase - DO NOT invent or guess method names or locations - Follow indentation rules precisely for call hierarchy - Mark uncertain execution paths clearly - Provide contextual descriptions in Usage Points and Entry Points sections - Include comprehensive side effects categorization (database, network, filesystem, state, memory)""" def _get_dependencies_rendering_instructions(self) -> str: """Get rendering instructions for dependencies trace mode.""" return """ ## MANDATORY RENDERING INSTRUCTIONS FOR DEPENDENCIES TRACE You MUST render the trace analysis using ONLY the Bidirectional Arrow Flow Style: ### DEPENDENCY FLOW DIAGRAM - Bidirectional Arrow Style **EXACT FORMAT TO FOLLOW:** ``` INCOMING DEPENDENCIES → [TARGET_CLASS/MODULE] → OUTGOING DEPENDENCIES CallerClass::callerMethod ←────┐ AnotherCaller::anotherMethod ←─┤ ThirdCaller::thirdMethod ←─────┤ │ [TARGET_CLASS/MODULE] │ ├────→ FirstDependency::method ├────→ SecondDependency::method └────→ ThirdDependency::method TYPE RELATIONSHIPS: InterfaceName ──implements──→ [TARGET_CLASS] ──extends──→ BaseClass DTOClass ──uses──→ [TARGET_CLASS] ──uses──→ EntityClass ``` **CRITICAL FORMATTING RULES:** 1. **Target Placement**: Always place the target class/module in square brackets `[TARGET_NAME]` at the center 2. **Incoming Dependencies**: Show on the left side with `←` arrows pointing INTO the target 3. **Outgoing Dependencies**: Show on the right side with `→` arrows pointing OUT FROM the target 4. **Arrow Alignment**: Use consistent spacing and alignment for visual clarity 5. **Method Naming**: Use the project's actual naming conventions detected from the codebase 6. **File References**: Include complete file paths and line numbers **VISUAL LAYOUT RULES:** 1. **Header Format**: Always start with the flow direction indicator 2. **Left Side (Incoming)**: - List all callers with `←` arrows - Use `┐`, `┤`, `┘` box drawing characters for clean connection lines - Align arrows consistently 3. **Center (Target)**: - Enclose target in square brackets - Position centrally between incoming and outgoing 4. **Right Side (Outgoing)**: - List all dependencies with `→` arrows - Use `├`, `└` box drawing characters for branching - Maintain consistent spacing 5. **Type Relationships Section**: - Use `──relationship──→` format with double hyphens - Show inheritance, implementation, and usage relationships - Place below the main flow diagram **DEPENDENCY TABLE:** | Type | From/To | Method | File | Line | |------|---------|--------|------|------| | incoming_call | From: CallerClass | callerMethod | /complete/path/file.ext | ## | | outgoing_call | To: TargetClass | targetMethod | /complete/path/file.ext | ## | | implements | Self: ThisClass | — | /complete/path/file.ext | — | | extends | Self: ThisClass | — | /complete/path/file.ext | — | | uses_type | Self: ThisClass | — | /complete/path/file.ext | — | **ABSOLUTE REQUIREMENTS:** - Use ONLY the bidirectional arrow flow style shown above - Automatically detect and use the project's naming conventions - Use exact file paths and line numbers from the actual codebase - DO NOT invent or guess method/class names - Maintain visual alignment and consistent spacing - Include type relationships section when applicable - Show clear directional flow with proper arrows""" # ================================================================================ # Hook Method Overrides for Tracer-Specific Behavior # ================================================================================ def get_completion_status(self) -> str: """Tracer uses tracing-specific status.""" return "tracing_complete" def get_completion_data_key(self) -> str: """Tracer uses 'complete_tracing' key.""" return "complete_tracing" def get_completion_message(self) -> str: """Tracer-specific completion message.""" return ( "Tracing analysis complete. Present the comprehensive trace analysis to the user " "using the specified rendering format and offer to help with related tracing tasks." ) def get_skip_reason(self) -> str: """Tracer-specific skip reason.""" return "Tracer is self-contained and completes analysis without external assistance" def get_skip_expert_analysis_status(self) -> str: """Tracer-specific expert analysis skip status.""" return "skipped_by_tool_design" def store_initial_issue(self, step_description: str): """Store initial tracing description.""" self.initial_tracing_description = step_description def get_initial_request(self, fallback_step: str) -> str: """Get initial tracing description.""" try: return self.initial_tracing_description except AttributeError: return fallback_step def get_request_confidence(self, request) -> str: """Get confidence from request for tracer workflow.""" try: return request.confidence or "exploring" except AttributeError: return "exploring" def get_trace_mode(self) -> str: """Get current trace mode. Override for custom trace mode handling.""" try: return self.trace_config.get("trace_mode", "ask") except AttributeError: return "ask" # Required abstract methods from BaseTool def get_request_model(self): """Return the tracer-specific request model.""" return TracerRequest async def prepare_prompt(self, request) -> str: """Not used - workflow tools use execute_workflow().""" return "" # Workflow tools use execute_workflow() directly ================================================ FILE: tools/version.py ================================================ """ Version Tool - Display PAL MCP Server version and system information This tool provides version information about the PAL MCP Server including version number, last update date, author, and basic system information. It also checks for updates from the GitHub repository. """ import logging import platform import re import sys from pathlib import Path from typing import Any, Optional try: from urllib.error import HTTPError, URLError from urllib.request import urlopen HAS_URLLIB = True except ImportError: HAS_URLLIB = False from mcp.types import TextContent from config import __author__, __updated__, __version__ from tools.models import ToolModelCategory, ToolOutput from tools.shared.base_models import ToolRequest from tools.shared.base_tool import BaseTool logger = logging.getLogger(__name__) def parse_version(version_str: str) -> tuple[int, int, int]: """ Parse version string to tuple of integers for comparison. Args: version_str: Version string like "5.5.5" Returns: Tuple of (major, minor, patch) as integers """ try: parts = version_str.strip().split(".") if len(parts) >= 3: return (int(parts[0]), int(parts[1]), int(parts[2])) elif len(parts) == 2: return (int(parts[0]), int(parts[1]), 0) elif len(parts) == 1: return (int(parts[0]), 0, 0) else: return (0, 0, 0) except (ValueError, IndexError): return (0, 0, 0) def compare_versions(current: str, remote: str) -> int: """ Compare two version strings. Args: current: Current version string remote: Remote version string Returns: -1 if current < remote (update available) 0 if current == remote (up to date) 1 if current > remote (ahead of remote) """ current_tuple = parse_version(current) remote_tuple = parse_version(remote) if current_tuple < remote_tuple: return -1 elif current_tuple > remote_tuple: return 1 else: return 0 def fetch_github_version() -> Optional[tuple[str, str]]: """ Fetch the latest version information from GitHub repository. Returns: Tuple of (version, last_updated) if successful, None if failed """ if not HAS_URLLIB: logger.warning("urllib not available, cannot check for updates") return None github_url = "https://raw.githubusercontent.com/BeehiveInnovations/pal-mcp-server/main/config.py" try: # Set a 10-second timeout with urlopen(github_url, timeout=10) as response: if response.status != 200: logger.warning(f"HTTP error while checking GitHub: {response.status}") return None content = response.read().decode("utf-8") # Extract version using regex version_match = re.search(r'__version__\s*=\s*["\']([^"\']+)["\']', content) updated_match = re.search(r'__updated__\s*=\s*["\']([^"\']+)["\']', content) if version_match: remote_version = version_match.group(1) remote_updated = updated_match.group(1) if updated_match else "Unknown" return (remote_version, remote_updated) else: logger.warning("Could not parse version from GitHub config.py") return None except HTTPError as e: logger.warning(f"HTTP error while checking GitHub: {e.code}") return None except URLError as e: logger.warning(f"URL error while checking GitHub: {e.reason}") return None except Exception as e: logger.warning(f"Error checking GitHub for updates: {e}") return None class VersionTool(BaseTool): """ Tool for displaying PAL MCP Server version and system information. This tool provides: - Current server version - Last update date - Author information - Python version - Platform information """ def get_name(self) -> str: return "version" def get_description(self) -> str: return "Get server version, configuration details, and list of available tools." def get_input_schema(self) -> dict[str, Any]: """Return the JSON schema for the tool's input""" return { "type": "object", "properties": {}, "required": [], "additionalProperties": False, } def get_annotations(self) -> Optional[dict[str, Any]]: """Return tool annotations indicating this is a read-only tool""" return {"readOnlyHint": True} def get_system_prompt(self) -> str: """No AI model needed for this tool""" return "" def get_request_model(self): """Return the Pydantic model for request validation.""" return ToolRequest def requires_model(self) -> bool: return False async def prepare_prompt(self, request: ToolRequest) -> str: """Not used for this utility tool""" return "" def format_response(self, response: str, request: ToolRequest, model_info: dict = None) -> str: """Not used for this utility tool""" return response async def execute(self, arguments: dict[str, Any]) -> list[TextContent]: """ Display PAL MCP Server version and system information. This overrides the base class execute to provide direct output without AI model calls. Args: arguments: Standard tool arguments (none required) Returns: Formatted version and system information """ output_lines = ["# PAL MCP Server Version\n"] # Server version information output_lines.append("## Server Information") output_lines.append(f"**Current Version**: {__version__}") output_lines.append(f"**Last Updated**: {__updated__}") output_lines.append(f"**Author**: {__author__}") model_selection_metadata = {"mode": "unknown", "default_model": None} model_selection_display = "Model selection status unavailable" # Model selection configuration try: from config import DEFAULT_MODEL from tools.shared.base_tool import BaseTool auto_mode = BaseTool.is_effective_auto_mode(self) if auto_mode: output_lines.append( "**Model Selection**: Auto model selection mode (call `listmodels` to inspect options)" ) model_selection_metadata = {"mode": "auto", "default_model": DEFAULT_MODEL} model_selection_display = "Auto model selection (use `listmodels` for options)" else: output_lines.append(f"**Model Selection**: Default model set to `{DEFAULT_MODEL}`") model_selection_metadata = {"mode": "default", "default_model": DEFAULT_MODEL} model_selection_display = f"Default model: `{DEFAULT_MODEL}`" except Exception as exc: logger.debug(f"Could not determine model selection mode: {exc}") output_lines.append("") output_lines.append("## Quick Summary — relay everything below") output_lines.append(f"- Version `{__version__}` (updated {__updated__})") output_lines.append(f"- {model_selection_display}") output_lines.append("- Run `listmodels` for the complete model catalog and capabilities") output_lines.append("") # Try to get client information try: # We need access to the server instance # This is a bit hacky but works for now import server as server_module from utils.client_info import format_client_info, get_client_info_from_context client_info = get_client_info_from_context(server_module.server) if client_info: formatted = format_client_info(client_info) output_lines.append(f"**Connected Client**: {formatted}") except Exception as e: logger.debug(f"Could not get client info: {e}") # Get the current working directory (MCP server location) current_path = Path.cwd() output_lines.append(f"**Installation Path**: `{current_path}`") output_lines.append("") output_lines.append("## Agent Reporting Guidance") output_lines.append( "Agents MUST report: version, model-selection status, configured providers, and available-model count." ) output_lines.append("Repeat the quick-summary bullets verbatim in your reply.") output_lines.append("Reference `listmodels` when users ask about model availability or capabilities.") output_lines.append("") # Check for updates from GitHub output_lines.append("## Update Status") try: github_info = fetch_github_version() if github_info: remote_version, remote_updated = github_info comparison = compare_versions(__version__, remote_version) output_lines.append(f"**Latest Version (GitHub)**: {remote_version}") output_lines.append(f"**Latest Updated**: {remote_updated}") if comparison < 0: # Update available output_lines.append("") output_lines.append("🚀 **UPDATE AVAILABLE!**") output_lines.append( f"Your version `{__version__}` is older than the latest version `{remote_version}`" ) output_lines.append("") output_lines.append("**To update:**") output_lines.append("```bash") output_lines.append(f"cd {current_path}") output_lines.append("git pull") output_lines.append("```") output_lines.append("") output_lines.append("*Note: Restart your session after updating to use the new version.*") elif comparison == 0: # Up to date output_lines.append("") output_lines.append("✅ **UP TO DATE**") output_lines.append("You are running the latest version.") else: # Ahead of remote (development version) output_lines.append("") output_lines.append("🔬 **DEVELOPMENT VERSION**") output_lines.append( f"Your version `{__version__}` is ahead of the published version `{remote_version}`" ) output_lines.append("You may be running a development or custom build.") else: output_lines.append("❌ **Could not check for updates**") output_lines.append("Unable to connect to GitHub or parse version information.") output_lines.append("Check your internet connection or try again later.") except Exception as e: logger.error(f"Error during version check: {e}") output_lines.append("❌ **Error checking for updates**") output_lines.append(f"Error: {str(e)}") output_lines.append("") # Configuration information output_lines.append("## Configuration") # Check for configured providers try: from providers.registry import ModelProviderRegistry from providers.shared import ProviderType provider_status = [] # Check each provider type provider_types = [ ProviderType.GOOGLE, ProviderType.OPENAI, ProviderType.XAI, ProviderType.DIAL, ProviderType.OPENROUTER, ProviderType.CUSTOM, ] provider_names = ["Google Gemini", "OpenAI", "X.AI", "DIAL", "OpenRouter", "Custom/Local"] for provider_type, provider_name in zip(provider_types, provider_names): provider = ModelProviderRegistry.get_provider(provider_type) status = "✅ Configured" if provider is not None else "❌ Not configured" provider_status.append(f"- **{provider_name}**: {status}") output_lines.append("**Providers**:") output_lines.extend(provider_status) # Get total available models try: available_models = ModelProviderRegistry.get_available_models(respect_restrictions=True) output_lines.append(f"\n\n**Available Models**: {len(available_models)}") except Exception: output_lines.append("\n\n**Available Models**: Unknown") except Exception as e: logger.warning(f"Error checking provider configuration: {e}") output_lines.append("\n\n**Providers**: Error checking configuration") output_lines.append("") # Format output content = "\n".join(output_lines) tool_output = ToolOutput( status="success", content=content, content_type="text", metadata={ "tool_name": self.name, "server_version": __version__, "last_updated": __updated__, "python_version": f"{sys.version_info.major}.{sys.version_info.minor}.{sys.version_info.micro}", "platform": f"{platform.system()} {platform.release()}", "model_selection_mode": model_selection_metadata["mode"], "default_model": model_selection_metadata["default_model"], }, ) return [TextContent(type="text", text=tool_output.model_dump_json())] def get_model_category(self) -> ToolModelCategory: """Return the model category for this tool.""" return ToolModelCategory.FAST_RESPONSE # Simple version info, no AI needed ================================================ FILE: tools/workflow/__init__.py ================================================ """ Workflow tools for PAL MCP. Workflow tools follow a multi-step pattern with forced pauses between steps to encourage thorough investigation and analysis. They inherit from WorkflowTool which combines BaseTool with BaseWorkflowMixin. Available workflow tools: - debug: Systematic investigation and root cause analysis - planner: Sequential planning (special case - no AI calls) - analyze: Code analysis workflow - codereview: Code review workflow - precommit: Pre-commit validation workflow - refactor: Refactoring analysis workflow - thinkdeep: Deep thinking workflow """ from .base import WorkflowTool from .schema_builders import WorkflowSchemaBuilder from .workflow_mixin import BaseWorkflowMixin __all__ = ["WorkflowTool", "WorkflowSchemaBuilder", "BaseWorkflowMixin"] ================================================ FILE: tools/workflow/base.py ================================================ """ Base class for workflow MCP tools. Workflow tools follow a multi-step pattern: 1. CLI calls tool with work step data 2. Tool tracks findings and progress 3. Tool forces the CLI to pause and investigate between steps 4. Once work is complete, tool calls external AI model for expert analysis 5. Tool returns structured response combining investigation + expert analysis They combine BaseTool's capabilities with BaseWorkflowMixin's workflow functionality and use SchemaBuilder for consistent schema generation. """ from abc import abstractmethod from typing import Any, Optional from tools.shared.base_models import WorkflowRequest from tools.shared.base_tool import BaseTool from .schema_builders import WorkflowSchemaBuilder from .workflow_mixin import BaseWorkflowMixin class WorkflowTool(BaseTool, BaseWorkflowMixin): """ Base class for workflow (multi-step) tools. Workflow tools perform systematic multi-step work with expert analysis. They benefit from: - Automatic workflow orchestration from BaseWorkflowMixin - Automatic schema generation using SchemaBuilder - Inherited conversation handling and file processing from BaseTool - Progress tracking with ConsolidatedFindings - Expert analysis integration To create a workflow tool: 1. Inherit from WorkflowTool 2. Tool name is automatically provided by get_name() method 3. Implement get_required_actions() for step guidance 4. Implement should_call_expert_analysis() for completion criteria 5. Implement prepare_expert_analysis_context() for expert prompts 6. Optionally implement get_tool_fields() for additional fields 7. Optionally override workflow behavior methods Example: class DebugTool(WorkflowTool): # get_name() is inherited from BaseTool def get_tool_fields(self) -> Dict[str, Dict[str, Any]]: return { "hypothesis": { "type": "string", "description": "Current theory about the issue", } } def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int ) -> List[str]: return ["Examine relevant code files", "Trace execution flow", "Check error logs"] def should_call_expert_analysis(self, consolidated_findings) -> bool: return len(consolidated_findings.relevant_files) > 0 """ def __init__(self): """Initialize WorkflowTool with proper multiple inheritance.""" BaseTool.__init__(self) BaseWorkflowMixin.__init__(self) def get_tool_fields(self) -> dict[str, dict[str, Any]]: """ Return tool-specific field definitions beyond the standard workflow fields. Workflow tools automatically get all standard workflow fields: - step, step_number, total_steps, next_step_required - findings, files_checked, relevant_files, relevant_context - issues_found, confidence, hypothesis - plus common fields (model, temperature, etc.) Override this method to add additional tool-specific fields. Returns: Dict mapping field names to JSON schema objects Example: return { "severity_filter": { "type": "string", "enum": ["low", "medium", "high"], "description": "Minimum severity level to report", } } """ return {} def get_required_fields(self) -> list[str]: """ Return additional required fields beyond the standard workflow requirements. Workflow tools automatically require: - step, step_number, total_steps, next_step_required, findings - model (if in auto mode) Override this to add additional required fields. Returns: List of additional required field names """ return [] def get_annotations(self) -> Optional[dict[str, Any]]: """ Return tool annotations. Workflow tools are read-only by default. All workflow tools perform analysis and investigation without modifying the environment. They may call external AI models for expert analysis, but they don't write files or make system changes. Override this method if your workflow tool needs different annotations. Returns: Dictionary with readOnlyHint set to True """ return {"readOnlyHint": True} def get_input_schema(self) -> dict[str, Any]: """ Generate the complete input schema using SchemaBuilder. This method automatically combines: - Standard workflow fields (step, findings, etc.) - Common fields (temperature, thinking_mode, etc.) - Model field with proper auto-mode handling - Tool-specific fields from get_tool_fields() - Required fields from get_required_fields() Returns: Complete JSON schema for the workflow tool """ requires_model = self.requires_model() model_field_schema = self.get_model_field_schema() if requires_model else None auto_mode = self.is_effective_auto_mode() if requires_model else False return WorkflowSchemaBuilder.build_schema( tool_specific_fields=self.get_tool_fields(), required_fields=self.get_required_fields(), model_field_schema=model_field_schema, auto_mode=auto_mode, tool_name=self.get_name(), require_model=requires_model, ) def get_workflow_request_model(self): """ Return the workflow request model class. Workflow tools use WorkflowRequest by default, which includes all the standard workflow fields. Override this if your tool needs a custom request model. """ return WorkflowRequest # Implement the abstract method from BaseWorkflowMixin def get_work_steps(self, request) -> list[str]: """ Default implementation - workflow tools typically don't need predefined steps. The workflow is driven by the CLI's investigation process rather than predefined steps. Override this if your tool needs specific step guidance. """ return [] # Default implementations for common workflow patterns def get_standard_required_actions(self, step_number: int, confidence: str, base_actions: list[str]) -> list[str]: """ Helper method to generate standard required actions based on confidence and step. This provides common patterns that most workflow tools can use: - Early steps: broad exploration - Low confidence: deeper investigation - Medium/high confidence: verification and confirmation Args: step_number: Current step number confidence: Current confidence level base_actions: Tool-specific base actions Returns: List of required actions appropriate for the current state """ if step_number == 1: # Initial investigation return [ "Search for code related to the reported issue or symptoms", "Examine relevant files and understand the current implementation", "Understand the project structure and locate relevant modules", "Identify how the affected functionality is supposed to work", ] elif confidence in ["exploring", "low"]: # Need deeper investigation return base_actions + [ "Trace method calls and data flow through the system", "Check for edge cases, boundary conditions, and assumptions in the code", "Look for related configuration, dependencies, or external factors", ] elif confidence in ["medium", "high"]: # Close to solution - need confirmation return base_actions + [ "Examine the exact code sections where you believe the issue occurs", "Trace the execution path that leads to the failure", "Verify your hypothesis with concrete code evidence", "Check for any similar patterns elsewhere in the codebase", ] else: # General continued investigation return base_actions + [ "Continue examining the code paths identified in your hypothesis", "Gather more evidence using appropriate investigation tools", "Test edge cases and boundary conditions", "Look for patterns that confirm or refute your theory", ] def should_call_expert_analysis_default(self, consolidated_findings) -> bool: """ Default implementation for expert analysis decision. This provides a reasonable default that most workflow tools can use: - Call expert analysis if we have relevant files or significant findings - Skip if confidence is "certain" (handled by the workflow mixin) Override this for tool-specific logic. Args: consolidated_findings: The consolidated findings from all work steps Returns: True if expert analysis should be called """ # Call expert analysis if we have relevant files or substantial findings return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_standard_expert_context( self, consolidated_findings, initial_description: str, context_sections: dict[str, str] = None ) -> str: """ Helper method to prepare standard expert analysis context. This provides a common structure that most workflow tools can use, with the ability to add tool-specific sections. Args: consolidated_findings: The consolidated findings from all work steps initial_description: Description of the initial request/issue context_sections: Optional additional sections to include Returns: Formatted context string for expert analysis """ context_parts = [f"=== ISSUE DESCRIPTION ===\n{initial_description}\n=== END DESCRIPTION ==="] # Add work progression if consolidated_findings.findings: findings_text = "\n".join(consolidated_findings.findings) context_parts.append(f"\n=== INVESTIGATION FINDINGS ===\n{findings_text}\n=== END FINDINGS ===") # Add relevant methods if available if consolidated_findings.relevant_context: methods_text = "\n".join(f"- {method}" for method in consolidated_findings.relevant_context) context_parts.append(f"\n=== RELEVANT METHODS/FUNCTIONS ===\n{methods_text}\n=== END METHODS ===") # Add hypothesis evolution if available if consolidated_findings.hypotheses: hypotheses_text = "\n".join( f"Step {h['step']} ({h['confidence']} confidence): {h['hypothesis']}" for h in consolidated_findings.hypotheses ) context_parts.append(f"\n=== HYPOTHESIS EVOLUTION ===\n{hypotheses_text}\n=== END HYPOTHESES ===") # Add issues found if available if consolidated_findings.issues_found: issues_text = "\n".join( f"[{issue.get('severity', 'unknown').upper()}] {issue.get('description', 'No description')}" for issue in consolidated_findings.issues_found ) context_parts.append(f"\n=== ISSUES IDENTIFIED ===\n{issues_text}\n=== END ISSUES ===") # Add tool-specific sections if context_sections: for section_title, section_content in context_sections.items(): context_parts.append( f"\n=== {section_title.upper()} ===\n{section_content}\n=== END {section_title.upper()} ===" ) return "\n".join(context_parts) def handle_completion_without_expert_analysis( self, request, consolidated_findings, initial_description: str = None ) -> dict[str, Any]: """ Generic handler for completion when expert analysis is not needed. This provides a standard response format for when the tool determines that external expert analysis is not required. All workflow tools can use this generic implementation or override for custom behavior. Args: request: The workflow request object consolidated_findings: The consolidated findings from all work steps initial_description: Optional initial description (defaults to request.step) Returns: Dictionary with completion response data """ # Prepare work summary using inheritance hook work_summary = self.prepare_work_summary() return { "status": self.get_completion_status(), self.get_completion_data_key(): { "initial_request": initial_description or request.step, "steps_taken": len(consolidated_findings.findings), "files_examined": list(consolidated_findings.files_checked), "relevant_files": list(consolidated_findings.relevant_files), "relevant_context": list(consolidated_findings.relevant_context), "work_summary": work_summary, "final_analysis": self.get_final_analysis_from_request(request), "confidence_level": self.get_confidence_level(request), }, "next_steps": self.get_completion_message(), "skip_expert_analysis": True, "expert_analysis": { "status": self.get_skip_expert_analysis_status(), "reason": self.get_skip_reason(), }, } # Inheritance hooks for customization def prepare_work_summary(self) -> str: """ Prepare a summary of the work performed. Override for custom summaries. Default implementation provides a basic summary. """ try: return self._prepare_work_summary() except AttributeError: try: return f"Completed {len(self.work_history)} work steps" except AttributeError: return "Completed 0 work steps" def get_completion_status(self) -> str: """Get the status to use when completing without expert analysis.""" return "high_confidence_completion" def get_completion_data_key(self) -> str: """Get the key name for completion data in the response.""" return f"complete_{self.get_name()}" def get_final_analysis_from_request(self, request) -> Optional[str]: """Extract final analysis from request. Override for tool-specific extraction.""" try: return request.hypothesis except AttributeError: return None def get_confidence_level(self, request) -> str: """Get confidence level from request. Override for tool-specific logic.""" try: return request.confidence or "high" except AttributeError: return "high" def get_completion_message(self) -> str: """Get completion message. Override for tool-specific messaging.""" return ( f"{self.get_name().capitalize()} complete with high confidence. You have identified the exact " "analysis and solution. MANDATORY: Present the user with the results " "and proceed with implementing the solution without requiring further " "consultation. Focus on the precise, actionable steps needed." ) def get_skip_reason(self) -> str: """Get reason for skipping expert analysis. Override for tool-specific reasons.""" return f"{self.get_name()} completed with sufficient confidence" def get_skip_expert_analysis_status(self) -> str: """Get status for skipped expert analysis. Override for tool-specific status.""" return "skipped_by_tool_design" def is_continuation_workflow(self, request) -> bool: """ Check if this is a continuation workflow that should skip multi-step investigation. When continuation_id is provided, the workflow typically continues from a previous conversation and should go directly to expert analysis rather than starting a new multi-step investigation. Args: request: The workflow request object Returns: True if this is a continuation that should skip multi-step workflow """ continuation_id = self.get_request_continuation_id(request) return bool(continuation_id) # Abstract methods that must be implemented by specific workflow tools # (These are inherited from BaseWorkflowMixin and must be implemented) @abstractmethod def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each work phase. Args: step_number: Current step number confidence: Current confidence level findings: Current findings text total_steps: Total estimated steps request: Optional request object for continuation-aware decisions Returns: List of required actions for the current step """ pass @abstractmethod def should_call_expert_analysis(self, consolidated_findings) -> bool: """Decide when to call external model based on tool-specific criteria""" pass @abstractmethod def prepare_expert_analysis_context(self, consolidated_findings) -> str: """Prepare context for external model call""" pass # Default execute method - delegates to workflow async def execute(self, arguments: dict[str, Any]) -> list: """Execute the workflow tool - delegates to BaseWorkflowMixin.""" return await self.execute_workflow(arguments) ================================================ FILE: tools/workflow/schema_builders.py ================================================ """ Schema builders for workflow MCP tools. This module provides workflow-specific schema generation functionality, keeping workflow concerns separated from simple tool concerns. """ from typing import Any from ..shared.base_models import WORKFLOW_FIELD_DESCRIPTIONS from ..shared.schema_builders import SchemaBuilder class WorkflowSchemaBuilder: """ Schema builder for workflow MCP tools. This class extends the base SchemaBuilder with workflow-specific fields and schema generation logic, maintaining separation of concerns. """ # Workflow-specific field schemas WORKFLOW_FIELD_SCHEMAS = { "step": { "type": "string", "description": WORKFLOW_FIELD_DESCRIPTIONS["step"], }, "step_number": { "type": "integer", "minimum": 1, "description": WORKFLOW_FIELD_DESCRIPTIONS["step_number"], }, "total_steps": { "type": "integer", "minimum": 1, "description": WORKFLOW_FIELD_DESCRIPTIONS["total_steps"], }, "next_step_required": { "type": "boolean", "description": WORKFLOW_FIELD_DESCRIPTIONS["next_step_required"], }, "findings": { "type": "string", "description": WORKFLOW_FIELD_DESCRIPTIONS["findings"], }, "files_checked": { "type": "array", "items": {"type": "string"}, "description": WORKFLOW_FIELD_DESCRIPTIONS["files_checked"], }, "relevant_files": { "type": "array", "items": {"type": "string"}, "description": WORKFLOW_FIELD_DESCRIPTIONS["relevant_files"], }, "relevant_context": { "type": "array", "items": {"type": "string"}, "description": WORKFLOW_FIELD_DESCRIPTIONS["relevant_context"], }, "issues_found": { "type": "array", "items": {"type": "object"}, "description": WORKFLOW_FIELD_DESCRIPTIONS["issues_found"], }, "confidence": { "type": "string", "enum": ["exploring", "low", "medium", "high", "very_high", "almost_certain", "certain"], "description": WORKFLOW_FIELD_DESCRIPTIONS["confidence"], }, "hypothesis": { "type": "string", "description": WORKFLOW_FIELD_DESCRIPTIONS["hypothesis"], }, "use_assistant_model": { "type": "boolean", "default": True, "description": WORKFLOW_FIELD_DESCRIPTIONS["use_assistant_model"], }, } @staticmethod def build_schema( tool_specific_fields: dict[str, dict[str, Any]] = None, required_fields: list[str] = None, model_field_schema: dict[str, Any] = None, auto_mode: bool = False, tool_name: str = None, excluded_workflow_fields: list[str] = None, excluded_common_fields: list[str] = None, require_model: bool = False, ) -> dict[str, Any]: """ Build complete schema for workflow tools. Args: tool_specific_fields: Additional fields specific to the tool required_fields: List of required field names (beyond workflow defaults) model_field_schema: Schema for the model field auto_mode: Whether the tool is in auto mode (affects model requirement) tool_name: Name of the tool (for schema title) excluded_workflow_fields: Workflow fields to exclude from schema (e.g., for planning tools) excluded_common_fields: Common fields to exclude from schema Returns: Complete JSON schema for the workflow tool """ properties = {} # Add workflow fields first, excluding any specified fields workflow_fields = WorkflowSchemaBuilder.WORKFLOW_FIELD_SCHEMAS.copy() if excluded_workflow_fields: for field in excluded_workflow_fields: workflow_fields.pop(field, None) properties.update(workflow_fields) # Add common fields (temperature, thinking_mode, etc.) from base builder, excluding any specified fields common_fields = SchemaBuilder.COMMON_FIELD_SCHEMAS.copy() if excluded_common_fields: for field in excluded_common_fields: common_fields.pop(field, None) properties.update(common_fields) # Add model field if provided if model_field_schema: properties["model"] = model_field_schema # Add tool-specific fields if provided if tool_specific_fields: properties.update(tool_specific_fields) # Build required fields list - workflow tools have standard required fields standard_required = ["step", "step_number", "total_steps", "next_step_required", "findings"] # Filter out excluded fields from required fields if excluded_workflow_fields: standard_required = [field for field in standard_required if field not in excluded_workflow_fields] required = standard_required + (required_fields or []) if (auto_mode or require_model) and "model" not in required: required.append("model") # Build the complete schema schema = { "$schema": "http://json-schema.org/draft-07/schema#", "type": "object", "properties": properties, "required": required, "additionalProperties": False, } if tool_name: schema["title"] = f"{tool_name.capitalize()}Request" return schema @staticmethod def get_workflow_fields() -> dict[str, dict[str, Any]]: """Get the standard field schemas for workflow tools.""" combined = {} combined.update(WorkflowSchemaBuilder.WORKFLOW_FIELD_SCHEMAS) combined.update(SchemaBuilder.COMMON_FIELD_SCHEMAS) return combined @staticmethod def get_workflow_only_fields() -> dict[str, dict[str, Any]]: """Get only the workflow-specific field schemas.""" return WorkflowSchemaBuilder.WORKFLOW_FIELD_SCHEMAS.copy() ================================================ FILE: tools/workflow/workflow_mixin.py ================================================ """ Workflow Mixin for PAL MCP Tools This module provides a sophisticated workflow-based pattern that enables tools to perform multi-step work with structured findings and expert analysis. Key Components: - BaseWorkflowMixin: Abstract base class providing comprehensive workflow functionality The workflow pattern enables tools like debug, precommit, and codereview to perform systematic multi-step work with pause/resume capabilities, context-aware file embedding, and seamless integration with external AI models for expert analysis. Features: - Multi-step workflow orchestration with pause/resume - Context-aware file embedding optimization - Expert analysis integration with token budgeting - Conversation memory and threading support - Proper inheritance-based architecture (no hasattr/getattr) - Comprehensive type annotations for IDE support """ import json import logging import os import re from abc import ABC, abstractmethod from typing import Any, Optional from mcp.types import TextContent from config import MCP_PROMPT_SIZE_LIMIT from utils.conversation_memory import add_turn, create_thread from ..shared.base_models import ConsolidatedFindings from ..shared.exceptions import ToolExecutionError logger = logging.getLogger(__name__) class BaseWorkflowMixin(ABC): """ Abstract base class providing guided workflow functionality for tools. This class implements a sophisticated workflow pattern where the CLI performs systematic local work before calling external models for expert analysis. Tools can inherit from this class to gain comprehensive workflow capabilities. Architecture: - Uses proper inheritance patterns instead of hasattr/getattr - Provides hook methods with default implementations - Requires abstract methods to be implemented by subclasses - Fully type-annotated for excellent IDE support Context-Aware File Embedding: - Intermediate steps: Only reference file names (saves the CLI's context) - Final steps: Embed full file content for expert analysis - Integrates with existing token budgeting infrastructure Requirements: This class expects to be used with BaseTool and requires implementation of: - get_model_provider(model_name) - _resolve_model_context(arguments, request) - get_system_prompt() - get_default_temperature() - _prepare_file_content_for_prompt() """ def __init__(self) -> None: super().__init__() self.work_history: list[dict[str, Any]] = [] self.consolidated_findings: ConsolidatedFindings = ConsolidatedFindings() self.initial_request: Optional[str] = None # ================================================================================ # Abstract Methods - Required Implementation by BaseTool or Subclasses # ================================================================================ @abstractmethod def get_name(self) -> str: """Return the name of this tool. Usually provided by BaseTool.""" pass @abstractmethod def get_workflow_request_model(self) -> type: """Return the request model class for this workflow tool.""" pass @abstractmethod def get_system_prompt(self) -> str: """Return the system prompt for this tool. Usually provided by BaseTool.""" pass @abstractmethod def get_language_instruction(self) -> str: """Return the language instruction for localization. Usually provided by BaseTool.""" pass @abstractmethod def get_default_temperature(self) -> float: """Return the default temperature for this tool. Usually provided by BaseTool.""" pass @abstractmethod def get_model_provider(self, model_name: str) -> Any: """Get model provider for the given model. Usually provided by BaseTool.""" pass @abstractmethod def _resolve_model_context(self, arguments: dict[str, Any], request: Any) -> tuple[str, Any]: """Resolve model context from arguments. Usually provided by BaseTool.""" pass @abstractmethod def _prepare_file_content_for_prompt( self, request_files: list[str], continuation_id: Optional[str], context_description: str = "New files", max_tokens: Optional[int] = None, reserve_tokens: int = 1_000, remaining_budget: Optional[int] = None, arguments: Optional[dict[str, Any]] = None, model_context: Optional[Any] = None, ) -> tuple[str, list[str]]: """Prepare file content for prompts. Usually provided by BaseTool.""" pass # ================================================================================ # Abstract Methods - Tool-Specific Implementation Required # ================================================================================ @abstractmethod def get_work_steps(self, request: Any) -> list[str]: """Define tool-specific work steps and criteria""" pass @abstractmethod def get_required_actions( self, step_number: int, confidence: str, findings: str, total_steps: int, request=None ) -> list[str]: """Define required actions for each work phase. Args: step_number: Current step (1-based) confidence: Current confidence level (exploring, low, medium, high, certain) findings: Current findings text total_steps: Total estimated steps for this work request: Optional request object for continuation-aware decisions Returns: List of specific actions the CLI should take before calling tool again """ pass # ================================================================================ # Hook Methods - Default Implementations with Override Capability # ================================================================================ def should_call_expert_analysis(self, consolidated_findings: ConsolidatedFindings, request=None) -> bool: """ Decide when to call external model based on tool-specific criteria. Default implementation for tools that don't use expert analysis. Override this for tools that do use expert analysis. Args: consolidated_findings: Findings from workflow steps request: Current request object (optional for backwards compatibility) """ if not self.requires_expert_analysis(): return False # Check if user requested to skip assistant model if request and not self.get_request_use_assistant_model(request): return False # Default logic for tools that support expert analysis return ( len(consolidated_findings.relevant_files) > 0 or len(consolidated_findings.findings) >= 2 or len(consolidated_findings.issues_found) > 0 ) def prepare_expert_analysis_context(self, consolidated_findings: ConsolidatedFindings) -> str: """ Prepare context for external model call. Default implementation for tools that don't use expert analysis. Override this for tools that do use expert analysis. """ if not self.requires_expert_analysis(): return "" # Default context preparation context_parts = [ f"=== {self.get_name().upper()} WORK SUMMARY ===", f"Total steps: {len(consolidated_findings.findings)}", f"Files examined: {len(consolidated_findings.files_checked)}", f"Relevant files: {len(consolidated_findings.relevant_files)}", "", "=== WORK PROGRESSION ===", ] for finding in consolidated_findings.findings: context_parts.append(finding) return "\n".join(context_parts) def requires_expert_analysis(self) -> bool: """ Override this to completely disable expert analysis for the tool. Returns True if the tool supports expert analysis (default). Returns False if the tool is self-contained (like planner). """ return True def should_include_files_in_expert_prompt(self) -> bool: """ Whether to include file content in the expert analysis prompt. Override this to return True if your tool needs files in the prompt. """ return False def should_embed_system_prompt(self) -> bool: """ Whether to embed the system prompt in the main prompt. Override this to return True if your tool needs the system prompt embedded. """ return False def get_expert_thinking_mode(self) -> str: """ Get the thinking mode for expert analysis. Override this to customize the thinking mode. """ return "high" def get_request_temperature(self, request) -> float: """Get temperature from request. Override for custom temperature handling.""" try: return request.temperature if request.temperature is not None else self.get_default_temperature() except AttributeError: return self.get_default_temperature() def get_validated_temperature(self, request, model_context: Any) -> tuple[float, list[str]]: """ Get temperature from request and validate it against model constraints. This is a convenience method that combines temperature extraction and validation for workflow tools. It ensures temperature is within valid range for the model. Args: request: The request object containing temperature model_context: Model context object containing model info Returns: Tuple of (validated_temperature, warning_messages) """ temperature = self.get_request_temperature(request) return self.validate_and_correct_temperature(temperature, model_context) def get_request_thinking_mode(self, request) -> str: """Get thinking mode from request. Override for custom thinking mode handling.""" try: return request.thinking_mode if request.thinking_mode is not None else self.get_expert_thinking_mode() except AttributeError: return self.get_expert_thinking_mode() def get_expert_analysis_instruction(self) -> str: """ Get the instruction to append after the expert context. Override this to provide tool-specific instructions. """ return "Please provide expert analysis based on the investigation findings." def get_request_use_assistant_model(self, request) -> bool: """ Get use_assistant_model from request. Override for custom assistant model handling. Args: request: Current request object Returns: True if assistant model should be used, False otherwise """ try: return request.use_assistant_model if request.use_assistant_model is not None else True except AttributeError: return True def get_step_guidance_message(self, request) -> str: """ Get step guidance message. Override for tool-specific guidance. Default implementation uses required actions. """ required_actions = self.get_required_actions( request.step_number, self.get_request_confidence(request), request.findings, request.total_steps, request ) next_step_number = request.step_number + 1 return ( f"MANDATORY: DO NOT call the {self.get_name()} tool again immediately. " f"You MUST first work using appropriate tools. " f"REQUIRED ACTIONS before calling {self.get_name()} step {next_step_number}:\n" + "\n".join(f"{i + 1}. {action}" for i, action in enumerate(required_actions)) + f"\n\nOnly call {self.get_name()} again with step_number: {next_step_number} " f"AFTER completing this work." ) def _prepare_files_for_expert_analysis(self) -> str: """ Prepare file content for expert analysis. EXPERT ANALYSIS REQUIRES ACTUAL FILE CONTENT: Expert analysis needs actual file content of all unique files marked as relevant throughout the workflow, regardless of conversation history optimization. SIMPLIFIED LOGIC: Expert analysis gets all unique files from relevant_files across the entire workflow. This includes: - Current step's relevant_files (consolidated_findings.relevant_files) - Plus any additional relevant_files from conversation history (if continued workflow) This ensures expert analysis has complete context without including irrelevant files. """ all_relevant_files = set() # 1. Get files from current consolidated relevant_files all_relevant_files.update(self.consolidated_findings.relevant_files) # 2. Get additional relevant_files from conversation history (if continued workflow) try: current_arguments = self.get_current_arguments() if current_arguments: continuation_id = current_arguments.get("continuation_id") if continuation_id: from utils.conversation_memory import get_conversation_file_list, get_thread thread_context = get_thread(continuation_id) if thread_context: # Get all files from conversation (these were relevant_files in previous steps) conversation_files = get_conversation_file_list(thread_context) all_relevant_files.update(conversation_files) logger.debug( f"[WORKFLOW_FILES] {self.get_name()}: Added {len(conversation_files)} files from conversation history" ) except Exception as e: logger.warning(f"[WORKFLOW_FILES] {self.get_name()}: Could not get conversation files: {e}") # Convert to list and remove any empty/None values files_for_expert = [f for f in all_relevant_files if f and f.strip()] if not files_for_expert: logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: No relevant files found for expert analysis") return "" # Expert analysis needs actual file content, bypassing conversation optimization try: file_content, processed_files = self._force_embed_files_for_expert_analysis(files_for_expert) logger.info( f"[WORKFLOW_FILES] {self.get_name()}: Prepared {len(processed_files)} unique relevant files for expert analysis " f"(from {len(self.consolidated_findings.relevant_files)} current relevant files)" ) return file_content except Exception as e: logger.error(f"[WORKFLOW_FILES] {self.get_name()}: Failed to prepare files for expert analysis: {e}") return "" def _force_embed_files_for_expert_analysis(self, files: list[str]) -> tuple[str, list[str]]: """ Force embed files for expert analysis, bypassing conversation history filtering. Expert analysis has different requirements than normal workflow steps: - Normal steps: Optimize tokens by skipping files in conversation history - Expert analysis: Needs actual file content regardless of conversation history Args: files: List of file paths to embed Returns: tuple[str, list[str]]: (file_content, processed_files) """ # Use read_files directly with token budgeting, bypassing filter_new_files from utils.file_utils import expand_paths, read_files # Get token budget for files current_model_context = self.get_current_model_context() if current_model_context: try: token_allocation = current_model_context.calculate_token_allocation() max_tokens = token_allocation.file_tokens logger.debug( f"[WORKFLOW_FILES] {self.get_name()}: Using {max_tokens:,} tokens for expert analysis files" ) except Exception as e: logger.warning(f"[WORKFLOW_FILES] {self.get_name()}: Failed to get token allocation: {e}") max_tokens = 100_000 # Fallback else: max_tokens = 100_000 # Fallback # Read files directly without conversation history filtering logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: Force embedding {len(files)} files for expert analysis") file_content = read_files( files, max_tokens=max_tokens, reserve_tokens=1000, include_line_numbers=self.wants_line_numbers_by_default(), ) # Expand paths to get individual files for tracking processed_files = expand_paths(files) logger.debug( f"[WORKFLOW_FILES] {self.get_name()}: Expert analysis embedding: {len(processed_files)} files, " f"{len(file_content):,} characters" ) return file_content, processed_files def wants_line_numbers_by_default(self) -> bool: """ Whether this tool wants line numbers in file content by default. Override this to customize line number behavior. """ return True # Most workflow tools benefit from line numbers for analysis def _add_files_to_expert_context(self, expert_context: str, file_content: str) -> str: """ Add file content to the expert context. Override this to customize how files are added to the context. """ return f"{expert_context}\n\n=== ESSENTIAL FILES ===\n{file_content}\n=== END ESSENTIAL FILES ===" # ================================================================================ # Context-Aware File Embedding - Core Implementation # ================================================================================ def _handle_workflow_file_context(self, request: Any, arguments: dict[str, Any]) -> None: """ Handle file context appropriately based on workflow phase. CONTEXT-AWARE FILE EMBEDDING STRATEGY: 1. Intermediate steps + continuation: Only reference file names (save the CLI's context) 2. Final step: Embed full file content for expert analysis 3. Expert analysis: Always embed relevant files with token budgeting This prevents wasting the CLI's limited context on intermediate steps while ensuring the final expert analysis has complete file context. """ continuation_id = self.get_request_continuation_id(request) is_final_step = not self.get_request_next_step_required(request) step_number = self.get_request_step_number(request) # Extract model context for token budgeting model_context = arguments.get("_model_context") self._model_context = model_context # Clear any previous file context to ensure clean state self._embedded_file_content = "" self._file_reference_note = "" self._actually_processed_files = [] # Determine if we should embed files or just reference them should_embed_files = self._should_embed_files_in_workflow_step(step_number, continuation_id, is_final_step) if should_embed_files: # Final step or expert analysis - embed full file content logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: Embedding files for final step/expert analysis") self._embed_workflow_files(request, arguments) else: # Intermediate step with continuation - only reference file names logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: Only referencing file names for intermediate step") self._reference_workflow_files(request) def _should_embed_files_in_workflow_step( self, step_number: int, continuation_id: Optional[str], is_final_step: bool ) -> bool: """ Determine whether to embed file content based on workflow context. CORRECT LOGIC: - NEVER embed files when the CLI is getting the next step (next_step_required=True) - ONLY embed files when sending to external model (next_step_required=False) Args: step_number: Current step number continuation_id: Thread continuation ID (None for new conversations) is_final_step: Whether this is the final step (next_step_required == False) Returns: bool: True if files should be embedded, False if only referenced """ # RULE 1: Final steps (no more steps needed) - embed files for expert analysis if is_final_step: logger.debug("[WORKFLOW_FILES] Final step - will embed files for expert analysis") return True # RULE 2: Any intermediate step (more steps needed) - NEVER embed files # This includes: # - New conversations with next_step_required=True # - Steps with continuation_id and next_step_required=True logger.debug("[WORKFLOW_FILES] Intermediate step (more work needed) - will only reference files") return False def _embed_workflow_files(self, request: Any, arguments: dict[str, Any]) -> None: """ Embed full file content for final steps and expert analysis. Uses proper token budgeting like existing debug.py. """ # Use relevant_files as the standard field for workflow tools request_files = self.get_request_relevant_files(request) if not request_files: logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: No relevant_files to embed") return try: # Model context should be available from early validation, but might be deferred for tests current_model_context = self.get_current_model_context() if not current_model_context: # Try to resolve model context now (deferred from early validation) try: model_name, model_context = self._resolve_model_context(arguments, request) self._model_context = model_context self._current_model_name = model_name except Exception as e: logger.error(f"[WORKFLOW_FILES] {self.get_name()}: Failed to resolve model context: {e}") # Create fallback model context (preserves existing test behavior) from utils.model_context import ModelContext model_name = self.get_request_model_name(request) self._model_context = ModelContext(model_name) self._current_model_name = model_name # Use the same file preparation logic as BaseTool with token budgeting continuation_id = self.get_request_continuation_id(request) remaining_tokens = arguments.get("_remaining_tokens") file_content, processed_files = self._prepare_file_content_for_prompt( request_files, continuation_id, "Workflow files for analysis", remaining_budget=remaining_tokens, arguments=arguments, model_context=self._model_context, ) # Store for use in expert analysis self._embedded_file_content = file_content self._actually_processed_files = processed_files logger.info( f"[WORKFLOW_FILES] {self.get_name()}: Embedded {len(processed_files)} relevant_files for final analysis" ) except Exception as e: logger.error(f"[WORKFLOW_FILES] {self.get_name()}: Failed to embed files: {e}") # Continue without file embedding rather than failing self._embedded_file_content = "" self._actually_processed_files = [] def _reference_workflow_files(self, request: Any) -> None: """ Reference file names without embedding content for intermediate steps. Saves the CLI's context while still providing file awareness. """ # Workflow tools use relevant_files, not files request_files = self.get_request_relevant_files(request) logger.debug( f"[WORKFLOW_FILES] {self.get_name()}: _reference_workflow_files called with {len(request_files)} relevant_files" ) if not request_files: logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: No files to reference, skipping") return # Store file references for conversation context self._referenced_files = request_files # Create a simple reference note file_names = [os.path.basename(f) for f in request_files] reference_note = f"Files referenced in this step: {', '.join(file_names)}\n" self._file_reference_note = reference_note logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: Set _file_reference_note: {self._file_reference_note}") logger.info( f"[WORKFLOW_FILES] {self.get_name()}: Referenced {len(request_files)} files without embedding content" ) # ================================================================================ # Main Workflow Orchestration # ================================================================================ async def execute_workflow(self, arguments: dict[str, Any]) -> list[TextContent]: """ Main workflow orchestration following debug tool pattern. Comprehensive workflow implementation that handles all common patterns: 1. Request validation and step management 2. Continuation and backtracking support 3. Step data processing and consolidation 4. Tool-specific field mapping and customization 5. Completion logic with optional expert analysis 6. Generic "certain confidence" handling 7. Step guidance and required actions 8. Conversation memory integration """ from mcp.types import TextContent try: # Store arguments for access by helper methods self._current_arguments = arguments # Validate request using tool-specific model request = self.get_workflow_request_model()(**arguments) # Validate step field size (basic validation for workflow instructions) # If step is too large, user should use shorter instructions and put details in files step_content = request.step if step_content and len(step_content) > MCP_PROMPT_SIZE_LIMIT: from tools.models import ToolOutput error_output = ToolOutput( status="resend_prompt", content="Step instructions are too long. Please use shorter instructions and provide detailed context via file paths instead.", content_type="text", metadata={"prompt_size": len(step_content), "limit": MCP_PROMPT_SIZE_LIMIT}, ) raise ValueError(f"MCP_SIZE_CHECK:{error_output.model_dump_json()}") # Validate file paths for security (same as base tool) # Use try/except instead of hasattr as per coding standards try: path_error = self.validate_file_paths(request) if path_error: from tools.models import ToolOutput error_output = ToolOutput( status="error", content=path_error, content_type="text", ) logger.error("Path validation failed for %s: %s", self.get_name(), path_error) raise ToolExecutionError(error_output.model_dump_json()) except AttributeError: # validate_file_paths method not available - skip validation pass # Try to validate model availability early for production scenarios # For tests, defer model validation to later to allow mocks to work try: model_name, model_context = self._resolve_model_context(arguments, request) # Store for later use self._current_model_name = model_name self._model_context = model_context except ValueError as e: # Model resolution failed - in production this would be an error, # but for tests we defer to allow mocks to handle model resolution logger.debug(f"Early model validation failed, deferring to later: {e}") self._current_model_name = None self._model_context = None # Handle continuation continuation_id = request.continuation_id # Restore workflow state on continuation if continuation_id: from utils.conversation_memory import get_thread thread = get_thread(continuation_id) if thread and thread.turns: # Find the most recent assistant turn from this tool with workflow state for turn in reversed(thread.turns): if turn.role == "assistant" and turn.tool_name == self.get_name() and turn.model_metadata: state = turn.model_metadata if isinstance(state, dict) and "work_history" in state: self.work_history = state.get("work_history", []) self.initial_request = state.get("initial_request") # Rebuild consolidated findings from restored history self._reprocess_consolidated_findings() logger.debug( f"[{self.get_name()}] Restored workflow state with {len(self.work_history)} history items" ) break # State restored, exit loop # Adjust total steps if needed if request.step_number > request.total_steps: request.total_steps = request.step_number # Create thread for first step if not continuation_id and request.step_number == 1: clean_args = {k: v for k, v in arguments.items() if k not in ["_model_context", "_resolved_model_name"]} continuation_id = create_thread(self.get_name(), clean_args) self.initial_request = request.step # Allow tools to store initial description for expert analysis self.store_initial_issue(request.step) # Process work step - allow tools to customize field mapping step_data = self.prepare_step_data(request) # Store in history self.work_history.append(step_data) # Update consolidated findings self._update_consolidated_findings(step_data) # Handle file context appropriately based on workflow phase self._handle_workflow_file_context(request, arguments) # Build response with tool-specific customization response_data = self.build_base_response(request, continuation_id) # If work is complete, handle completion logic if not request.next_step_required: response_data = await self.handle_work_completion(response_data, request, arguments) else: # Force CLI to work before calling tool again response_data = self.handle_work_continuation(response_data, request) # Allow tools to customize the final response response_data = self.customize_workflow_response(response_data, request) # Add metadata (provider_used and model_used) to workflow response self._add_workflow_metadata(response_data, arguments) # Store in conversation memory if continuation_id: self.store_conversation_turn(continuation_id, response_data, request) return [TextContent(type="text", text=json.dumps(response_data, indent=2, ensure_ascii=False))] except ToolExecutionError: raise except Exception as e: if str(e).startswith("MCP_SIZE_CHECK:"): payload = str(e)[len("MCP_SIZE_CHECK:") :] raise ToolExecutionError(payload) logger.error(f"Error in {self.get_name()} work: {e}", exc_info=True) error_data = { "status": f"{self.get_name()}_failed", "error": str(e), "step_number": arguments.get("step_number", 0), } # Add metadata to error responses too self._add_workflow_metadata(error_data, arguments) raise ToolExecutionError(json.dumps(error_data, indent=2, ensure_ascii=False)) from e # Hook methods for tool customization def prepare_step_data(self, request) -> dict: """ Prepare step data from request. Tools can override to customize field mapping. """ step_data = { "step": request.step, "step_number": request.step_number, "findings": request.findings, "files_checked": self.get_request_files_checked(request), "relevant_files": self.get_request_relevant_files(request), "relevant_context": self.get_request_relevant_context(request), "issues_found": self.get_request_issues_found(request), "confidence": self.get_request_confidence(request), "hypothesis": self.get_request_hypothesis(request), "images": self.get_request_images(request), } return step_data def build_base_response(self, request, continuation_id: str = None) -> dict: """ Build the base response structure. Tools can override for custom response fields. """ response_data = { "status": f"{self.get_name()}_in_progress", "step_number": request.step_number, "total_steps": request.total_steps, "next_step_required": request.next_step_required, f"{self.get_name()}_status": { "files_checked": len(self.consolidated_findings.files_checked), "relevant_files": len(self.consolidated_findings.relevant_files), "relevant_context": len(self.consolidated_findings.relevant_context), "issues_found": len(self.consolidated_findings.issues_found), "images_collected": len(self.consolidated_findings.images), "current_confidence": self.get_request_confidence(request), }, } if continuation_id: response_data["continuation_id"] = continuation_id # Add file context information based on workflow phase embedded_content = self.get_embedded_file_content() reference_note = self.get_file_reference_note() processed_files = self.get_actually_processed_files() logger.debug( f"[WORKFLOW_FILES] {self.get_name()}: Building response - has embedded_content: {bool(embedded_content)}, has reference_note: {bool(reference_note)}" ) # Prioritize embedded content over references for final steps if embedded_content: # Final step - include embedded file information logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: Adding fully_embedded file context") response_data["file_context"] = { "type": "fully_embedded", "files_embedded": len(processed_files), "context_optimization": "Full file content embedded for expert analysis", } elif reference_note: # Intermediate step - include file reference note logger.debug(f"[WORKFLOW_FILES] {self.get_name()}: Adding reference_only file context") response_data["file_context"] = { "type": "reference_only", "note": reference_note, "context_optimization": "Files referenced but not embedded to preserve the context window", } return response_data def should_skip_expert_analysis(self, request, consolidated_findings) -> bool: """ Determine if expert analysis should be skipped due to high certainty. Default: False (always call expert analysis) Override in tools like debug to check for "certain" confidence. """ return False def handle_completion_without_expert_analysis(self, request, consolidated_findings) -> dict: """ Handle completion when skipping expert analysis. Tools can override this for custom high-confidence completion handling. Default implementation provides generic response. """ work_summary = self.prepare_work_summary() continuation_id = self.get_request_continuation_id(request) response_data = { "status": self.get_completion_status(), f"complete_{self.get_name()}": { "initial_request": self.get_initial_request(request.step), "steps_taken": len(consolidated_findings.findings), "files_examined": list(consolidated_findings.files_checked), "relevant_files": list(consolidated_findings.relevant_files), "relevant_context": list(consolidated_findings.relevant_context), "work_summary": work_summary, "final_analysis": self.get_final_analysis_from_request(request), "confidence_level": self.get_confidence_level(request), }, "next_steps": self.get_completion_message(), "skip_expert_analysis": True, "expert_analysis": { "status": self.get_skip_expert_analysis_status(), "reason": self.get_skip_reason(), }, } if continuation_id: response_data["continuation_id"] = continuation_id return response_data # ================================================================================ # Inheritance Hook Methods - Replace hasattr/getattr Anti-patterns # ================================================================================ def get_request_confidence(self, request: Any) -> str: """Get confidence from request. Override for custom confidence handling.""" try: return request.confidence or "low" except AttributeError: return "low" def get_request_relevant_context(self, request: Any) -> list[str]: """Get relevant context from request. Override for custom field mapping.""" try: return request.relevant_context or [] except AttributeError: return [] def get_request_issues_found(self, request: Any) -> list[str]: """Get issues found from request. Override for custom field mapping.""" try: return request.issues_found or [] except AttributeError: return [] def get_request_hypothesis(self, request: Any) -> Optional[str]: """Get hypothesis from request. Override for custom field mapping.""" try: return request.hypothesis except AttributeError: return None def get_request_images(self, request: Any) -> list[str]: """Get images from request. Override for custom field mapping.""" try: return request.images or [] except AttributeError: return [] # File Context Access Methods def get_embedded_file_content(self) -> str: """Get embedded file content. Returns empty string if not available.""" try: return self._embedded_file_content or "" except AttributeError: return "" def get_file_reference_note(self) -> str: """Get file reference note. Returns empty string if not available.""" try: return self._file_reference_note or "" except AttributeError: return "" def get_actually_processed_files(self) -> list[str]: """Get list of actually processed files. Returns empty list if not available.""" try: return self._actually_processed_files or [] except AttributeError: return [] def get_current_model_context(self): """Get current model context. Returns None if not available.""" try: return self._model_context except AttributeError: return None def get_request_model_name(self, request: Any) -> str: """Get model name from request. Override for custom model handling.""" try: return request.model or "flash" except AttributeError: return "flash" def get_request_continuation_id(self, request: Any) -> Optional[str]: """Get continuation ID from request. Override for custom continuation handling.""" try: return request.continuation_id except AttributeError: return None def get_request_next_step_required(self, request: Any) -> bool: """Get next step required from request. Override for custom step handling.""" try: return request.next_step_required except AttributeError: return True def get_request_step_number(self, request: Any) -> int: """Get step number from request. Override for custom step handling.""" try: return request.step_number or 1 except AttributeError: return 1 def get_request_relevant_files(self, request: Any) -> list[str]: """Get relevant files from request. Override for custom file handling.""" try: return request.relevant_files or [] except AttributeError: return [] def get_request_files_checked(self, request: Any) -> list[str]: """Get files checked from request. Override for custom file handling.""" try: return request.files_checked or [] except AttributeError: return [] def get_current_arguments(self) -> dict[str, Any]: """Get current arguments. Returns empty dict if not available.""" try: return self._current_arguments or {} except AttributeError: return {} def store_initial_issue(self, step_description: str): """Store initial issue description. Override for custom storage.""" # Default implementation - tools can override to store differently self.initial_issue = step_description def get_initial_request(self, fallback_step: str) -> str: """Get initial request description. Override for custom retrieval.""" try: return self.initial_request or fallback_step except AttributeError: return fallback_step # Default implementations for inheritance hooks def prepare_work_summary(self) -> str: """Prepare work summary. Override for custom implementation.""" return f"Completed {len(self.consolidated_findings.findings)} work steps" def get_completion_status(self) -> str: """Get completion status. Override for tool-specific status.""" return "high_confidence_completion" def get_final_analysis_from_request(self, request): """Extract final analysis from request. Override for tool-specific fields.""" return self.get_request_hypothesis(request) def get_confidence_level(self, request) -> str: """Get confidence level. Override for tool-specific confidence handling.""" return self.get_request_confidence(request) or "high" def get_completion_message(self) -> str: """Get completion message. Override for tool-specific messaging.""" return ( f"{self.get_name().capitalize()} complete with high confidence. Present results " "and proceed with implementation without requiring further consultation." ) def get_skip_reason(self) -> str: """Get reason for skipping expert analysis. Override for tool-specific reasons.""" return f"{self.get_name()} completed with sufficient confidence" def get_skip_expert_analysis_status(self) -> str: """Get status for skipped expert analysis. Override for tool-specific status.""" return "skipped_by_tool_design" def get_completion_next_steps_message(self, expert_analysis_used: bool = False) -> str: """ Get the message to show when work is complete. Tools can override for custom messaging. Args: expert_analysis_used: True if expert analysis was successfully executed """ base_message = ( f"{self.get_name().upper()} IS COMPLETE. You MUST now summarize and present ALL key findings, confirmed " "hypotheses, and exact recommended solutions. Clearly identify the most likely root cause and " "provide concrete, actionable implementation guidance. Highlight affected code paths and display " "reasoning that led to this conclusion—make it easy for a developer to understand exactly where " "the problem lies." ) # Add expert analysis guidance only when expert analysis was actually used if expert_analysis_used: expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: return f"{base_message}\n\n{expert_guidance}" return base_message def get_expert_analysis_guidance(self) -> str: """ Get additional guidance for handling expert analysis results. Subclasses can override this to provide specific instructions about how to validate and use expert analysis findings. Returns empty string by default. When expert analysis is called, this guidance will be: 1. Appended to the completion next steps message 2. Added as "important_considerations" field in the response data Example implementation: ```python def get_expert_analysis_guidance(self) -> str: return ( "IMPORTANT: Expert analysis provided above. You MUST validate " "the expert findings rather than accepting them blindly. " "Cross-reference with your own investigation and ensure " "recommendations align with the codebase context." ) ``` Returns: Additional guidance text or empty string if no guidance needed """ return "" def customize_workflow_response(self, response_data: dict, request) -> dict: """ Allow tools to customize the workflow response before returning. Tools can override this to add tool-specific fields, modify status names, customize field mapping, etc. Default implementation returns unchanged. """ # Ensure file context information is preserved in all response paths if not response_data.get("file_context"): embedded_content = self.get_embedded_file_content() reference_note = self.get_file_reference_note() processed_files = self.get_actually_processed_files() # Prioritize embedded content over references for final steps if embedded_content: response_data["file_context"] = { "type": "fully_embedded", "files_embedded": len(processed_files), "context_optimization": "Full file content embedded for expert analysis", } elif reference_note: response_data["file_context"] = { "type": "reference_only", "note": reference_note, "context_optimization": "Files referenced but not embedded to preserve the context window", } return response_data def store_conversation_turn(self, continuation_id: str, response_data: dict, request): """ Store the conversation turn. Tools can override for custom memory storage. """ # CRITICAL: Extract clean content for conversation history (exclude internal workflow metadata) clean_content = self._extract_clean_workflow_content_for_history(response_data) # Serialize workflow state for persistence across stateless tool calls workflow_state = {"work_history": self.work_history, "initial_request": getattr(self, "initial_request", None)} add_turn( thread_id=continuation_id, role="assistant", content=clean_content, # Use cleaned content instead of full response_data tool_name=self.get_name(), files=self.get_request_relevant_files(request), images=self.get_request_images(request), model_metadata=workflow_state, # Persist the state ) def _add_workflow_metadata(self, response_data: dict, arguments: dict[str, Any]) -> None: """ Add metadata (provider_used and model_used) to workflow response. This ensures workflow tools have the same metadata as regular tools, making it consistent across all tool types for tracking which provider and model were used for the response. Args: response_data: The response data dictionary to modify arguments: The original arguments containing model context """ try: # Get model information from arguments (set by server.py) resolved_model_name = arguments.get("_resolved_model_name") model_context = arguments.get("_model_context") if resolved_model_name and model_context: # Extract provider information from model context provider = model_context.provider provider_name = provider.get_provider_type().value if provider else "unknown" # Create metadata dictionary metadata = { "tool_name": self.get_name(), "model_used": resolved_model_name, "provider_used": provider_name, } # Preserve existing metadata and add workflow metadata if "metadata" not in response_data: response_data["metadata"] = {} response_data["metadata"].update(metadata) logger.debug( f"[WORKFLOW_METADATA] {self.get_name()}: Added metadata - " f"model: {resolved_model_name}, provider: {provider_name}" ) else: # Fallback - try to get model info from request request = self.get_workflow_request_model()(**arguments) model_name = self.get_request_model_name(request) # Basic metadata without provider info metadata = { "tool_name": self.get_name(), "model_used": model_name, "provider_used": "unknown", } # Preserve existing metadata and add workflow metadata if "metadata" not in response_data: response_data["metadata"] = {} response_data["metadata"].update(metadata) logger.debug( f"[WORKFLOW_METADATA] {self.get_name()}: Added fallback metadata - " f"model: {model_name}, provider: unknown" ) except Exception as e: # Don't fail the workflow if metadata addition fails logger.warning(f"[WORKFLOW_METADATA] {self.get_name()}: Failed to add metadata: {e}") # Still add basic metadata with tool name response_data["metadata"] = {"tool_name": self.get_name()} def _extract_clean_workflow_content_for_history(self, response_data: dict) -> str: """ Extract clean content from workflow response suitable for conversation history. This method removes internal workflow metadata, continuation offers, and status information that should not appear when the conversation is reconstructed for expert models or other tools. Args: response_data: The full workflow response data Returns: str: Clean content suitable for conversation history storage """ # Create a clean copy with only essential content for conversation history clean_data = {} # Include core content if present if "content" in response_data: clean_data["content"] = response_data["content"] # Include expert analysis if present (but clean it) if "expert_analysis" in response_data: expert_analysis = response_data["expert_analysis"] if isinstance(expert_analysis, dict): # Only include the actual analysis content, not metadata clean_expert = {} if "raw_analysis" in expert_analysis: clean_expert["analysis"] = expert_analysis["raw_analysis"] elif "content" in expert_analysis: clean_expert["analysis"] = expert_analysis["content"] if clean_expert: clean_data["expert_analysis"] = clean_expert # Include findings/issues if present (core workflow output) if "complete_analysis" in response_data: complete_analysis = response_data["complete_analysis"] if isinstance(complete_analysis, dict): clean_complete = {} # Include essential analysis data without internal metadata for key in ["findings", "issues_found", "relevant_context", "insights"]: if key in complete_analysis: clean_complete[key] = complete_analysis[key] if clean_complete: clean_data["analysis_summary"] = clean_complete # Include step information for context but remove internal workflow metadata if "step_number" in response_data: clean_data["step_info"] = { "step": response_data.get("step", ""), "step_number": response_data.get("step_number", 1), "total_steps": response_data.get("total_steps", 1), } # Exclude problematic fields that should never appear in conversation history: # - continuation_id (confuses LLMs with old IDs) # - status (internal workflow state) # - next_step_required (internal control flow) # - analysis_status (internal tracking) # - file_context (internal optimization info) # - required_actions (internal workflow instructions) return json.dumps(clean_data, indent=2, ensure_ascii=False) # Core workflow logic methods async def handle_work_completion(self, response_data: dict, request, arguments: dict) -> dict: """ Handle work completion logic - expert analysis decision and response building. """ response_data[f"{self.get_name()}_complete"] = True # Check if tool wants to skip expert analysis due to high certainty if self.should_skip_expert_analysis(request, self.consolidated_findings): # Handle completion without expert analysis completion_response = self.handle_completion_without_expert_analysis(request, self.consolidated_findings) response_data.update(completion_response) elif self.requires_expert_analysis() and self.should_call_expert_analysis(self.consolidated_findings, request): # Standard expert analysis path response_data["status"] = "calling_expert_analysis" # Call expert analysis expert_analysis = await self._call_expert_analysis(arguments, request) response_data["expert_analysis"] = expert_analysis # Handle special expert analysis statuses if isinstance(expert_analysis, dict) and expert_analysis.get("status") in [ "files_required_to_continue", "investigation_paused", "refactoring_paused", ]: # Promote the special status to the main response special_status = expert_analysis["status"] response_data["status"] = special_status response_data["content"] = expert_analysis.get( "raw_analysis", json.dumps(expert_analysis, ensure_ascii=False) ) del response_data["expert_analysis"] # Update next steps for special status if special_status == "files_required_to_continue": response_data["next_steps"] = "Provide the requested files and continue the analysis." else: response_data["next_steps"] = expert_analysis.get( "next_steps", "Continue based on expert analysis." ) elif isinstance(expert_analysis, dict) and expert_analysis.get("status") == "analysis_error": # Expert analysis failed - promote error status response_data["status"] = "error" response_data["content"] = expert_analysis.get("error", "Expert analysis failed") response_data["content_type"] = "text" del response_data["expert_analysis"] else: # Expert analysis was successfully executed - include expert guidance response_data["next_steps"] = self.get_completion_next_steps_message(expert_analysis_used=True) # Add expert analysis guidance as important considerations expert_guidance = self.get_expert_analysis_guidance() if expert_guidance: response_data["important_considerations"] = expert_guidance # Prepare complete work summary work_summary = self._prepare_work_summary() response_data[f"complete_{self.get_name()}"] = { "initial_request": self.get_initial_request(request.step), "steps_taken": len(self.work_history), "files_examined": list(self.consolidated_findings.files_checked), "relevant_files": list(self.consolidated_findings.relevant_files), "relevant_context": list(self.consolidated_findings.relevant_context), "issues_found": self.consolidated_findings.issues_found, "work_summary": work_summary, } else: # Tool doesn't require expert analysis or local work was sufficient if not self.requires_expert_analysis(): # Tool is self-contained (like planner) response_data["status"] = f"{self.get_name()}_complete" response_data["next_steps"] = ( f"{self.get_name().capitalize()} work complete. Present results to the user." ) else: # Local work was sufficient for tools that support expert analysis response_data["status"] = "local_work_complete" response_data["next_steps"] = ( f"Local {self.get_name()} complete with sufficient confidence. Present findings " "and recommendations to the user based on the work results." ) return response_data def handle_work_continuation(self, response_data: dict, request) -> dict: """ Handle work continuation - force pause and provide guidance. """ response_data["status"] = f"pause_for_{self.get_name()}" response_data[f"{self.get_name()}_required"] = True # Get tool-specific required actions required_actions = self.get_required_actions( request.step_number, self.get_request_confidence(request), request.findings, request.total_steps, request ) response_data["required_actions"] = required_actions # Generate step guidance response_data["next_steps"] = self.get_step_guidance_message(request) return response_data def _update_consolidated_findings(self, step_data: dict): """Update consolidated findings with new step data""" self.consolidated_findings.files_checked.update(step_data.get("files_checked", [])) self.consolidated_findings.relevant_files.update(step_data.get("relevant_files", [])) self.consolidated_findings.relevant_context.update(step_data.get("relevant_context", [])) self.consolidated_findings.findings.append(f"Step {step_data['step_number']}: {step_data['findings']}") if step_data.get("hypothesis"): self.consolidated_findings.hypotheses.append( { "step": step_data["step_number"], "hypothesis": step_data["hypothesis"], "confidence": step_data["confidence"], } ) if step_data.get("issues_found"): self.consolidated_findings.issues_found.extend(step_data["issues_found"]) if step_data.get("images"): self.consolidated_findings.images.extend(step_data["images"]) # Update confidence to latest value from this step if step_data.get("confidence"): self.consolidated_findings.confidence = step_data["confidence"] def _reprocess_consolidated_findings(self): """Reprocess consolidated findings after backtracking""" self.consolidated_findings = ConsolidatedFindings() for step in self.work_history: self._update_consolidated_findings(step) def _prepare_work_summary(self) -> str: """Prepare a comprehensive summary of the work""" summary_parts = [ f"=== {self.get_name().upper()} WORK SUMMARY ===", f"Total steps: {len(self.work_history)}", f"Files examined: {len(self.consolidated_findings.files_checked)}", f"Relevant files identified: {len(self.consolidated_findings.relevant_files)}", f"Methods/functions involved: {len(self.consolidated_findings.relevant_context)}", f"Issues found: {len(self.consolidated_findings.issues_found)}", "", "=== WORK PROGRESSION ===", ] for finding in self.consolidated_findings.findings: summary_parts.append(finding) if self.consolidated_findings.hypotheses: summary_parts.extend( [ "", "=== HYPOTHESIS EVOLUTION ===", ] ) for hyp in self.consolidated_findings.hypotheses: summary_parts.append(f"Step {hyp['step']} ({hyp['confidence']} confidence): {hyp['hypothesis']}") if self.consolidated_findings.issues_found: summary_parts.extend( [ "", "=== ISSUES IDENTIFIED ===", ] ) for issue in self.consolidated_findings.issues_found: severity = issue.get("severity", "unknown") description = issue.get("description", "No description") summary_parts.append(f"[{severity.upper()}] {description}") return "\n".join(summary_parts) async def _call_expert_analysis(self, arguments: dict, request) -> dict: """Call external model for expert analysis""" try: # Model context should be resolved from early validation, but handle fallback for tests if not self._model_context: # Try to resolve model context for expert analysis (deferred from early validation) try: model_name, model_context = self._resolve_model_context(arguments, request) self._model_context = model_context self._current_model_name = model_name except Exception as e: logger.error(f"Failed to resolve model context for expert analysis: {e}") # Use request model as fallback (preserves existing test behavior) model_name = self.get_request_model_name(request) from utils.model_context import ModelContext model_context = ModelContext(model_name) self._model_context = model_context self._current_model_name = model_name else: model_name = self._current_model_name provider = self._model_context.provider # Prepare expert analysis context expert_context = self.prepare_expert_analysis_context(self.consolidated_findings) # Check if tool wants to include files in prompt if self.should_include_files_in_expert_prompt(): file_content = self._prepare_files_for_expert_analysis() if file_content: expert_context = self._add_files_to_expert_context(expert_context, file_content) # Get system prompt for this tool with localization support base_system_prompt = self.get_system_prompt() capability_augmented_prompt = self._augment_system_prompt_with_capabilities( base_system_prompt, getattr(self._model_context, "capabilities", None) ) language_instruction = self.get_language_instruction() system_prompt = language_instruction + capability_augmented_prompt # Check if tool wants system prompt embedded in main prompt if self.should_embed_system_prompt(): prompt = f"{system_prompt}\n\n{expert_context}\n\n{self.get_expert_analysis_instruction()}" system_prompt = "" # Clear it since we embedded it else: prompt = expert_context # Validate temperature against model constraints validated_temperature, temp_warnings = self.get_validated_temperature(request, self._model_context) # Log any temperature corrections for warning in temp_warnings: logger.warning(warning) # Generate AI response - use request parameters if available model_response = provider.generate_content( prompt=prompt, model_name=model_name, system_prompt=system_prompt, temperature=validated_temperature, thinking_mode=self.get_request_thinking_mode(request), images=list(set(self.consolidated_findings.images)) if self.consolidated_findings.images else None, ) if model_response.content: content = model_response.content.strip() # Try to extract JSON from markdown code blocks if present if "```json" in content or "```" in content: json_match = re.search(r"```(?:json)?\s*(.*?)\s*```", content, re.DOTALL) if json_match: content = json_match.group(1).strip() try: # Try to parse as JSON analysis_result = json.loads(content) return analysis_result except json.JSONDecodeError as e: # Log the parse error with more details but don't fail logger.info( f"[{self.get_name()}] Expert analysis returned non-JSON response (this is OK for smaller models). " f"Parse error: {str(e)}. Response length: {len(model_response.content)} chars." ) logger.debug(f"First 500 chars of response: {model_response.content[:500]!r}") # Still return the analysis as plain text - this is valid return { "status": "analysis_complete", "raw_analysis": model_response.content, "format": "text", # Indicate it's plain text, not an error "note": "Analysis provided in plain text format", } else: return {"error": "No response from model", "status": "empty_response"} except Exception as e: logger.error(f"Error calling expert analysis: {e}", exc_info=True) return {"error": str(e), "status": "analysis_error"} def _process_work_step(self, step_data: dict): """ Process a single work step and update internal state. This method is useful for testing and manual step processing. It adds the step to work history and updates consolidated findings. Args: step_data: Dictionary containing step information including: step, step_number, findings, files_checked, etc. """ # Store in history self.work_history.append(step_data) # Update consolidated findings self._update_consolidated_findings(step_data) # Common execute method for workflow-based tools async def execute(self, arguments: dict[str, Any]) -> list[TextContent]: """ Common execute logic for workflow-based tools. This method provides common validation and delegates to execute_workflow. Tools that need custom execute logic can override this method. """ try: # Common validation if not arguments: error_data = {"status": "error", "content": "No arguments provided"} # Add basic metadata even for validation errors error_data["metadata"] = {"tool_name": self.get_name()} raise ToolExecutionError(json.dumps(error_data, ensure_ascii=False)) # Delegate to execute_workflow return await self.execute_workflow(arguments) except ToolExecutionError: raise except Exception as e: logger.error(f"Error in {self.get_name()} tool execution: {e}", exc_info=True) error_data = { "status": "error", "content": f"Error in {self.get_name()}: {str(e)}", } # Add metadata to error responses self._add_workflow_metadata(error_data, arguments) raise ToolExecutionError(json.dumps(error_data, ensure_ascii=False)) from e # Default implementations for methods that workflow-based tools typically don't need async def prepare_prompt(self, request) -> str: """ Base implementation for workflow tools - compatible with BaseTool signature. Workflow tools typically don't need to return a prompt since they handle their own prompt preparation internally through the workflow execution. Args: request: The validated request object Returns: Empty string since workflow tools manage prompts internally """ # Workflow tools handle their prompts internally during workflow execution return "" def format_response(self, response: str, request, model_info=None): """ Workflow tools handle their own response formatting. The BaseWorkflowMixin formats responses internally. """ return response ================================================ FILE: utils/__init__.py ================================================ """ Utility functions for PAL MCP Server """ from .file_types import CODE_EXTENSIONS, FILE_CATEGORIES, PROGRAMMING_EXTENSIONS, TEXT_EXTENSIONS from .file_utils import expand_paths, read_file_content, read_files from .security_config import EXCLUDED_DIRS from .token_utils import check_token_limit, estimate_tokens __all__ = [ "read_files", "read_file_content", "expand_paths", "CODE_EXTENSIONS", "PROGRAMMING_EXTENSIONS", "TEXT_EXTENSIONS", "FILE_CATEGORIES", "EXCLUDED_DIRS", "estimate_tokens", "check_token_limit", ] ================================================ FILE: utils/client_info.py ================================================ """ Client Information Utility for MCP Server This module provides utilities to extract and format client information from the MCP protocol's clientInfo sent during initialization. It also provides friendly name mapping and caching for consistent client identification across the application. """ import logging from typing import Any, Optional logger = logging.getLogger(__name__) # Global cache for client information _client_info_cache: Optional[dict[str, Any]] = None # Mapping of known client names to friendly names # This is case-insensitive and checks if the key is contained in the client name CLIENT_NAME_MAPPINGS = { # Claude variants "claude-ai": "Claude", "claude": "Claude", "claude-desktop": "Claude", "claude-code": "Claude", "anthropic": "Claude", # Gemini variants "gemini-cli-mcp-client": "Gemini", "gemini-cli": "Gemini", "gemini": "Gemini", "google": "Gemini", # Other known clients "cursor": "Cursor", "vscode": "VS Code", "codeium": "Codeium", "copilot": "GitHub Copilot", # Generic MCP clients "mcp-client": "MCP Client", "test-client": "Test Client", } # Default friendly name when no match is found DEFAULT_FRIENDLY_NAME = "Claude" def get_friendly_name(client_name: str) -> str: """ Map a client name to a friendly name. Args: client_name: The raw client name from clientInfo Returns: A friendly name for display (e.g., "Claude", "Gemini") """ if not client_name: return DEFAULT_FRIENDLY_NAME # Convert to lowercase for case-insensitive matching client_name_lower = client_name.lower() # Check each mapping - using 'in' to handle partial matches for key, friendly_name in CLIENT_NAME_MAPPINGS.items(): if key.lower() in client_name_lower: return friendly_name # If no match found, return the default return DEFAULT_FRIENDLY_NAME def get_cached_client_info() -> Optional[dict[str, Any]]: """ Get cached client information if available. Returns: Cached client info dictionary or None """ global _client_info_cache return _client_info_cache def get_client_info_from_context(server: Any) -> Optional[dict[str, Any]]: """ Extract client information from the MCP server's request context. The MCP protocol sends clientInfo during initialization containing: - name: The client application name (e.g., "Claude Code", "Claude Desktop") - version: The client version string This function also adds a friendly_name field and caches the result. Args: server: The MCP server instance Returns: Dictionary with client info or None if not available: { "name": "claude-ai", "version": "1.0.0", "friendly_name": "Claude" } """ global _client_info_cache # Return cached info if available if _client_info_cache is not None: return _client_info_cache try: # Try to access the request context and session if not server: return None # Check if server has request_context property request_context = None try: request_context = server.request_context except AttributeError: logger.debug("Server does not have request_context property") return None if not request_context: logger.debug("Request context is None") return None # Try to access session from request context session = None try: session = request_context.session except AttributeError: logger.debug("Request context does not have session property") return None if not session: logger.debug("Session is None") return None # Try to access client params from session client_params = None try: # The clientInfo is stored in _client_params.clientInfo client_params = session._client_params except AttributeError: logger.debug("Session does not have _client_params property") return None if not client_params: logger.debug("Client params is None") return None # Try to extract clientInfo client_info = None try: client_info = client_params.clientInfo except AttributeError: logger.debug("Client params does not have clientInfo property") return None if not client_info: logger.debug("Client info is None") return None # Extract name and version result = {} try: result["name"] = client_info.name except AttributeError: logger.debug("Client info does not have name property") try: result["version"] = client_info.version except AttributeError: logger.debug("Client info does not have version property") if not result: return None # Add friendly name raw_name = result.get("name", "") result["friendly_name"] = get_friendly_name(raw_name) # Cache the result _client_info_cache = result logger.debug(f"Cached client info: {result}") return result except Exception as e: logger.debug(f"Error extracting client info: {e}") return None def format_client_info(client_info: Optional[dict[str, Any]], use_friendly_name: bool = True) -> str: """ Format client information for display. Args: client_info: Dictionary with client info or None use_friendly_name: If True, use the friendly name instead of raw name Returns: Formatted string like "Claude v1.0.0" or "Claude" """ if not client_info: return DEFAULT_FRIENDLY_NAME if use_friendly_name: name = client_info.get("friendly_name", client_info.get("name", DEFAULT_FRIENDLY_NAME)) else: name = client_info.get("name", "Unknown") version = client_info.get("version", "") if version and not use_friendly_name: return f"{name} v{version}" else: # For friendly names, we just return the name without version return name def get_client_friendly_name() -> str: """ Get the cached client's friendly name. This is a convenience function that returns just the friendly name from the cached client info, defaulting to "Claude" if not available. Returns: The friendly name (e.g., "Claude", "Gemini") """ cached_info = get_cached_client_info() if cached_info: return cached_info.get("friendly_name", DEFAULT_FRIENDLY_NAME) return DEFAULT_FRIENDLY_NAME def log_client_info(server: Any, logger_instance: Optional[logging.Logger] = None) -> None: """ Log client information extracted from the server. Args: server: The MCP server instance logger_instance: Optional logger to use (defaults to module logger) """ log = logger_instance or logger client_info = get_client_info_from_context(server) if client_info: # Log with both raw and friendly names for debugging raw_name = client_info.get("name", "Unknown") friendly_name = client_info.get("friendly_name", DEFAULT_FRIENDLY_NAME) version = client_info.get("version", "") if raw_name != friendly_name: log.info(f"MCP Client Connected: {friendly_name} (raw: {raw_name} v{version})") else: log.info(f"MCP Client Connected: {friendly_name} v{version}") # Log to activity logger as well try: activity_logger = logging.getLogger("mcp_activity") activity_logger.info(f"CLIENT_IDENTIFIED: {friendly_name} (name={raw_name}, version={version})") except Exception: pass else: log.debug("Could not extract client info from MCP protocol") # Example usage in tools: # # from utils.client_info import get_client_friendly_name, get_cached_client_info # # # In a tool's execute method: # def execute(self, arguments: dict[str, Any]) -> list[TextContent]: # # Get the friendly name of the connected client # client_name = get_client_friendly_name() # Returns "Claude" or "Gemini" etc. # # # Or get full cached info if needed # client_info = get_cached_client_info() # if client_info: # raw_name = client_info['name'] # e.g., "claude-ai" # version = client_info['version'] # e.g., "1.0.0" # friendly = client_info['friendly_name'] # e.g., "Claude" # # # Customize response based on client # if client_name == "Claude": # response = f"Hello from PAL MCP Server to {client_name}!" # elif client_name == "Gemini": # response = f"Greetings {client_name}, welcome to PAL MCP Server!" # else: # response = f"Welcome {client_name}!" ================================================ FILE: utils/conversation_memory.py ================================================ """ Conversation Memory for AI-to-AI Multi-turn Discussions This module provides conversation persistence and context reconstruction for stateless MCP (Model Context Protocol) environments. It enables multi-turn conversations between the agent and downstream models by storing conversation state in memory across independent request cycles. CRITICAL ARCHITECTURAL REQUIREMENT: This conversation memory system is designed for PERSISTENT MCP SERVER PROCESSES. It uses in-memory storage that persists only within a single Python process. ⚠️ IMPORTANT: This system will NOT work correctly if MCP tool calls are made as separate subprocess invocations (each subprocess starts with empty memory). WORKING SCENARIO: Claude Desktop with persistent MCP server process FAILING SCENARIO: Simulator tests calling server.py as individual subprocesses Root cause of test failures: Each subprocess call loses the conversation state from previous calls because memory is process-specific, not shared across subprocess boundaries. ARCHITECTURE OVERVIEW: The MCP protocol is inherently stateless - each tool request is independent with no memory of previous interactions. This module bridges that gap by: 1. Creating persistent conversation threads with unique UUIDs 2. Storing complete conversation context (turns, files, metadata) in memory 3. Reconstructing conversation history when tools are called with continuation_id 4. Supporting cross-tool continuation - seamlessly switch between different tools while maintaining full conversation context and file references CROSS-TOOL CONTINUATION: A conversation started with one tool (e.g., 'analyze') can be continued with any other tool (e.g., 'codereview', 'debug', 'chat') using the same continuation_id. The second tool will have access to: - All previous conversation turns and responses - File context from previous tools (preserved in conversation history) - Original thread metadata and timing information - Accumulated knowledge from the entire conversation Key Features: - UUID-based conversation thread identification with security validation - Turn-by-turn conversation history storage with tool attribution - Cross-tool continuation support - switch tools while preserving context - File context preservation - files shared in earlier turns remain accessible - NEWEST-FIRST FILE PRIORITIZATION - when the same file appears in multiple turns, references from newer turns take precedence over older ones. This ensures the most recent file context is preserved when token limits require exclusions. - Automatic turn limiting (20 turns max) to prevent runaway conversations - Context reconstruction for stateless request continuity - In-memory persistence with automatic expiration (3 hour TTL) - Thread-safe operations for concurrent access - Graceful degradation when storage is unavailable DUAL PRIORITIZATION STRATEGY (Files & Conversations): The conversation memory system implements sophisticated prioritization for both files and conversation turns, using a consistent "newest-first" approach during collection but presenting information in the optimal format for LLM consumption: FILE PRIORITIZATION (Newest-First Throughout): 1. When collecting files across conversation turns, the system walks BACKWARDS through turns (newest to oldest) and builds a unique file list 2. If the same file path appears in multiple turns, only the reference from the NEWEST turn is kept in the final list 3. This "newest-first" ordering is preserved throughout the entire pipeline: - get_conversation_file_list() establishes the order - build_conversation_history() maintains it during token budgeting - When token limits are hit, OLDER files are excluded first 4. This strategy works across conversation chains - files from newer turns in ANY thread take precedence over files from older turns in ANY thread CONVERSATION TURN PRIORITIZATION (Newest-First Collection, Chronological Presentation): 1. COLLECTION PHASE: Processes turns newest-to-oldest to prioritize recent context - When token budget is tight, OLDER turns are excluded first - Ensures most contextually relevant recent exchanges are preserved 2. PRESENTATION PHASE: Reverses collected turns to chronological order (oldest-first) - LLM sees natural conversation flow: "Turn 1 → Turn 2 → Turn 3..." - Maintains proper sequential understanding while preserving recency prioritization This dual approach ensures optimal context preservation (newest-first) with natural conversation flow (chronological) for maximum LLM comprehension and relevance. USAGE EXAMPLE: 1. Tool A creates thread: create_thread("analyze", request_data) → returns UUID 2. Tool A adds response: add_turn(UUID, "assistant", response, files=[...], tool_name="analyze") 3. Tool B continues thread: get_thread(UUID) → retrieves full context 4. Tool B sees conversation history via build_conversation_history() 5. Tool B adds its response: add_turn(UUID, "assistant", response, tool_name="codereview") DUAL STRATEGY EXAMPLE: Conversation has 5 turns, token budget allows only 3 turns: Collection Phase (Newest-First Priority): - Evaluates: Turn 5 → Turn 4 → Turn 3 → Turn 2 → Turn 1 - Includes: Turn 5, Turn 4, Turn 3 (newest 3 fit in budget) - Excludes: Turn 2, Turn 1 (oldest, dropped due to token limits) Presentation Phase (Chronological Order): - LLM sees: "--- Turn 3 (Agent) ---", "--- Turn 4 (Model) ---", "--- Turn 5 (Agent) ---" - Natural conversation flow maintained despite prioritizing recent context This enables true AI-to-AI collaboration across the entire tool ecosystem with optimal context preservation and natural conversation understanding. """ import logging import os import uuid from datetime import datetime, timezone from typing import Any, Optional from pydantic import BaseModel from utils.env import get_env logger = logging.getLogger(__name__) # Configuration constants # Get max conversation turns from environment, default to 20 turns (10 exchanges) try: max_turns_raw = (get_env("MAX_CONVERSATION_TURNS", "50") or "50").strip() MAX_CONVERSATION_TURNS = int(max_turns_raw) if MAX_CONVERSATION_TURNS <= 0: logger.warning(f"Invalid MAX_CONVERSATION_TURNS value ({MAX_CONVERSATION_TURNS}), using default of 50 turns") MAX_CONVERSATION_TURNS = 50 except ValueError: logger.warning( f"Invalid MAX_CONVERSATION_TURNS value ('{get_env('MAX_CONVERSATION_TURNS')}'), using default of 50 turns" ) MAX_CONVERSATION_TURNS = 50 # Get conversation timeout from environment (in hours), default to 3 hours try: timeout_raw = (get_env("CONVERSATION_TIMEOUT_HOURS", "3") or "3").strip() CONVERSATION_TIMEOUT_HOURS = int(timeout_raw) if CONVERSATION_TIMEOUT_HOURS <= 0: logger.warning( f"Invalid CONVERSATION_TIMEOUT_HOURS value ({CONVERSATION_TIMEOUT_HOURS}), using default of 3 hours" ) CONVERSATION_TIMEOUT_HOURS = 3 except ValueError: logger.warning( f"Invalid CONVERSATION_TIMEOUT_HOURS value ('{get_env('CONVERSATION_TIMEOUT_HOURS')}'), using default of 3 hours" ) CONVERSATION_TIMEOUT_HOURS = 3 CONVERSATION_TIMEOUT_SECONDS = CONVERSATION_TIMEOUT_HOURS * 3600 class ConversationTurn(BaseModel): """ Single turn in a conversation Represents one exchange in the AI-to-AI conversation, tracking both the content and metadata needed for cross-tool continuation. Attributes: role: "user" (Agent request) or "assistant" (model response) content: The actual message content/response timestamp: ISO timestamp when this turn was created files: List of file paths referenced in this specific turn images: List of image paths referenced in this specific turn tool_name: Which tool generated this turn (for cross-tool tracking) model_provider: Provider used (e.g., "google", "openai") model_name: Specific model used (e.g., "gemini-2.5-flash", "o3-mini") model_metadata: Additional model-specific metadata (e.g., thinking mode, token usage) """ role: str # "user" or "assistant" content: str timestamp: str files: Optional[list[str]] = None # Files referenced in this turn images: Optional[list[str]] = None # Images referenced in this turn tool_name: Optional[str] = None # Tool used for this turn model_provider: Optional[str] = None # Model provider (google, openai, etc) model_name: Optional[str] = None # Specific model used model_metadata: Optional[dict[str, Any]] = None # Additional model info class ThreadContext(BaseModel): """ Complete conversation context for a thread Contains all information needed to reconstruct a conversation state across different tools and request cycles. This is the core data structure that enables cross-tool continuation. Attributes: thread_id: UUID identifying this conversation thread parent_thread_id: UUID of parent thread (for conversation chains) created_at: ISO timestamp when thread was created last_updated_at: ISO timestamp of last modification tool_name: Name of the tool that initiated this thread turns: List of all conversation turns in chronological order initial_context: Original request data that started the conversation """ thread_id: str parent_thread_id: Optional[str] = None # Parent thread for conversation chains created_at: str last_updated_at: str tool_name: str # Tool that created this thread (preserved for attribution) turns: list[ConversationTurn] initial_context: dict[str, Any] # Original request parameters def get_storage(): """ Get in-memory storage backend for conversation persistence. Returns: InMemoryStorage: Thread-safe in-memory storage backend """ from .storage_backend import get_storage_backend return get_storage_backend() def create_thread(tool_name: str, initial_request: dict[str, Any], parent_thread_id: Optional[str] = None) -> str: """ Create new conversation thread and return thread ID Initializes a new conversation thread for AI-to-AI discussions. This is called when a tool wants to enable follow-up conversations or when Claude explicitly starts a multi-turn interaction. Args: tool_name: Name of the tool creating this thread (e.g., "analyze", "chat") initial_request: Original request parameters (will be filtered for serialization) parent_thread_id: Optional parent thread ID for conversation chains Returns: str: UUID thread identifier that can be used for continuation Note: - Thread expires after the configured timeout (default: 3 hours) - Non-serializable parameters are filtered out automatically - Thread can be continued by any tool using the returned UUID - Parent thread creates a chain for conversation history traversal """ thread_id = str(uuid.uuid4()) now = datetime.now(timezone.utc).isoformat() # Filter out non-serializable parameters to avoid JSON encoding issues filtered_context = { k: v for k, v in initial_request.items() if k not in ["temperature", "thinking_mode", "model", "continuation_id"] } context = ThreadContext( thread_id=thread_id, parent_thread_id=parent_thread_id, # Link to parent for conversation chains created_at=now, last_updated_at=now, tool_name=tool_name, # Track which tool initiated this conversation turns=[], # Empty initially, turns added via add_turn() initial_context=filtered_context, ) # Store in memory with configurable TTL to prevent indefinite accumulation storage = get_storage() key = f"thread:{thread_id}" storage.setex(key, CONVERSATION_TIMEOUT_SECONDS, context.model_dump_json()) logger.debug(f"[THREAD] Created new thread {thread_id} with parent {parent_thread_id}") return thread_id def get_thread(thread_id: str) -> Optional[ThreadContext]: """ Retrieve thread context from in-memory storage Fetches complete conversation context for cross-tool continuation. This is the core function that enables tools to access conversation history from previous interactions. Args: thread_id: UUID of the conversation thread Returns: ThreadContext: Complete conversation context if found None: If thread doesn't exist, expired, or invalid UUID Security: - Validates UUID format to prevent injection attacks - Handles storage connection failures gracefully - No error information leakage on failure """ if not thread_id or not _is_valid_uuid(thread_id): return None try: storage = get_storage() key = f"thread:{thread_id}" data = storage.get(key) if data: return ThreadContext.model_validate_json(data) return None except Exception: # Silently handle errors to avoid exposing storage details return None def add_turn( thread_id: str, role: str, content: str, files: Optional[list[str]] = None, images: Optional[list[str]] = None, tool_name: Optional[str] = None, model_provider: Optional[str] = None, model_name: Optional[str] = None, model_metadata: Optional[dict[str, Any]] = None, ) -> bool: """ Add turn to existing thread with atomic file ordering. Appends a new conversation turn to an existing thread. This is the core function for building conversation history and enabling cross-tool continuation. Each turn preserves the tool and model that generated it. Args: thread_id: UUID of the conversation thread role: "user" (Agent request) or "assistant" (model response) content: The actual message/response content files: Optional list of files referenced in this turn images: Optional list of images referenced in this turn tool_name: Name of the tool adding this turn (for attribution) model_provider: Provider used (e.g., "google", "openai") model_name: Specific model used (e.g., "gemini-2.5-flash", "o3-mini") model_metadata: Additional model info (e.g., thinking mode, token usage) Returns: bool: True if turn was successfully added, False otherwise Failure cases: - Thread doesn't exist or expired - Maximum turn limit reached - Storage connection failure Note: - Refreshes thread TTL to configured timeout on successful update - Turn limits prevent runaway conversations - File references are preserved for cross-tool access with atomic ordering - Image references are preserved for cross-tool visual context - Model information enables cross-provider conversations """ logger.debug(f"[FLOW] Adding {role} turn to {thread_id} ({tool_name})") context = get_thread(thread_id) if not context: logger.debug(f"[FLOW] Thread {thread_id} not found for turn addition") return False # Check turn limit to prevent runaway conversations if len(context.turns) >= MAX_CONVERSATION_TURNS: logger.debug(f"[FLOW] Thread {thread_id} at max turns ({MAX_CONVERSATION_TURNS})") return False # Create new turn with complete metadata turn = ConversationTurn( role=role, content=content, timestamp=datetime.now(timezone.utc).isoformat(), files=files, # Preserved for cross-tool file context images=images, # Preserved for cross-tool visual context tool_name=tool_name, # Track which tool generated this turn model_provider=model_provider, # Track model provider model_name=model_name, # Track specific model model_metadata=model_metadata, # Additional model info ) context.turns.append(turn) context.last_updated_at = datetime.now(timezone.utc).isoformat() # Save back to storage and refresh TTL try: storage = get_storage() key = f"thread:{thread_id}" storage.setex(key, CONVERSATION_TIMEOUT_SECONDS, context.model_dump_json()) # Refresh TTL to configured timeout return True except Exception as e: logger.debug(f"[FLOW] Failed to save turn to storage: {type(e).__name__}") return False def get_thread_chain(thread_id: str, max_depth: int = 20) -> list[ThreadContext]: """ Traverse the parent chain to get all threads in conversation sequence. Retrieves the complete conversation chain by following parent_thread_id links. Returns threads in chronological order (oldest first). Args: thread_id: Starting thread ID max_depth: Maximum chain depth to prevent infinite loops Returns: list[ThreadContext]: All threads in chain, oldest first """ chain = [] current_id = thread_id seen_ids = set() # Build chain from current to oldest while current_id and len(chain) < max_depth: # Prevent circular references if current_id in seen_ids: logger.warning(f"[THREAD] Circular reference detected in thread chain at {current_id}") break seen_ids.add(current_id) context = get_thread(current_id) if not context: logger.debug(f"[THREAD] Thread {current_id} not found in chain traversal") break chain.append(context) current_id = context.parent_thread_id # Reverse to get chronological order (oldest first) chain.reverse() logger.debug(f"[THREAD] Retrieved chain of {len(chain)} threads for {thread_id}") return chain def get_conversation_file_list(context: ThreadContext) -> list[str]: """ Extract all unique files from conversation turns with newest-first prioritization. This function implements the core file prioritization logic used throughout the conversation memory system. It walks backwards through conversation turns (from newest to oldest) and collects unique file references, ensuring that when the same file appears in multiple turns, the reference from the NEWEST turn takes precedence. PRIORITIZATION ALGORITHM: 1. Iterate through turns in REVERSE order (index len-1 down to 0) 2. For each turn, process files in the order they appear in turn.files 3. Add file to result list only if not already seen (newest reference wins) 4. Skip duplicate files that were already added from newer turns This ensures that: - Files from newer conversation turns appear first in the result - When the same file is referenced multiple times, only the newest reference is kept - The order reflects the most recent conversation context Example: Turn 1: files = ["main.py", "utils.py"] Turn 2: files = ["test.py"] Turn 3: files = ["main.py", "config.py"] # main.py appears again Result: ["main.py", "config.py", "test.py", "utils.py"] (main.py from Turn 3 takes precedence over Turn 1) Args: context: ThreadContext containing all conversation turns to process Returns: list[str]: Unique file paths ordered by newest reference first. Empty list if no turns exist or no files are referenced. Performance: - Time Complexity: O(n*m) where n=turns, m=avg files per turn - Space Complexity: O(f) where f=total unique files - Uses set for O(1) duplicate detection """ if not context.turns: logger.debug("[FILES] No turns found, returning empty file list") return [] # Collect files by walking backwards (newest to oldest turns) seen_files = set() file_list = [] logger.debug(f"[FILES] Collecting files from {len(context.turns)} turns (newest first)") # Process turns in reverse order (newest first) - this is the CORE of newest-first prioritization # By iterating from len-1 down to 0, we encounter newer turns before older turns # When we find a duplicate file, we skip it because the newer version is already in our list for i in range(len(context.turns) - 1, -1, -1): # REVERSE: newest turn first turn = context.turns[i] if turn.files: logger.debug(f"[FILES] Turn {i + 1} has {len(turn.files)} files: {turn.files}") for file_path in turn.files: if file_path not in seen_files: # First time seeing this file - add it (this is the NEWEST reference) seen_files.add(file_path) file_list.append(file_path) logger.debug(f"[FILES] Added new file: {file_path} (from turn {i + 1})") else: # File already seen from a NEWER turn - skip this older reference logger.debug(f"[FILES] Skipping duplicate file: {file_path} (newer version already included)") logger.debug(f"[FILES] Final file list ({len(file_list)}): {file_list}") return file_list def get_conversation_image_list(context: ThreadContext) -> list[str]: """ Extract all unique images from conversation turns with newest-first prioritization. This function implements the identical prioritization logic as get_conversation_file_list() to ensure consistency in how images are handled across conversation turns. It walks backwards through conversation turns (from newest to oldest) and collects unique image references, ensuring that when the same image appears in multiple turns, the reference from the NEWEST turn takes precedence. PRIORITIZATION ALGORITHM: 1. Iterate through turns in REVERSE order (index len-1 down to 0) 2. For each turn, process images in the order they appear in turn.images 3. Add image to result list only if not already seen (newest reference wins) 4. Skip duplicate images that were already added from newer turns This ensures that: - Images from newer conversation turns appear first in the result - When the same image is referenced multiple times, only the newest reference is kept - The order reflects the most recent conversation context Example: Turn 1: images = ["diagram.png", "flow.jpg"] Turn 2: images = ["error.png"] Turn 3: images = ["diagram.png", "updated.png"] # diagram.png appears again Result: ["diagram.png", "updated.png", "error.png", "flow.jpg"] (diagram.png from Turn 3 takes precedence over Turn 1) Args: context: ThreadContext containing all conversation turns to process Returns: list[str]: Unique image paths ordered by newest reference first. Empty list if no turns exist or no images are referenced. Performance: - Time Complexity: O(n*m) where n=turns, m=avg images per turn - Space Complexity: O(i) where i=total unique images - Uses set for O(1) duplicate detection """ if not context.turns: logger.debug("[IMAGES] No turns found, returning empty image list") return [] # Collect images by walking backwards (newest to oldest turns) seen_images = set() image_list = [] logger.debug(f"[IMAGES] Collecting images from {len(context.turns)} turns (newest first)") # Process turns in reverse order (newest first) - this is the CORE of newest-first prioritization # By iterating from len-1 down to 0, we encounter newer turns before older turns # When we find a duplicate image, we skip it because the newer version is already in our list for i in range(len(context.turns) - 1, -1, -1): # REVERSE: newest turn first turn = context.turns[i] if turn.images: logger.debug(f"[IMAGES] Turn {i + 1} has {len(turn.images)} images: {turn.images}") for image_path in turn.images: if image_path not in seen_images: # First time seeing this image - add it (this is the NEWEST reference) seen_images.add(image_path) image_list.append(image_path) logger.debug(f"[IMAGES] Added new image: {image_path} (from turn {i + 1})") else: # Image already seen from a NEWER turn - skip this older reference logger.debug(f"[IMAGES] Skipping duplicate image: {image_path} (newer version already included)") logger.debug(f"[IMAGES] Final image list ({len(image_list)}): {image_list}") return image_list def _plan_file_inclusion_by_size(all_files: list[str], max_file_tokens: int) -> tuple[list[str], list[str], int]: """ Plan which files to include based on size constraints. This is ONLY used for conversation history building, not MCP boundary checks. Args: all_files: List of files to consider for inclusion max_file_tokens: Maximum tokens available for file content Returns: Tuple of (files_to_include, files_to_skip, estimated_total_tokens) """ if not all_files: return [], [], 0 files_to_include = [] files_to_skip = [] total_tokens = 0 logger.debug(f"[FILES] Planning inclusion for {len(all_files)} files with budget {max_file_tokens:,} tokens") for file_path in all_files: try: from utils.file_utils import estimate_file_tokens if os.path.exists(file_path) and os.path.isfile(file_path): # Use centralized token estimation for consistency estimated_tokens = estimate_file_tokens(file_path) if total_tokens + estimated_tokens <= max_file_tokens: files_to_include.append(file_path) total_tokens += estimated_tokens logger.debug( f"[FILES] Including {file_path} - {estimated_tokens:,} tokens (total: {total_tokens:,})" ) else: files_to_skip.append(file_path) logger.debug( f"[FILES] Skipping {file_path} - would exceed budget (needs {estimated_tokens:,} tokens)" ) else: files_to_skip.append(file_path) # More descriptive message for missing files if not os.path.exists(file_path): logger.debug( f"[FILES] Skipping {file_path} - file no longer exists (may have been moved/deleted since conversation)" ) else: logger.debug(f"[FILES] Skipping {file_path} - file not accessible (not a regular file)") except Exception as e: files_to_skip.append(file_path) logger.debug(f"[FILES] Skipping {file_path} - error during processing: {type(e).__name__}: {e}") logger.debug( f"[FILES] Inclusion plan: {len(files_to_include)} include, {len(files_to_skip)} skip, {total_tokens:,} tokens" ) return files_to_include, files_to_skip, total_tokens def build_conversation_history(context: ThreadContext, model_context=None, read_files_func=None) -> tuple[str, int]: """ Build formatted conversation history for tool prompts with embedded file contents. Creates a comprehensive conversation history that includes both conversation turns and file contents, with intelligent prioritization to maximize relevant context within token limits. This function enables stateless tools to access complete conversation context from previous interactions, including cross-tool continuations. FILE PRIORITIZATION BEHAVIOR: Files from newer conversation turns are prioritized over files from older turns. When the same file appears in multiple turns, the reference from the NEWEST turn takes precedence. This ensures the most recent file context is preserved when token limits require file exclusions. CONVERSATION CHAIN HANDLING: If the thread has a parent_thread_id, this function traverses the entire chain to include complete conversation history across multiple linked threads. File prioritization works across the entire chain, not just the current thread. CONVERSATION TURN ORDERING STRATEGY: The function employs a sophisticated two-phase approach for optimal token utilization: PHASE 1 - COLLECTION (Newest-First for Token Budget): - Processes conversation turns in REVERSE chronological order (newest to oldest) - Prioritizes recent turns within token constraints - If token budget is exceeded, OLDER turns are excluded first - Ensures the most contextually relevant recent exchanges are preserved PHASE 2 - PRESENTATION (Chronological for LLM Understanding): - Reverses the collected turns back to chronological order (oldest to newest) - Presents conversation flow naturally for LLM comprehension - Maintains "--- Turn 1, Turn 2, Turn 3..." sequential numbering - Enables LLM to follow conversation progression logically This approach balances recency prioritization with natural conversation flow. TOKEN MANAGEMENT: - Uses model-specific token allocation (file_tokens + history_tokens) - Files are embedded ONCE at the start to prevent duplication - Turn collection prioritizes newest-first, presentation shows chronologically - Stops adding turns when token budget would be exceeded - Gracefully handles token limits with informative notes Args: context: ThreadContext containing the conversation to format model_context: ModelContext for token allocation (optional, uses DEFAULT_MODEL fallback) read_files_func: Optional function to read files (primarily for testing) Returns: tuple[str, int]: (formatted_conversation_history, total_tokens_used) Returns ("", 0) if no conversation turns exist in the context Output Format: === CONVERSATION HISTORY (CONTINUATION) === Thread: Tool: Turn / You are continuing this conversation thread from where it left off. === FILES REFERENCED IN THIS CONVERSATION === The following files have been shared and analyzed during our conversation. [NOTE: X files omitted due to size constraints] Refer to these when analyzing the context and requests below: === END REFERENCED FILES === Previous conversation turns: --- Turn 1 (Claude) --- Files used in this turn: file1.py, file2.py --- Turn 2 (gemini-2.5-flash using analyze via google) --- Files used in this turn: file3.py === END CONVERSATION HISTORY === IMPORTANT: You are continuing an existing conversation thread... This is turn X of the conversation - use the conversation history above... Cross-Tool Collaboration: This formatted history allows any tool to "see" both conversation context AND file contents from previous tools, enabling seamless handoffs between analyze, codereview, debug, chat, and other tools while maintaining complete context. Performance Characteristics: - O(n) file collection with newest-first prioritization - Intelligent token budgeting prevents context window overflow - In-memory persistence with automatic TTL management - Graceful degradation when files are inaccessible or too large """ # Get the complete thread chain if context.parent_thread_id: # This thread has a parent, get the full chain chain = get_thread_chain(context.thread_id) # Collect all turns from all threads in chain all_turns = [] total_turns = 0 for thread in chain: all_turns.extend(thread.turns) total_turns += len(thread.turns) # Use centralized file collection logic for consistency across the entire chain # This ensures files from newer turns across ALL threads take precedence # over files from older turns, maintaining the newest-first prioritization # even when threads are chained together temp_context = ThreadContext( thread_id="merged_chain", created_at=context.created_at, last_updated_at=context.last_updated_at, tool_name=context.tool_name, turns=all_turns, # All turns from entire chain in chronological order initial_context=context.initial_context, ) all_files = get_conversation_file_list(temp_context) # Applies newest-first logic to entire chain logger.debug(f"[THREAD] Built history from {len(chain)} threads with {total_turns} total turns") else: # Single thread, no parent chain all_turns = context.turns total_turns = len(context.turns) all_files = get_conversation_file_list(context) if not all_turns: return "", 0 logger.debug(f"[FILES] Found {len(all_files)} unique files in conversation history") # Get model-specific token allocation early (needed for both files and turns) if model_context is None: from config import DEFAULT_MODEL, IS_AUTO_MODE from utils.model_context import ModelContext # In auto mode, use an intelligent fallback model for token calculations # since "auto" is not a real model with a provider model_name = DEFAULT_MODEL if IS_AUTO_MODE and model_name.lower() == "auto": # Use intelligent fallback based on available API keys from providers.registry import ModelProviderRegistry model_name = ModelProviderRegistry.get_preferred_fallback_model() model_context = ModelContext(model_name) token_allocation = model_context.calculate_token_allocation() max_file_tokens = token_allocation.file_tokens max_history_tokens = token_allocation.history_tokens logger.debug(f"[HISTORY] Using model-specific limits for {model_context.model_name}:") logger.debug(f"[HISTORY] Max file tokens: {max_file_tokens:,}") logger.debug(f"[HISTORY] Max history tokens: {max_history_tokens:,}") history_parts = [ "=== CONVERSATION HISTORY (CONTINUATION) ===", f"Thread: {context.thread_id}", f"Tool: {context.tool_name}", # Original tool that started the conversation f"Turn {total_turns}/{MAX_CONVERSATION_TURNS}", "You are continuing this conversation thread from where it left off.", "", ] # Embed files referenced in this conversation with size-aware selection if all_files: logger.debug(f"[FILES] Starting embedding for {len(all_files)} files") # Plan file inclusion based on size constraints # CRITICAL: all_files is already ordered by newest-first prioritization from get_conversation_file_list() # So when _plan_file_inclusion_by_size() hits token limits, it naturally excludes OLDER files first # while preserving the most recent file references - exactly what we want! files_to_include, files_to_skip, estimated_tokens = _plan_file_inclusion_by_size(all_files, max_file_tokens) if files_to_skip: logger.info(f"[FILES] Excluding {len(files_to_skip)} files from conversation history: {files_to_skip}") logger.debug("[FILES] Files excluded for various reasons (size constraints, missing files, access issues)") if files_to_include: history_parts.extend( [ "=== FILES REFERENCED IN THIS CONVERSATION ===", "The following files have been shared and analyzed during our conversation.", ( "" if not files_to_skip else f"[NOTE: {len(files_to_skip)} files omitted (size constraints, missing files, or access issues)]" ), "Refer to these when analyzing the context and requests below:", "", ] ) if read_files_func is None: from utils.file_utils import read_file_content # Process files for embedding file_contents = [] total_tokens = 0 files_included = 0 for file_path in files_to_include: try: logger.debug(f"[FILES] Processing file {file_path}") formatted_content, content_tokens = read_file_content(file_path) if formatted_content: file_contents.append(formatted_content) total_tokens += content_tokens files_included += 1 logger.debug( f"File embedded in conversation history: {file_path} ({content_tokens:,} tokens)" ) else: logger.debug(f"File skipped (empty content): {file_path}") except Exception as e: # More descriptive error handling for missing files try: if not os.path.exists(file_path): logger.info( f"File no longer accessible for conversation history: {file_path} - file was moved/deleted since conversation (marking as excluded)" ) else: logger.warning( f"Failed to embed file in conversation history: {file_path} - {type(e).__name__}: {e}" ) except Exception: # Fallback if path translation also fails logger.warning( f"Failed to embed file in conversation history: {file_path} - {type(e).__name__}: {e}" ) continue if file_contents: files_content = "".join(file_contents) if files_to_skip: files_content += ( f"\n[NOTE: {len(files_to_skip)} additional file(s) were omitted due to size constraints, missing files, or access issues. " f"These were older files from earlier conversation turns.]\n" ) history_parts.append(files_content) logger.debug( f"Conversation history file embedding complete: {files_included} files embedded, {len(files_to_skip)} omitted, {total_tokens:,} total tokens" ) else: history_parts.append("(No accessible files found)") logger.debug(f"[FILES] No accessible files found from {len(files_to_include)} planned files") else: # Fallback to original read_files function files_content = read_files_func(all_files) if files_content: # Add token validation for the combined file content from utils.token_utils import check_token_limit within_limit, estimated_tokens = check_token_limit(files_content) if within_limit: history_parts.append(files_content) else: # Handle token limit exceeded for conversation files error_message = f"ERROR: The total size of files referenced in this conversation has exceeded the context limit and cannot be displayed.\nEstimated tokens: {estimated_tokens}, but limit is {max_file_tokens}." history_parts.append(error_message) else: history_parts.append("(No accessible files found)") history_parts.extend( [ "", "=== END REFERENCED FILES ===", "", ] ) history_parts.append("Previous conversation turns:") # === PHASE 1: COLLECTION (Newest-First for Token Budget) === # Build conversation turns bottom-up (most recent first) to prioritize recent context within token limits # This ensures we include as many recent turns as possible within the token budget by excluding # OLDER turns first when space runs out, preserving the most contextually relevant exchanges turn_entries = [] # Will store (index, formatted_turn_content) for chronological ordering later total_turn_tokens = 0 file_embedding_tokens = sum(model_context.estimate_tokens(part) for part in history_parts) # CRITICAL: Process turns in REVERSE chronological order (newest to oldest) # This prioritization strategy ensures recent context is preserved when token budget is tight for idx in range(len(all_turns) - 1, -1, -1): turn = all_turns[idx] turn_num = idx + 1 if turn.role == "user": role_label = "Agent" else: role_label = turn.model_name or "Assistant" # Build the complete turn content turn_parts = [] # Add turn header with tool attribution for cross-tool tracking turn_header = f"\n--- Turn {turn_num} ({role_label}" if turn.tool_name: turn_header += f" using {turn.tool_name}" # Add model info if available if turn.model_provider: provider_descriptor = turn.model_provider if turn.model_name and turn.model_name != role_label: provider_descriptor += f"/{turn.model_name}" turn_header += f" via {provider_descriptor}" elif turn.model_name and turn.model_name != role_label: turn_header += f" via {turn.model_name}" turn_header += ") ---" turn_parts.append(turn_header) # Get tool-specific formatting if available # This includes file references and the actual content tool_formatted_content = _get_tool_formatted_content(turn) turn_parts.extend(tool_formatted_content) # Calculate tokens for this turn turn_content = "\n".join(turn_parts) turn_tokens = model_context.estimate_tokens(turn_content) # Check if adding this turn would exceed history budget if file_embedding_tokens + total_turn_tokens + turn_tokens > max_history_tokens: # Stop adding turns - we've reached the limit logger.debug(f"[HISTORY] Stopping at turn {turn_num} - would exceed history budget") logger.debug(f"[HISTORY] File tokens: {file_embedding_tokens:,}") logger.debug(f"[HISTORY] Turn tokens so far: {total_turn_tokens:,}") logger.debug(f"[HISTORY] This turn: {turn_tokens:,}") logger.debug(f"[HISTORY] Would total: {file_embedding_tokens + total_turn_tokens + turn_tokens:,}") logger.debug(f"[HISTORY] Budget: {max_history_tokens:,}") break # Add this turn to our collection (we'll reverse it later for chronological presentation) # Store the original index to maintain proper turn numbering in final output turn_entries.append((idx, turn_content)) total_turn_tokens += turn_tokens # === PHASE 2: PRESENTATION (Chronological for LLM Understanding) === # Reverse the collected turns to restore chronological order (oldest first) # This gives the LLM a natural conversation flow: Turn 1 → Turn 2 → Turn 3... # while still having prioritized recent turns during the token-constrained collection phase turn_entries.reverse() # Add the turns in chronological order for natural LLM comprehension # The LLM will see: "--- Turn 1 (Agent) ---" followed by "--- Turn 2 (Model) ---" etc. for _, turn_content in turn_entries: history_parts.append(turn_content) # Log what we included included_turns = len(turn_entries) total_turns = len(all_turns) if included_turns < total_turns: logger.info(f"[HISTORY] Included {included_turns}/{total_turns} turns due to token limit") history_parts.append(f"\n[Note: Showing {included_turns} most recent turns out of {total_turns} total]") history_parts.extend( [ "", "=== END CONVERSATION HISTORY ===", "", "IMPORTANT: You are continuing an existing conversation thread. Build upon the previous exchanges shown above,", "reference earlier points, and maintain consistency with what has been discussed.", "", "DO NOT repeat or summarize previous analysis, findings, or instructions that are already covered in the", "conversation history. Instead, provide only new insights, additional analysis, or direct answers to", "the follow-up question / concerns / insights. Assume the user has read the prior conversation.", "", f"This is turn {len(all_turns) + 1} of the conversation - use the conversation history above to provide a coherent continuation.", ] ) # Calculate total tokens for the complete conversation history complete_history = "\n".join(history_parts) from utils.token_utils import estimate_tokens total_conversation_tokens = estimate_tokens(complete_history) # Summary log of what was built user_turns = len([t for t in all_turns if t.role == "user"]) assistant_turns = len([t for t in all_turns if t.role == "assistant"]) logger.debug( f"[FLOW] Built conversation history: {user_turns} user + {assistant_turns} assistant turns, {len(all_files)} files, {total_conversation_tokens:,} tokens" ) return complete_history, total_conversation_tokens def _get_tool_formatted_content(turn: ConversationTurn) -> list[str]: """ Get tool-specific formatting for a conversation turn. This function attempts to use the tool's custom formatting method if available, falling back to default formatting if the tool cannot be found or doesn't provide custom formatting. Args: turn: The conversation turn to format Returns: list[str]: Formatted content lines for this turn """ if turn.tool_name: try: # Dynamically import to avoid circular dependencies from server import TOOLS tool = TOOLS.get(turn.tool_name) if tool: # Use inheritance pattern - try to call the method directly # If it doesn't exist or raises AttributeError, fall back to default try: return tool.format_conversation_turn(turn) except AttributeError: # Tool doesn't implement format_conversation_turn - use default pass except Exception as e: # Log but don't fail - fall back to default formatting logger.debug(f"[HISTORY] Could not get tool-specific formatting for {turn.tool_name}: {e}") # Default formatting return _default_turn_formatting(turn) def _default_turn_formatting(turn: ConversationTurn) -> list[str]: """ Default formatting for conversation turns. This provides the standard formatting when no tool-specific formatting is available. Args: turn: The conversation turn to format Returns: list[str]: Default formatted content lines """ parts = [] # Add files context if present if turn.files: parts.append(f"Files used in this turn: {', '.join(turn.files)}") parts.append("") # Empty line for readability # Add the actual content parts.append(turn.content) return parts def _is_valid_uuid(val: str) -> bool: """ Validate UUID format for security Ensures thread IDs are valid UUIDs to prevent injection attacks and malformed requests. Args: val: String to validate as UUID Returns: bool: True if valid UUID format, False otherwise """ try: uuid.UUID(val) return True except ValueError: return False ================================================ FILE: utils/env.py ================================================ """Centralized environment variable access for PAL MCP Server.""" from __future__ import annotations import os from collections.abc import Mapping from contextlib import contextmanager from pathlib import Path try: from dotenv import dotenv_values, load_dotenv except ImportError: # pragma: no cover - optional dependency dotenv_values = None # type: ignore[assignment] load_dotenv = None # type: ignore[assignment] _PROJECT_ROOT = Path(__file__).resolve().parent.parent _ENV_PATH = _PROJECT_ROOT / ".env" _DOTENV_VALUES: dict[str, str | None] = {} _FORCE_ENV_OVERRIDE = False def _read_dotenv_values() -> dict[str, str | None]: if dotenv_values is not None and _ENV_PATH.exists(): loaded = dotenv_values(_ENV_PATH) return dict(loaded) return {} def _compute_force_override(values: Mapping[str, str | None]) -> bool: raw = (values.get("PAL_MCP_FORCE_ENV_OVERRIDE") or "false").strip().lower() return raw == "true" def reload_env(dotenv_mapping: Mapping[str, str | None] | None = None) -> None: """Reload .env values and recompute override semantics. Args: dotenv_mapping: Optional mapping used instead of reading the .env file. Intended for tests; when provided, load_dotenv is not invoked. """ global _DOTENV_VALUES, _FORCE_ENV_OVERRIDE if dotenv_mapping is not None: _DOTENV_VALUES = dict(dotenv_mapping) _FORCE_ENV_OVERRIDE = _compute_force_override(_DOTENV_VALUES) return _DOTENV_VALUES = _read_dotenv_values() _FORCE_ENV_OVERRIDE = _compute_force_override(_DOTENV_VALUES) if load_dotenv is not None and _ENV_PATH.exists(): load_dotenv(dotenv_path=_ENV_PATH, override=_FORCE_ENV_OVERRIDE) reload_env() def env_override_enabled() -> bool: """Return True when PAL_MCP_FORCE_ENV_OVERRIDE is enabled via the .env file.""" return _FORCE_ENV_OVERRIDE def get_env(key: str, default: str | None = None) -> str | None: """Retrieve environment variables respecting PAL_MCP_FORCE_ENV_OVERRIDE.""" if env_override_enabled(): if key in _DOTENV_VALUES: value = _DOTENV_VALUES[key] return value if value is not None else default return default return os.getenv(key, default) def get_env_bool(key: str, default: bool = False) -> bool: """Boolean helper that respects override semantics.""" raw_default = "true" if default else "false" raw_value = get_env(key, raw_default) return (raw_value or raw_default).strip().lower() == "true" def get_all_env() -> dict[str, str | None]: """Expose the loaded .env mapping for diagnostics/logging.""" return dict(_DOTENV_VALUES) @contextmanager def suppress_env_vars(*names: str): """Temporarily remove environment variables during the context. Args: names: Environment variable names to remove. Empty or falsy names are ignored. """ removed: dict[str, str] = {} try: for name in names: if not name: continue if name in os.environ: removed[name] = os.environ[name] del os.environ[name] yield finally: for name, value in removed.items(): os.environ[name] = value ================================================ FILE: utils/file_types.py ================================================ """ File type definitions and constants for file processing This module centralizes all file type and extension definitions used throughout the MCP server for consistent file handling. """ # Programming language file extensions - core code files PROGRAMMING_LANGUAGES = { ".py", # Python ".js", # JavaScript ".ts", # TypeScript ".jsx", # React JavaScript ".tsx", # React TypeScript ".java", # Java ".cpp", # C++ ".c", # C ".h", # C/C++ Header ".hpp", # C++ Header ".cs", # C# ".go", # Go ".rs", # Rust ".rb", # Ruby ".php", # PHP ".swift", # Swift ".kt", # Kotlin ".scala", # Scala ".r", # R ".m", # Objective-C ".mm", # Objective-C++ } # Script and shell file extensions SCRIPTS = { ".sql", # SQL ".sh", # Shell ".bash", # Bash ".zsh", # Zsh ".fish", # Fish shell ".ps1", # PowerShell ".bat", # Batch ".cmd", # Command } # Configuration and data file extensions CONFIGS = { ".yml", # YAML ".yaml", # YAML ".json", # JSON ".xml", # XML ".toml", # TOML ".ini", # INI ".cfg", # Config ".conf", # Config ".properties", # Properties ".env", # Environment } # Documentation and markup file extensions DOCS = { ".txt", # Text ".md", # Markdown ".rst", # reStructuredText ".tex", # LaTeX } # Web development file extensions WEB = { ".html", # HTML ".css", # CSS ".scss", # Sass ".sass", # Sass ".less", # Less } # Additional text file extensions for logs and data TEXT_DATA = { ".log", # Log files ".csv", # CSV ".tsv", # TSV ".gitignore", # Git ignore ".dockerfile", # Dockerfile ".makefile", # Make ".cmake", # CMake ".gradle", # Gradle ".sbt", # SBT ".pom", # Maven POM ".lock", # Lock files ".changeset", # Precommit changeset } # Image file extensions - limited to what AI models actually support # Based on OpenAI and Gemini supported formats: PNG, JPEG, GIF, WebP IMAGES = {".jpg", ".jpeg", ".png", ".gif", ".webp"} # Binary executable and library extensions BINARIES = { ".exe", # Windows executable ".dll", # Windows library ".so", # Linux shared object ".dylib", # macOS dynamic library ".bin", # Binary ".class", # Java class } # Archive and package file extensions ARCHIVES = { ".jar", ".war", ".ear", # Java archives ".zip", ".tar", ".gz", # General archives ".7z", ".rar", # Compression ".deb", ".rpm", # Linux packages ".dmg", ".pkg", # macOS packages } # Derived sets for different use cases CODE_EXTENSIONS = PROGRAMMING_LANGUAGES | SCRIPTS | CONFIGS | DOCS | WEB PROGRAMMING_EXTENSIONS = PROGRAMMING_LANGUAGES # For line numbering TEXT_EXTENSIONS = CODE_EXTENSIONS | TEXT_DATA IMAGE_EXTENSIONS = IMAGES BINARY_EXTENSIONS = BINARIES | ARCHIVES # All extensions by category for easy access FILE_CATEGORIES = { "programming": PROGRAMMING_LANGUAGES, "scripts": SCRIPTS, "configs": CONFIGS, "docs": DOCS, "web": WEB, "text_data": TEXT_DATA, "images": IMAGES, "binaries": BINARIES, "archives": ARCHIVES, } def get_file_category(file_path: str) -> str: """ Determine the category of a file based on its extension. Args: file_path: Path to the file Returns: Category name or "unknown" if not recognized """ from pathlib import Path extension = Path(file_path).suffix.lower() for category, extensions in FILE_CATEGORIES.items(): if extension in extensions: return category return "unknown" def is_code_file(file_path: str) -> bool: """Check if a file is a code file (programming language).""" from pathlib import Path return Path(file_path).suffix.lower() in PROGRAMMING_LANGUAGES def is_text_file(file_path: str) -> bool: """Check if a file is a text file.""" from pathlib import Path return Path(file_path).suffix.lower() in TEXT_EXTENSIONS def is_binary_file(file_path: str) -> bool: """Check if a file is a binary file.""" from pathlib import Path return Path(file_path).suffix.lower() in BINARY_EXTENSIONS # File-type specific token-to-byte ratios for accurate token estimation # Based on empirical analysis of file compression characteristics and tokenization patterns TOKEN_ESTIMATION_RATIOS = { # Programming languages ".py": 3.5, # Python - moderate verbosity ".js": 3.2, # JavaScript - compact syntax ".ts": 3.3, # TypeScript - type annotations add tokens ".jsx": 3.1, # React JSX - JSX tags are tokenized efficiently ".tsx": 3.0, # React TSX - combination of TypeScript + JSX ".java": 3.6, # Java - verbose syntax, long identifiers ".cpp": 3.7, # C++ - preprocessor directives, templates ".c": 3.8, # C - function definitions, struct declarations ".go": 3.9, # Go - explicit error handling, package names ".rs": 3.5, # Rust - similar to Python in verbosity ".php": 3.3, # PHP - mixed HTML/code, variable prefixes ".rb": 3.6, # Ruby - descriptive method names ".swift": 3.4, # Swift - modern syntax, type inference ".kt": 3.5, # Kotlin - similar to modern languages ".scala": 3.2, # Scala - functional programming, concise # Scripts and configuration ".sh": 4.1, # Shell scripts - commands and paths ".bat": 4.0, # Batch files - similar to shell ".ps1": 3.8, # PowerShell - more structured than bash ".sql": 3.8, # SQL - keywords and table/column names # Data and configuration formats ".json": 2.5, # JSON - lots of punctuation and quotes ".yaml": 3.0, # YAML - structured but readable ".yml": 3.0, # YAML (alternative extension) ".xml": 2.8, # XML - tags and attributes ".toml": 3.2, # TOML - similar to config files # Documentation and text ".md": 4.2, # Markdown - natural language with formatting ".txt": 4.0, # Plain text - mostly natural language ".rst": 4.1, # reStructuredText - documentation format # Web technologies ".html": 2.9, # HTML - tags and attributes ".css": 3.4, # CSS - properties and selectors # Logs and data ".log": 4.5, # Log files - timestamps, messages, stack traces ".csv": 3.1, # CSV - data with delimiters # Infrastructure files ".dockerfile": 3.7, # Dockerfile - commands and paths ".tf": 3.5, # Terraform - infrastructure as code } def get_token_estimation_ratio(file_path: str) -> float: """ Get the token estimation ratio for a file based on its extension. Args: file_path: Path to the file Returns: Token-to-byte ratio for the file type (default: 3.5 for unknown types) """ from pathlib import Path extension = Path(file_path).suffix.lower() return TOKEN_ESTIMATION_RATIOS.get(extension, 3.5) # Conservative default # MIME type mappings for image files - limited to what AI models actually support # Based on OpenAI and Gemini supported formats: PNG, JPEG, GIF, WebP IMAGE_MIME_TYPES = { ".jpg": "image/jpeg", ".jpeg": "image/jpeg", ".png": "image/png", ".gif": "image/gif", ".webp": "image/webp", } def get_image_mime_type(extension: str) -> str: """ Get the MIME type for an image file extension. Args: extension: File extension (with or without leading dot) Returns: MIME type string (default: image/jpeg for unknown extensions) """ if not extension.startswith("."): extension = "." + extension extension = extension.lower() return IMAGE_MIME_TYPES.get(extension, "image/jpeg") ================================================ FILE: utils/file_utils.py ================================================ """ File reading utilities with directory support and token management This module provides secure file access functionality for the MCP server. It implements critical security measures to prevent unauthorized file access and manages token limits to ensure efficient API usage. Key Features: - Path validation and sandboxing to prevent directory traversal attacks - Support for both individual files and recursive directory reading - Token counting and management to stay within API limits - Automatic file type detection and filtering - Comprehensive error handling with informative messages Security Model: - All file access is restricted to PROJECT_ROOT and its subdirectories - Absolute paths are required to prevent ambiguity - Symbolic links are resolved to ensure they stay within bounds CONVERSATION MEMORY INTEGRATION: This module works with the conversation memory system to support efficient multi-turn file handling: 1. DEDUPLICATION SUPPORT: - File reading functions are called by conversation-aware tools - Supports newest-first file prioritization by providing accurate token estimation - Enables efficient file content caching and token budget management 2. TOKEN BUDGET OPTIMIZATION: - Provides accurate token estimation for file content before reading - Supports the dual prioritization strategy by enabling precise budget calculations - Enables tools to make informed decisions about which files to include 3. CROSS-TOOL FILE PERSISTENCE: - File reading results are used across different tools in conversation chains - Consistent file access patterns support conversation continuation scenarios - Error handling preserves conversation flow when files become unavailable """ import json import logging import os from datetime import datetime, timezone from pathlib import Path from typing import Optional from .file_types import BINARY_EXTENSIONS, CODE_EXTENSIONS, IMAGE_EXTENSIONS, TEXT_EXTENSIONS from .security_config import EXCLUDED_DIRS, is_dangerous_path from .token_utils import DEFAULT_CONTEXT_WINDOW, estimate_tokens def _is_builtin_custom_models_config(path_str: str) -> bool: """ Check if path points to the server's built-in custom_models.json config file. This only matches the server's internal config, not user-specified CUSTOM_MODELS_CONFIG_PATH. We identify the built-in config by checking if it resolves to the server's conf directory. Args: path_str: Path to check Returns: True if this is the server's built-in custom_models.json config file """ try: path = Path(path_str) # Get the server root by going up from this file: utils/file_utils.py -> server_root server_root = Path(__file__).parent.parent builtin_config = server_root / "conf" / "custom_models.json" # Check if the path resolves to the same file as our built-in config # This handles both relative and absolute paths to the same file return path.resolve() == builtin_config.resolve() except Exception: # If path resolution fails, it's not our built-in config return False logger = logging.getLogger(__name__) def is_mcp_directory(path: Path) -> bool: """ Check if a directory is the MCP server's own directory. This prevents the MCP from including its own code when scanning projects where the MCP has been cloned as a subdirectory. Args: path: Directory path to check Returns: True if this is the MCP server directory or a subdirectory """ if not path.is_dir(): return False # Get the directory where the MCP server is running from # __file__ is utils/file_utils.py, so parent.parent is the MCP root mcp_server_dir = Path(__file__).parent.parent.resolve() # Check if the given path is the MCP server directory or a subdirectory try: path.resolve().relative_to(mcp_server_dir) logger.info(f"Detected MCP server directory at {path}, will exclude from scanning") return True except ValueError: # Not a subdirectory of MCP server return False def get_user_home_directory() -> Optional[Path]: """ Get the user's home directory. Returns: User's home directory path """ return Path.home() def is_home_directory_root(path: Path) -> bool: """ Check if the given path is the user's home directory root. This prevents scanning the entire home directory which could include sensitive data and non-project files. Args: path: Directory path to check Returns: True if this is the home directory root """ user_home = get_user_home_directory() if not user_home: return False try: resolved_path = path.resolve() resolved_home = user_home.resolve() # Check if this is exactly the home directory if resolved_path == resolved_home: logger.warning( f"Attempted to scan user home directory root: {path}. Please specify a subdirectory instead." ) return True # Also check common home directory patterns path_str = str(resolved_path).lower() home_patterns = [ "/users/", # macOS "/home/", # Linux "c:\\users\\", # Windows "c:/users/", # Windows with forward slashes ] for pattern in home_patterns: if pattern in path_str: # Extract the user directory path # e.g., /Users/fahad or /home/username parts = path_str.split(pattern) if len(parts) > 1: # Get the part after the pattern after_pattern = parts[1] # Check if we're at the user's root (no subdirectories) if "/" not in after_pattern and "\\" not in after_pattern: logger.warning( f"Attempted to scan user home directory root: {path}. " f"Please specify a subdirectory instead." ) return True except Exception as e: logger.debug(f"Error checking if path is home directory: {e}") return False def detect_file_type(file_path: str) -> str: """ Detect file type for appropriate processing strategy. This function is intended for specific file type handling (e.g., image processing, binary file analysis, or enhanced file filtering). Args: file_path: Path to the file to analyze Returns: str: "text", "binary", or "image" """ path = Path(file_path) # Check extension first (fast) extension = path.suffix.lower() if extension in TEXT_EXTENSIONS: return "text" elif extension in IMAGE_EXTENSIONS: return "image" elif extension in BINARY_EXTENSIONS: return "binary" # Fallback: check magic bytes for text vs binary # This is helpful for files without extensions or unknown extensions try: with open(path, "rb") as f: chunk = f.read(1024) # Simple heuristic: if we can decode as UTF-8, likely text chunk.decode("utf-8") return "text" except UnicodeDecodeError: return "binary" except (FileNotFoundError, PermissionError) as e: logger.warning(f"Could not access file {file_path} for type detection: {e}") return "unknown" def should_add_line_numbers(file_path: str, include_line_numbers: Optional[bool] = None) -> bool: """ Determine if line numbers should be added to a file. Args: file_path: Path to the file include_line_numbers: Explicit preference, or None for auto-detection Returns: bool: True if line numbers should be added """ if include_line_numbers is not None: return include_line_numbers # Default: DO NOT add line numbers # Tools that want line numbers must explicitly request them return False def _normalize_line_endings(content: str) -> str: """ Normalize line endings for consistent line numbering. Args: content: File content with potentially mixed line endings Returns: str: Content with normalized LF line endings """ # Normalize all line endings to LF for consistent counting return content.replace("\r\n", "\n").replace("\r", "\n") def _add_line_numbers(content: str) -> str: """ Add line numbers to text content for precise referencing. Args: content: Text content to number Returns: str: Content with line numbers in format " 45│ actual code line" Supports files up to 99,999 lines with dynamic width allocation """ # Normalize line endings first normalized_content = _normalize_line_endings(content) lines = normalized_content.split("\n") # Dynamic width allocation based on total line count # This supports files of any size by computing required width total_lines = len(lines) width = len(str(total_lines)) width = max(width, 4) # Minimum padding for readability # Format with dynamic width and clear separator numbered_lines = [f"{i + 1:{width}d}│ {line}" for i, line in enumerate(lines)] return "\n".join(numbered_lines) def resolve_and_validate_path(path_str: str) -> Path: """ Resolves and validates a path against security policies. This function ensures safe file access by: 1. Requiring absolute paths (no ambiguity) 2. Resolving symlinks to prevent deception 3. Blocking access to dangerous system directories Args: path_str: Path string (must be absolute) Returns: Resolved Path object that is safe to access Raises: ValueError: If path is not absolute or otherwise invalid PermissionError: If path is in a dangerous location """ # Step 1: Create a Path object user_path = Path(path_str) # Step 2: Security Policy - Require absolute paths # Relative paths could be interpreted differently depending on working directory if not user_path.is_absolute(): raise ValueError(f"Relative paths are not supported. Please provide an absolute path.\nReceived: {path_str}") # Step 3: Resolve the absolute path (follows symlinks, removes .. and .) # This is critical for security as it reveals the true destination of symlinks resolved_path = user_path.resolve() # Step 4: Check against dangerous paths if is_dangerous_path(resolved_path): logger.warning(f"Access denied - dangerous path: {resolved_path}") raise PermissionError(f"Access to system directory denied: {path_str}") # Step 5: Check if it's the home directory root if is_home_directory_root(resolved_path): raise PermissionError( f"Cannot scan entire home directory: {path_str}\n" f"Please specify a subdirectory within your home folder." ) return resolved_path def expand_paths(paths: list[str], extensions: Optional[set[str]] = None) -> list[str]: """ Expand paths to individual files, handling both files and directories. This function recursively walks directories to find all matching files. It automatically filters out hidden files and common non-code directories like __pycache__ to avoid including generated or system files. Args: paths: List of file or directory paths (must be absolute) extensions: Optional set of file extensions to include (defaults to CODE_EXTENSIONS) Returns: List of individual file paths, sorted for consistent ordering """ if extensions is None: extensions = CODE_EXTENSIONS expanded_files = [] seen = set() for path in paths: try: # Validate each path for security before processing path_obj = resolve_and_validate_path(path) except (ValueError, PermissionError): # Skip invalid paths silently to allow partial success continue if not path_obj.exists(): continue # Safety checks for directory scanning if path_obj.is_dir(): # Check 1: Prevent scanning user's home directory root if is_home_directory_root(path_obj): logger.warning(f"Skipping home directory root: {path}. Please specify a project subdirectory instead.") continue # Check 2: Skip if this is the MCP's own directory if is_mcp_directory(path_obj): logger.info( f"Skipping MCP server directory: {path}. The MCP server code is excluded from project scans." ) continue if path_obj.is_file(): # Add file directly if str(path_obj) not in seen: expanded_files.append(str(path_obj)) seen.add(str(path_obj)) elif path_obj.is_dir(): # Walk directory recursively to find all files for root, dirs, files in os.walk(path_obj): # Filter directories in-place to skip hidden and excluded directories # This prevents descending into .git, .venv, __pycache__, node_modules, etc. original_dirs = dirs[:] dirs[:] = [] for d in original_dirs: # Skip hidden directories if d.startswith("."): continue # Skip excluded directories if d in EXCLUDED_DIRS: continue # Skip MCP directories found during traversal dir_path = Path(root) / d if is_mcp_directory(dir_path): logger.debug(f"Skipping MCP directory during traversal: {dir_path}") continue dirs.append(d) for file in files: # Skip hidden files (e.g., .DS_Store, .gitignore) if file.startswith("."): continue file_path = Path(root) / file # Filter by extension if specified if not extensions or file_path.suffix.lower() in extensions: full_path = str(file_path) # Use set to prevent duplicates if full_path not in seen: expanded_files.append(full_path) seen.add(full_path) # Sort for consistent ordering across different runs # This makes output predictable and easier to debug expanded_files.sort() return expanded_files def read_file_content( file_path: str, max_size: int = 1_000_000, *, include_line_numbers: Optional[bool] = None ) -> tuple[str, int]: """ Read a single file and format it for inclusion in AI prompts. This function handles various error conditions gracefully and always returns formatted content, even for errors. This ensures the AI model gets context about what files were attempted but couldn't be read. Args: file_path: Path to file (must be absolute) max_size: Maximum file size to read (default 1MB to prevent memory issues) include_line_numbers: Whether to add line numbers. If None, auto-detects based on file type Returns: Tuple of (formatted_content, estimated_tokens) Content is wrapped with clear delimiters for AI parsing """ logger.debug(f"[FILES] read_file_content called for: {file_path}") try: # Validate path security before any file operations path = resolve_and_validate_path(file_path) logger.debug(f"[FILES] Path validated and resolved: {path}") except (ValueError, PermissionError) as e: # Return error in a format that provides context to the AI logger.debug(f"[FILES] Path validation failed for {file_path}: {type(e).__name__}: {e}") error_msg = str(e) content = f"\n--- ERROR ACCESSING FILE: {file_path} ---\nError: {error_msg}\n--- END FILE ---\n" tokens = estimate_tokens(content) logger.debug(f"[FILES] Returning error content for {file_path}: {tokens} tokens") return content, tokens try: # Validate file existence and type if not path.exists(): logger.debug(f"[FILES] File does not exist: {file_path}") content = f"\n--- FILE NOT FOUND: {file_path} ---\nError: File does not exist\n--- END FILE ---\n" return content, estimate_tokens(content) if not path.is_file(): logger.debug(f"[FILES] Path is not a file: {file_path}") content = f"\n--- NOT A FILE: {file_path} ---\nError: Path is not a file\n--- END FILE ---\n" return content, estimate_tokens(content) # Check file size to prevent memory exhaustion stat_result = path.stat() file_size = stat_result.st_size logger.debug(f"[FILES] File size for {file_path}: {file_size:,} bytes") if file_size > max_size: logger.debug(f"[FILES] File too large: {file_path} ({file_size:,} > {max_size:,} bytes)") modified_at = datetime.fromtimestamp(stat_result.st_mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") content = ( f"\n--- FILE TOO LARGE: {file_path} (Last modified: {modified_at}) ---\n" f"File size: {file_size:,} bytes (max: {max_size:,})\n" "--- END FILE ---\n" ) return content, estimate_tokens(content) # Determine if we should add line numbers add_line_numbers = should_add_line_numbers(file_path, include_line_numbers) logger.debug(f"[FILES] Line numbers for {file_path}: {'enabled' if add_line_numbers else 'disabled'}") # Read the file with UTF-8 encoding, replacing invalid characters # This ensures we can handle files with mixed encodings logger.debug(f"[FILES] Reading file content for {file_path}") with open(path, encoding="utf-8", errors="replace") as f: file_content = f.read() logger.debug(f"[FILES] Successfully read {len(file_content)} characters from {file_path}") # Add line numbers if requested or auto-detected if add_line_numbers: file_content = _add_line_numbers(file_content) logger.debug(f"[FILES] Added line numbers to {file_path}") else: # Still normalize line endings for consistency file_content = _normalize_line_endings(file_content) # Format with clear delimiters that help the AI understand file boundaries # Using consistent markers makes it easier for the model to parse # NOTE: These markers ("--- BEGIN FILE: ... ---") are distinct from git diff markers # ("--- BEGIN DIFF: ... ---") to allow AI to distinguish between complete file content # vs. partial diff content when files appear in both sections modified_at = datetime.fromtimestamp(stat_result.st_mtime, tz=timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") formatted = ( f"\n--- BEGIN FILE: {file_path} (Last modified: {modified_at}) ---\n" f"{file_content}\n" f"--- END FILE: {file_path} ---\n" ) tokens = estimate_tokens(formatted) logger.debug(f"[FILES] Formatted content for {file_path}: {len(formatted)} chars, {tokens} tokens") return formatted, tokens except Exception as e: logger.debug(f"[FILES] Exception reading file {file_path}: {type(e).__name__}: {e}") content = f"\n--- ERROR READING FILE: {file_path} ---\nError: {str(e)}\n--- END FILE ---\n" tokens = estimate_tokens(content) logger.debug(f"[FILES] Returning error content for {file_path}: {tokens} tokens") return content, tokens def read_files( file_paths: list[str], code: Optional[str] = None, max_tokens: Optional[int] = None, reserve_tokens: int = 50_000, *, include_line_numbers: bool = False, ) -> str: """ Read multiple files and optional direct code with smart token management. This function implements intelligent token budgeting to maximize the amount of relevant content that can be included in an AI prompt while staying within token limits. It prioritizes direct code and reads files until the token budget is exhausted. Args: file_paths: List of file or directory paths (absolute paths required) code: Optional direct code to include (prioritized over files) max_tokens: Maximum tokens to use (defaults to DEFAULT_CONTEXT_WINDOW) reserve_tokens: Tokens to reserve for prompt and response (default 50K) include_line_numbers: Whether to add line numbers to file content Returns: str: All file contents formatted for AI consumption """ if max_tokens is None: max_tokens = DEFAULT_CONTEXT_WINDOW logger.debug(f"[FILES] read_files called with {len(file_paths)} paths") logger.debug( f"[FILES] Token budget: max={max_tokens:,}, reserve={reserve_tokens:,}, available={max_tokens - reserve_tokens:,}" ) content_parts = [] total_tokens = 0 available_tokens = max_tokens - reserve_tokens files_skipped = [] # Priority 1: Handle direct code if provided # Direct code is prioritized because it's explicitly provided by the user if code: formatted_code = f"\n--- BEGIN DIRECT CODE ---\n{code}\n--- END DIRECT CODE ---\n" code_tokens = estimate_tokens(formatted_code) if code_tokens <= available_tokens: content_parts.append(formatted_code) total_tokens += code_tokens available_tokens -= code_tokens # Priority 2: Process file paths if file_paths: # Expand directories to get all individual files logger.debug(f"[FILES] Expanding {len(file_paths)} file paths") all_files = expand_paths(file_paths) logger.debug(f"[FILES] After expansion: {len(all_files)} individual files") if not all_files and file_paths: # No files found but paths were provided logger.debug("[FILES] No files found from provided paths") content_parts.append(f"\n--- NO FILES FOUND ---\nProvided paths: {', '.join(file_paths)}\n--- END ---\n") else: # Read files sequentially until token limit is reached logger.debug(f"[FILES] Reading {len(all_files)} files with token budget {available_tokens:,}") for i, file_path in enumerate(all_files): if total_tokens >= available_tokens: logger.debug(f"[FILES] Token budget exhausted, skipping remaining {len(all_files) - i} files") files_skipped.extend(all_files[i:]) break file_content, file_tokens = read_file_content(file_path, include_line_numbers=include_line_numbers) logger.debug(f"[FILES] File {file_path}: {file_tokens:,} tokens") # Check if adding this file would exceed limit if total_tokens + file_tokens <= available_tokens: content_parts.append(file_content) total_tokens += file_tokens logger.debug(f"[FILES] Added file {file_path}, total tokens: {total_tokens:,}") else: # File too large for remaining budget logger.debug( f"[FILES] File {file_path} too large for remaining budget ({file_tokens:,} tokens, {available_tokens - total_tokens:,} remaining)" ) files_skipped.append(file_path) # Add informative note about skipped files to help users understand # what was omitted and why if files_skipped: logger.debug(f"[FILES] {len(files_skipped)} files skipped due to token limits") skip_note = "\n\n--- SKIPPED FILES (TOKEN LIMIT) ---\n" skip_note += f"Total skipped: {len(files_skipped)}\n" # Show first 10 skipped files as examples for _i, file_path in enumerate(files_skipped[:10]): skip_note += f" - {file_path}\n" if len(files_skipped) > 10: skip_note += f" ... and {len(files_skipped) - 10} more\n" skip_note += "--- END SKIPPED FILES ---\n" content_parts.append(skip_note) result = "\n\n".join(content_parts) if content_parts else "" logger.debug(f"[FILES] read_files complete: {len(result)} chars, {total_tokens:,} tokens used") return result def estimate_file_tokens(file_path: str) -> int: """ Estimate tokens for a file using file-type aware ratios. Args: file_path: Path to the file Returns: Estimated token count for the file """ try: if not os.path.exists(file_path) or not os.path.isfile(file_path): return 0 file_size = os.path.getsize(file_path) # Get the appropriate ratio for this file type from .file_types import get_token_estimation_ratio ratio = get_token_estimation_ratio(file_path) return int(file_size / ratio) except Exception: return 0 def check_files_size_limit(files: list[str], max_tokens: int, threshold_percent: float = 1.0) -> tuple[bool, int, int]: """ Check if a list of files would exceed token limits. Args: files: List of file paths to check max_tokens: Maximum allowed tokens threshold_percent: Percentage of max_tokens to use as threshold (0.0-1.0) Returns: Tuple of (within_limit, total_estimated_tokens, file_count) """ if not files: return True, 0, 0 total_estimated_tokens = 0 file_count = 0 threshold = int(max_tokens * threshold_percent) for file_path in files: try: estimated_tokens = estimate_file_tokens(file_path) total_estimated_tokens += estimated_tokens if estimated_tokens > 0: # Only count accessible files file_count += 1 except Exception: # Skip files that can't be accessed for size check continue within_limit = total_estimated_tokens <= threshold return within_limit, total_estimated_tokens, file_count def read_json_file(file_path: str) -> Optional[dict]: """ Read and parse a JSON file with proper error handling. Args: file_path: Path to the JSON file Returns: Parsed JSON data as dict, or None if file doesn't exist or invalid """ try: if not os.path.exists(file_path): return None with open(file_path, encoding="utf-8") as f: return json.load(f) except (json.JSONDecodeError, OSError): return None def write_json_file(file_path: str, data: dict, indent: int = 2) -> bool: """ Write data to a JSON file with proper formatting. Args: file_path: Path to write the JSON file data: Dictionary data to serialize indent: JSON indentation level Returns: True if successful, False otherwise """ try: os.makedirs(os.path.dirname(file_path), exist_ok=True) with open(file_path, "w", encoding="utf-8") as f: json.dump(data, f, indent=indent, ensure_ascii=False) return True except (OSError, TypeError): return False def get_file_size(file_path: str) -> int: """ Get file size in bytes with proper error handling. Args: file_path: Path to the file Returns: File size in bytes, or 0 if file doesn't exist or error """ try: if os.path.exists(file_path) and os.path.isfile(file_path): return os.path.getsize(file_path) return 0 except OSError: return 0 def ensure_directory_exists(file_path: str) -> bool: """ Ensure the parent directory of a file path exists. Args: file_path: Path to file (directory will be created for parent) Returns: True if directory exists or was created, False on error """ try: directory = os.path.dirname(file_path) if directory: os.makedirs(directory, exist_ok=True) return True except OSError: return False def is_text_file(file_path: str) -> bool: """ Check if a file is likely a text file based on extension and content. Args: file_path: Path to the file Returns: True if file appears to be text, False otherwise """ from .file_types import is_text_file as check_text_type return check_text_type(file_path) def read_file_safely(file_path: str, max_size: int = 10 * 1024 * 1024) -> Optional[str]: """ Read a file with size limits and encoding handling. Args: file_path: Path to the file max_size: Maximum file size in bytes (default 10MB) Returns: File content as string, or None if file too large or unreadable """ try: if not os.path.exists(file_path) or not os.path.isfile(file_path): return None file_size = os.path.getsize(file_path) if file_size > max_size: return None with open(file_path, encoding="utf-8", errors="ignore") as f: return f.read() except OSError: return None def check_total_file_size(files: list[str], model_name: str) -> Optional[dict]: """ Check if total file sizes would exceed token threshold before embedding. IMPORTANT: This performs STRICT REJECTION at MCP boundary. No partial inclusion - either all files fit or request is rejected. This forces the CLI to make better file selection decisions. This function MUST be called with the effective model name (after resolution). It should never receive 'auto' or None - model resolution happens earlier. Args: files: List of file paths to check model_name: The resolved model name for context-aware thresholds (required) Returns: Dict with `code_too_large` response if too large, None if acceptable """ if not files: return None # Validate we have a proper model name (not auto or None) if not model_name or model_name.lower() == "auto": raise ValueError( f"check_total_file_size called with unresolved model: '{model_name}'. " "Model must be resolved before file size checking." ) logger.info(f"File size check: Using model '{model_name}' for token limit calculation") from utils.model_context import ModelContext model_context = ModelContext(model_name) token_allocation = model_context.calculate_token_allocation() # Dynamic threshold based on model capacity context_window = token_allocation.total_tokens if context_window >= 1_000_000: # Gemini-class models threshold_percent = 0.8 # Can be more generous elif context_window >= 500_000: # Mid-range models threshold_percent = 0.7 # Moderate else: # OpenAI-class models (200K) threshold_percent = 0.6 # Conservative max_file_tokens = int(token_allocation.file_tokens * threshold_percent) # Use centralized file size checking (threshold already applied to max_file_tokens) within_limit, total_estimated_tokens, file_count = check_files_size_limit(files, max_file_tokens) if not within_limit: return { "status": "code_too_large", "content": ( f"The selected files are too large for analysis " f"(estimated {total_estimated_tokens:,} tokens, limit {max_file_tokens:,}). " f"Please select fewer, more specific files that are most relevant " f"to your question, then invoke the tool again." ), "content_type": "text", "metadata": { "total_estimated_tokens": total_estimated_tokens, "limit": max_file_tokens, "file_count": file_count, "threshold_percent": threshold_percent, "model_context_window": context_window, "model_name": model_name, "instructions": "Reduce file selection and try again - all files must fit within budget. If this persists, please use a model with a larger context window where available.", }, } return None # Proceed with ALL files ================================================ FILE: utils/image_utils.py ================================================ """Utility helpers for validating image inputs.""" import base64 import binascii import os from collections.abc import Iterable from utils.file_types import IMAGES, get_image_mime_type DEFAULT_MAX_IMAGE_SIZE_MB = 20.0 __all__ = ["DEFAULT_MAX_IMAGE_SIZE_MB", "validate_image"] def _valid_mime_types() -> Iterable[str]: """Return the MIME types permitted by the IMAGES whitelist.""" return (get_image_mime_type(ext) for ext in IMAGES) def validate_image(image_path: str, max_size_mb: float = None) -> tuple[bytes, str]: """Validate a user-supplied image path or data URL. Args: image_path: Either a filesystem path or a data URL. max_size_mb: Optional size limit (defaults to ``DEFAULT_MAX_IMAGE_SIZE_MB``). Returns: A tuple ``(image_bytes, mime_type)`` ready for upstream providers. Raises: ValueError: When the image is missing, malformed, or exceeds limits. """ if max_size_mb is None: max_size_mb = DEFAULT_MAX_IMAGE_SIZE_MB if image_path.startswith("data:"): return _validate_data_url(image_path, max_size_mb) return _validate_file_path(image_path, max_size_mb) def _validate_data_url(image_data_url: str, max_size_mb: float) -> tuple[bytes, str]: """Validate a data URL and return image bytes plus MIME type.""" try: header, data = image_data_url.split(",", 1) mime_type = header.split(";")[0].split(":")[1] except (ValueError, IndexError) as exc: raise ValueError(f"Invalid data URL format: {exc}") valid_mime_types = list(_valid_mime_types()) if mime_type not in valid_mime_types: raise ValueError( "Unsupported image type: {mime}. Supported types: {supported}".format( mime=mime_type, supported=", ".join(valid_mime_types) ) ) try: image_bytes = base64.b64decode(data) except binascii.Error as exc: raise ValueError(f"Invalid base64 data: {exc}") _validate_size(image_bytes, max_size_mb) return image_bytes, mime_type def _validate_file_path(file_path: str, max_size_mb: float) -> tuple[bytes, str]: """Validate an image loaded from the filesystem.""" try: with open(file_path, "rb") as handle: image_bytes = handle.read() except FileNotFoundError: raise ValueError(f"Image file not found: {file_path}") except OSError as exc: raise ValueError(f"Failed to read image file: {exc}") ext = os.path.splitext(file_path)[1].lower() if ext not in IMAGES: raise ValueError( "Unsupported image format: {ext}. Supported formats: {supported}".format( ext=ext, supported=", ".join(sorted(IMAGES)) ) ) mime_type = get_image_mime_type(ext) _validate_size(image_bytes, max_size_mb) return image_bytes, mime_type def _validate_size(image_bytes: bytes, max_size_mb: float) -> None: """Ensure the image does not exceed the configured size limit.""" size_mb = len(image_bytes) / (1024 * 1024) if size_mb > max_size_mb: raise ValueError(f"Image too large: {size_mb:.1f}MB (max: {max_size_mb}MB)") ================================================ FILE: utils/model_context.py ================================================ """ Model context management for dynamic token allocation. This module provides a clean abstraction for model-specific token management, ensuring that token limits are properly calculated based on the current model being used, not global constants. CONVERSATION MEMORY INTEGRATION: This module works closely with the conversation memory system to provide optimal token allocation for multi-turn conversations: 1. DUAL PRIORITIZATION STRATEGY SUPPORT: - Provides separate token budgets for conversation history vs. files - Enables the conversation memory system to apply newest-first prioritization - Ensures optimal balance between context preservation and new content 2. MODEL-SPECIFIC ALLOCATION: - Dynamic allocation based on model capabilities (context window size) - Conservative allocation for smaller models (O3: 200K context) - Generous allocation for larger models (Gemini: 1M+ context) - Adapts token distribution ratios based on model capacity 3. CROSS-TOOL CONSISTENCY: - Provides consistent token budgets across different tools - Enables seamless conversation continuation between tools - Supports conversation reconstruction with proper budget management """ import logging from dataclasses import dataclass from typing import Any, Optional from config import DEFAULT_MODEL from providers import ModelCapabilities, ModelProviderRegistry logger = logging.getLogger(__name__) @dataclass class TokenAllocation: """Token allocation strategy for a model.""" total_tokens: int content_tokens: int response_tokens: int file_tokens: int history_tokens: int @property def available_for_prompt(self) -> int: """Tokens available for the actual prompt after allocations.""" return self.content_tokens - self.file_tokens - self.history_tokens class ModelContext: """ Encapsulates model-specific information and token calculations. This class provides a single source of truth for all model-related token calculations, ensuring consistency across the system. """ def __init__(self, model_name: str, model_option: Optional[str] = None): self.model_name = model_name self.model_option = model_option # Store optional model option (e.g., "for", "against", etc.) self._provider = None self._capabilities = None self._token_allocation = None @property def provider(self): """Get the model provider lazily.""" if self._provider is None: self._provider = ModelProviderRegistry.get_provider_for_model(self.model_name) if not self._provider: available_models = ModelProviderRegistry.get_available_model_names() if available_models: available_text = ", ".join(available_models) else: available_text = ( "No models detected. Configure provider credentials or set DEFAULT_MODEL to a valid option." ) raise ValueError( f"Model '{self.model_name}' is not available with current API keys. Available models: {available_text}." ) return self._provider @property def capabilities(self) -> ModelCapabilities: """Get model capabilities lazily.""" if self._capabilities is None: self._capabilities = self.provider.get_capabilities(self.model_name) return self._capabilities def calculate_token_allocation(self, reserved_for_response: Optional[int] = None) -> TokenAllocation: """ Calculate token allocation based on model capacity and conversation requirements. This method implements the core token budget calculation that supports the dual prioritization strategy used in conversation memory and file processing: TOKEN ALLOCATION STRATEGY: 1. CONTENT vs RESPONSE SPLIT: - Smaller models (< 300K): 60% content, 40% response (conservative) - Larger models (≥ 300K): 80% content, 20% response (generous) 2. CONTENT SUB-ALLOCATION: - File tokens: 30-40% of content budget for newest file versions - History tokens: 40-50% of content budget for conversation context - Remaining: Available for tool-specific prompt content 3. CONVERSATION MEMORY INTEGRATION: - History allocation enables conversation reconstruction in reconstruct_thread_context() - File allocation supports newest-first file prioritization in tools - Remaining budget passed to tools via _remaining_tokens parameter Args: reserved_for_response: Override response token reservation Returns: TokenAllocation with calculated budgets for dual prioritization strategy """ total_tokens = self.capabilities.context_window # Dynamic allocation based on model capacity if total_tokens < 300_000: # Smaller context models (O3): Conservative allocation content_ratio = 0.6 # 60% for content response_ratio = 0.4 # 40% for response file_ratio = 0.3 # 30% of content for files history_ratio = 0.5 # 50% of content for history else: # Larger context models (Gemini): More generous allocation content_ratio = 0.8 # 80% for content response_ratio = 0.2 # 20% for response file_ratio = 0.4 # 40% of content for files history_ratio = 0.4 # 40% of content for history # Calculate allocations content_tokens = int(total_tokens * content_ratio) response_tokens = reserved_for_response or int(total_tokens * response_ratio) # Sub-allocations within content budget file_tokens = int(content_tokens * file_ratio) history_tokens = int(content_tokens * history_ratio) allocation = TokenAllocation( total_tokens=total_tokens, content_tokens=content_tokens, response_tokens=response_tokens, file_tokens=file_tokens, history_tokens=history_tokens, ) logger.debug(f"Token allocation for {self.model_name}:") logger.debug(f" Total: {allocation.total_tokens:,}") logger.debug(f" Content: {allocation.content_tokens:,} ({content_ratio:.0%})") logger.debug(f" Response: {allocation.response_tokens:,} ({response_ratio:.0%})") logger.debug(f" Files: {allocation.file_tokens:,} ({file_ratio:.0%} of content)") logger.debug(f" History: {allocation.history_tokens:,} ({history_ratio:.0%} of content)") return allocation def estimate_tokens(self, text: str) -> int: """ Estimate token count for text using model-specific tokenizer. For now, uses simple estimation. Can be enhanced with model-specific tokenizers (tiktoken for OpenAI, etc.) in the future. """ # TODO: Integrate model-specific tokenizers # For now, use conservative estimation return len(text) // 3 # Conservative estimate @classmethod def from_arguments(cls, arguments: dict[str, Any]) -> "ModelContext": """Create ModelContext from tool arguments.""" model_name = arguments.get("model") or DEFAULT_MODEL return cls(model_name) ================================================ FILE: utils/model_restrictions.py ================================================ """ Model Restriction Service This module provides centralized management of model usage restrictions based on environment variables. It allows organizations to limit which models can be used from each provider for cost control, compliance, or standardization purposes. Environment Variables: - OPENAI_ALLOWED_MODELS: Comma-separated list of allowed OpenAI models - GOOGLE_ALLOWED_MODELS: Comma-separated list of allowed Gemini models - XAI_ALLOWED_MODELS: Comma-separated list of allowed X.AI GROK models - OPENROUTER_ALLOWED_MODELS: Comma-separated list of allowed OpenRouter models - DIAL_ALLOWED_MODELS: Comma-separated list of allowed DIAL models Example: OPENAI_ALLOWED_MODELS=o3-mini,o4-mini GOOGLE_ALLOWED_MODELS=flash XAI_ALLOWED_MODELS=grok-4,grok-4.1-fast-reasoning OPENROUTER_ALLOWED_MODELS=opus,sonnet,mistral """ import logging from collections import defaultdict from typing import Optional from providers.shared import ProviderType from utils.env import get_env logger = logging.getLogger(__name__) class ModelRestrictionService: """Central authority for environment-driven model allowlists. Role Interpret ``*_ALLOWED_MODELS`` environment variables, keep their entries normalised (lowercase), and answer whether a provider/model pairing is permitted. Responsibilities * Parse, cache, and expose per-provider restriction sets * Validate configuration by cross-checking each entry against the provider’s alias-aware model list * Offer helper methods such as ``is_allowed`` and ``filter_models`` to enforce policy everywhere model names appear (tool selection, CLI commands, etc.). """ # Environment variable names ENV_VARS = { ProviderType.OPENAI: "OPENAI_ALLOWED_MODELS", ProviderType.GOOGLE: "GOOGLE_ALLOWED_MODELS", ProviderType.XAI: "XAI_ALLOWED_MODELS", ProviderType.OPENROUTER: "OPENROUTER_ALLOWED_MODELS", ProviderType.DIAL: "DIAL_ALLOWED_MODELS", } def __init__(self): """Initialize the restriction service by loading from environment.""" self.restrictions: dict[ProviderType, set[str]] = {} self._alias_resolution_cache: dict[ProviderType, dict[str, str]] = defaultdict(dict) self._load_from_env() def _load_from_env(self) -> None: """Load restrictions from environment variables.""" for provider_type, env_var in self.ENV_VARS.items(): env_value = get_env(env_var) if env_value is None or env_value == "": # Not set or empty - no restrictions (allow all models) logger.debug(f"{env_var} not set or empty - all {provider_type.value} models allowed") continue # Parse comma-separated list models = set() for model in env_value.split(","): cleaned = model.strip().lower() if cleaned: models.add(cleaned) if models: self.restrictions[provider_type] = models self._alias_resolution_cache[provider_type] = {} logger.info(f"{provider_type.value} allowed models: {sorted(models)}") else: # All entries were empty after cleaning - treat as no restrictions logger.debug(f"{env_var} contains only whitespace - all {provider_type.value} models allowed") def validate_against_known_models(self, provider_instances: dict[ProviderType, any]) -> None: """ Validate restrictions against known models from providers. This should be called after providers are initialized to warn about typos or invalid model names in the restriction lists. Args: provider_instances: Dictionary of provider type to provider instance """ for provider_type, allowed_models in self.restrictions.items(): provider = provider_instances.get(provider_type) if not provider: continue # Get all supported models using the clean polymorphic interface try: # Gather canonical models and aliases with consistent formatting all_models = provider.list_models( respect_restrictions=False, include_aliases=True, lowercase=True, unique=True, ) supported_models = set(all_models) except Exception as e: logger.debug(f"Could not get model list from {provider_type.value} provider: {e}") supported_models = set() # Check each allowed model for allowed_model in allowed_models: if allowed_model not in supported_models: logger.warning( f"Model '{allowed_model}' in {self.ENV_VARS[provider_type]} " f"is not a recognized {provider_type.value} model. " f"Please check for typos. Known models: {sorted(supported_models)}" ) def is_allowed(self, provider_type: ProviderType, model_name: str, original_name: Optional[str] = None) -> bool: """ Check if a model is allowed for a specific provider. Args: provider_type: The provider type (OPENAI, GOOGLE, etc.) model_name: The canonical model name (after alias resolution) original_name: The original model name before alias resolution (optional) Returns: True if allowed (or no restrictions), False if restricted """ if provider_type not in self.restrictions: # No restrictions for this provider return True allowed_set = self.restrictions[provider_type] if len(allowed_set) == 0: # Empty set - allowed return True # Check both the resolved name and original name (if different) names_to_check = {model_name.lower()} if original_name and original_name.lower() != model_name.lower(): names_to_check.add(original_name.lower()) # If any of the names is in the allowed set, it's allowed if any(name in allowed_set for name in names_to_check): return True # Attempt to resolve canonical names for allowed aliases using provider metadata. try: from providers.registry import ModelProviderRegistry provider = ModelProviderRegistry.get_provider(provider_type) except Exception: # pragma: no cover - registry lookup failure shouldn't break validation provider = None if provider: cache = self._alias_resolution_cache.setdefault(provider_type, {}) for allowed_entry in list(allowed_set): normalized_resolved = cache.get(allowed_entry) if not normalized_resolved: try: resolved = provider._resolve_model_name(allowed_entry) except Exception: # pragma: no cover - resolution failures are treated as non-matches continue if not resolved: continue normalized_resolved = resolved.lower() cache[allowed_entry] = normalized_resolved if normalized_resolved in names_to_check: allowed_set.add(normalized_resolved) cache[normalized_resolved] = normalized_resolved return True return False def get_allowed_models(self, provider_type: ProviderType) -> Optional[set[str]]: """ Get the set of allowed models for a provider. Args: provider_type: The provider type Returns: Set of allowed model names, or None if no restrictions """ return self.restrictions.get(provider_type) def has_restrictions(self, provider_type: ProviderType) -> bool: """ Check if a provider has any restrictions. Args: provider_type: The provider type Returns: True if restrictions exist, False otherwise """ return provider_type in self.restrictions def filter_models(self, provider_type: ProviderType, models: list[str]) -> list[str]: """ Filter a list of models based on restrictions. Args: provider_type: The provider type models: List of model names to filter Returns: Filtered list containing only allowed models """ if not self.has_restrictions(provider_type): return models return [m for m in models if self.is_allowed(provider_type, m)] def get_restriction_summary(self) -> dict[str, any]: """ Get a summary of all restrictions for logging/debugging. Returns: Dictionary with provider names and their restrictions """ summary = {} for provider_type, allowed_set in self.restrictions.items(): if allowed_set: summary[provider_type.value] = sorted(allowed_set) else: summary[provider_type.value] = "none (provider disabled)" return summary # Global instance (singleton pattern) _restriction_service: Optional[ModelRestrictionService] = None def get_restriction_service() -> ModelRestrictionService: """ Get the global restriction service instance. Returns: The singleton ModelRestrictionService instance """ global _restriction_service if _restriction_service is None: _restriction_service = ModelRestrictionService() return _restriction_service ================================================ FILE: utils/security_config.py ================================================ """ Security configuration and path validation constants This module contains security-related constants and configurations for file access control. """ from pathlib import Path # Dangerous system paths - block these AND all their subdirectories # These are system directories where user code should never reside DANGEROUS_SYSTEM_PATHS = { "/", "/etc", "/usr", "/bin", "/var", "/root", "C:\\Windows", "C:\\Program Files", } # User home container paths - block ONLY the exact path, not subdirectories # Subdirectory access (e.g., /home/user/project) is controlled by is_home_directory_root() # This allows users to work in their home subdirectories while blocking overly broad access DANGEROUS_HOME_CONTAINERS = { "/home", "C:\\Users", } # Combined set for backward compatibility DANGEROUS_PATHS = DANGEROUS_SYSTEM_PATHS | DANGEROUS_HOME_CONTAINERS # Directories to exclude from recursive file search # These typically contain generated code, dependencies, or build artifacts EXCLUDED_DIRS = { # Python "__pycache__", ".venv", "venv", "env", ".env", "*.egg-info", ".eggs", "wheels", ".Python", ".mypy_cache", ".pytest_cache", ".tox", "htmlcov", ".coverage", "coverage", # Node.js / JavaScript "node_modules", ".next", ".nuxt", "bower_components", ".sass-cache", # Version Control ".git", ".svn", ".hg", # Build Output "build", "dist", "target", "out", # IDEs ".idea", ".vscode", ".sublime", ".atom", ".brackets", # Temporary / Cache ".cache", ".temp", ".tmp", "*.swp", "*.swo", "*~", # OS-specific ".DS_Store", "Thumbs.db", # Java / JVM ".gradle", ".m2", # Documentation build "_build", "site", # Mobile development ".expo", ".flutter", # Package managers "vendor", } def is_dangerous_path(path: Path) -> bool: """ Check if a path is in or under a dangerous directory. This function handles two categories of dangerous paths differently: 1. System paths (DANGEROUS_SYSTEM_PATHS): Block the path AND all subdirectories. Example: /etc is dangerous, so /etc/passwd is also blocked. 2. Home containers (DANGEROUS_HOME_CONTAINERS): Block ONLY the exact path. Example: /home is blocked, but /home/user/project is allowed. Subdirectory access control is delegated to is_home_directory_root(). Args: path: Path to check Returns: True if the path is dangerous and should not be accessed Security: Fixes path traversal vulnerability (CWE-22) while preserving user access to home subdirectories. """ try: resolved = path.resolve() def _dangerous_variants(p: Path) -> set[Path]: variants = {p} # Only resolve paths that are absolute on the current platform. # This avoids turning Windows-style strings into nonsense absolute paths on POSIX. if p.is_absolute(): try: variants.add(p.resolve()) except Exception: pass return variants # Check 1: Root directory (filesystem root) if resolved.parent == resolved: return True # Check 2: System paths - block exact match AND all subdirectories for dangerous in DANGEROUS_SYSTEM_PATHS: # Skip root "/" - already handled above if dangerous == "/": continue for dangerous_path in _dangerous_variants(Path(dangerous)): # is_relative_to() correctly handles both exact matches and subdirectories. # Resolving the dangerous base path also handles platform symlinks # (e.g., macOS /etc -> /private/etc, /var -> /private/var). if resolved == dangerous_path or resolved.is_relative_to(dangerous_path): return True # Check 3: Home containers - block ONLY exact match # Subdirectories like /home/user/project should pass through here # and be handled by is_home_directory_root() in resolve_and_validate_path() for container in DANGEROUS_HOME_CONTAINERS: for container_path in _dangerous_variants(Path(container)): if resolved == container_path: return True return False except Exception: return True # If we can't resolve, consider it dangerous ================================================ FILE: utils/storage_backend.py ================================================ """ In-memory storage backend for conversation threads This module provides a thread-safe, in-memory alternative to Redis for storing conversation contexts. It's designed for ephemeral MCP server sessions where conversations only need to persist during a single Claude session. ⚠️ PROCESS-SPECIFIC STORAGE: This storage is confined to a single Python process. Data stored in one process is NOT accessible from other processes or subprocesses. This is why simulator tests that run server.py as separate subprocesses cannot share conversation state between tool calls. Key Features: - Thread-safe operations using locks - TTL support with automatic expiration - Background cleanup thread for memory management - Singleton pattern for consistent state within a single process - Drop-in replacement for Redis storage (for single-process scenarios) """ import logging import threading import time from typing import Optional from utils.env import get_env logger = logging.getLogger(__name__) class InMemoryStorage: """Thread-safe in-memory storage for conversation threads""" def __init__(self): self._store: dict[str, tuple[str, float]] = {} self._lock = threading.Lock() # Match Redis behavior: cleanup interval based on conversation timeout # Run cleanup at 1/10th of timeout interval (e.g., 18 mins for 3 hour timeout) timeout_hours = int(get_env("CONVERSATION_TIMEOUT_HOURS", "3") or "3") self._cleanup_interval = (timeout_hours * 3600) // 10 self._cleanup_interval = max(300, self._cleanup_interval) # Minimum 5 minutes self._shutdown = False # Start background cleanup thread self._cleanup_thread = threading.Thread(target=self._cleanup_worker, daemon=True) self._cleanup_thread.start() logger.info( f"In-memory storage initialized with {timeout_hours}h timeout, cleanup every {self._cleanup_interval//60}m" ) def set_with_ttl(self, key: str, ttl_seconds: int, value: str) -> None: """Store value with expiration time""" with self._lock: expires_at = time.time() + ttl_seconds self._store[key] = (value, expires_at) logger.debug(f"Stored key {key} with TTL {ttl_seconds}s") def get(self, key: str) -> Optional[str]: """Retrieve value if not expired""" with self._lock: if key in self._store: value, expires_at = self._store[key] if time.time() < expires_at: logger.debug(f"Retrieved key {key}") return value else: # Clean up expired entry del self._store[key] logger.debug(f"Key {key} expired and removed") return None def setex(self, key: str, ttl_seconds: int, value: str) -> None: """Redis-compatible setex method""" self.set_with_ttl(key, ttl_seconds, value) def _cleanup_worker(self): """Background thread that periodically cleans up expired entries""" while not self._shutdown: time.sleep(self._cleanup_interval) self._cleanup_expired() def _cleanup_expired(self): """Remove all expired entries""" with self._lock: current_time = time.time() expired_keys = [k for k, (_, exp) in self._store.items() if exp < current_time] for key in expired_keys: del self._store[key] if expired_keys: logger.debug(f"Cleaned up {len(expired_keys)} expired conversation threads") def shutdown(self): """Graceful shutdown of background thread""" self._shutdown = True if self._cleanup_thread.is_alive(): self._cleanup_thread.join(timeout=1) # Global singleton instance _storage_instance = None _storage_lock = threading.Lock() def get_storage_backend() -> InMemoryStorage: """Get the global storage instance (singleton pattern)""" global _storage_instance if _storage_instance is None: with _storage_lock: if _storage_instance is None: _storage_instance = InMemoryStorage() logger.info("Initialized in-memory conversation storage") return _storage_instance ================================================ FILE: utils/token_utils.py ================================================ """ Token counting utilities for managing API context limits This module provides functions for estimating token counts to ensure requests stay within the Gemini API's context window limits. Note: The estimation uses a simple character-to-token ratio which is approximate. For production systems requiring precise token counts, consider using the actual tokenizer for the specific model. """ # Default fallback for token limit (conservative estimate) DEFAULT_CONTEXT_WINDOW = 200_000 # Conservative fallback for unknown models def estimate_tokens(text: str) -> int: """ Estimate token count using a character-based approximation. This uses a rough heuristic where 1 token ≈ 4 characters, which is a reasonable approximation for English text. The actual token count may vary based on: - Language (non-English text may have different ratios) - Code vs prose (code often has more tokens per character) - Special characters and formatting Args: text: The text to estimate tokens for Returns: int: Estimated number of tokens """ return len(text) // 4 def check_token_limit(text: str, context_window: int = DEFAULT_CONTEXT_WINDOW) -> tuple[bool, int]: """ Check if text exceeds the specified token limit. This function is used to validate that prepared prompts will fit within the model's context window, preventing API errors and ensuring reliable operation. Args: text: The text to check context_window: The model's context window size (defaults to conservative fallback) Returns: Tuple[bool, int]: (is_within_limit, estimated_tokens) - is_within_limit: True if the text fits within context_window - estimated_tokens: The estimated token count """ estimated = estimate_tokens(text) return estimated <= context_window, estimated