diff --git a/.env.example b/.env.example index 85581e6..2087089 100644 --- a/.env.example +++ b/.env.example @@ -35,6 +35,29 @@ AJARBOT_SLACK_APP_TOKEN=xapp-your-app-token # Get token from: https://t.me/BotFather AJARBOT_TELEGRAM_BOT_TOKEN=123456:ABC-your-bot-token +# ======================================== +# Obsidian MCP Integration (Optional) +# ======================================== +# Obsidian MCP server provides advanced vault operations via Obsidian REST API +# See: OBSIDIAN_MCP_INTEGRATION.md for setup instructions + +# Enable/disable Obsidian MCP integration +OBSIDIAN_MCP_ENABLED=false + +# Obsidian Local REST API Key +# Install "Local REST API" plugin in Obsidian first, then generate key in settings +OBSIDIAN_API_KEY=your-obsidian-api-key-here + +# Obsidian REST API endpoint (default: http://127.0.0.1:27123) +OBSIDIAN_BASE_URL=http://127.0.0.1:27123 + +# Path to your main Obsidian vault (overrides config/obsidian_mcp.yaml) +# OBSIDIAN_VAULT_PATH=C:/Users/YourName/Documents/MyVault + +# Tool routing strategy (optional, overrides config/obsidian_mcp.yaml) +# Options: obsidian_preferred, custom_preferred, obsidian_only +# OBSIDIAN_ROUTING_STRATEGY=obsidian_preferred + # ======================================== # Alternative LLM Providers (Optional) # ======================================== diff --git a/.gitignore b/.gitignore index 3cadc1c..0d6e9b2 100644 --- a/.gitignore +++ b/.gitignore @@ -42,6 +42,8 @@ Thumbs.db *.local.json .env .env.local +scripts/proxmox_ssh.sh # Contains Proxmox root password (legacy) +scripts/proxmox_ssh.py # Contains Proxmox root password (paramiko) config/scheduled_tasks.yaml # Use scheduled_tasks.example.yaml instead # Memory workspace (optional - remove if you want to version control) @@ -63,6 +65,12 @@ usage_data.json config/google_credentials.yaml config/google_oauth_token.json +# Obsidian MCP config (contains vault path - use obsidian_mcp.example.yaml) +config/obsidian_mcp.yaml + +# Gitea config (contains access token - use gitea_config.example.yaml) +config/gitea_config.yaml + # Logs *.log logs/ diff --git a/AGENT_SDK_IMPLEMENTATION.md b/AGENT_SDK_IMPLEMENTATION.md deleted file mode 100644 index 0838cfd..0000000 --- a/AGENT_SDK_IMPLEMENTATION.md +++ /dev/null @@ -1,253 +0,0 @@ -# Claude Agent SDK Implementation - -## Overview - -This implementation integrates the Claude Agent SDK as the **default backend** for the Ajarbot LLM interface, replacing the previous pay-per-token API model with Claude Pro subscription-based access. - -## Architecture - -### Strategy: Thin Wrapper (Strategy A) - -The Agent SDK is implemented as a **pure LLM backend replacement**: -- SDK handles only the LLM communication layer -- Existing `agent.py` tool execution loop remains unchanged -- All 17 tools (file operations, Gmail, Calendar, etc.) work identically -- Zero changes required to `agent.py`, `tools.py`, or adapters - -### Three Modes of Operation - -The system now supports three modes (in priority order): - -1. **Agent SDK Mode** (DEFAULT) - - Uses Claude Pro subscription - - No API token costs - - Enabled by default when `claude-agent-sdk` is installed - - Set `USE_AGENT_SDK=true` (default) - -2. **Direct API Mode** - - Pay-per-token using Anthropic API - - Requires `ANTHROPIC_API_KEY` - - Enable with `USE_DIRECT_API=true` - -3. **Legacy Server Mode** (Deprecated) - - Uses local FastAPI server wrapper - - Enable with `USE_CLAUDE_CODE_SERVER=true` - -## Key Implementation Details - -### 1. Async/Sync Bridge - -The Agent SDK is async-native, but the bot uses synchronous interfaces. We bridge them using `anyio.from_thread.run()`: - -```python -# Synchronous chat_with_tools() calls async _agent_sdk_chat_with_tools() -response = anyio.from_thread.run( - self._agent_sdk_chat_with_tools, - messages, - tools, - system, - max_tokens -) -``` - -### 2. Response Format Conversion - -Agent SDK responses are converted to `anthropic.types.Message` format for compatibility: - -```python -def _convert_sdk_response_to_message(self, sdk_response: Dict[str, Any]) -> Message: - """Convert Agent SDK response to anthropic.types.Message format.""" - # Extracts: - # - TextBlock for text content - # - ToolUseBlock for tool_use blocks - # - Usage information - # Returns MessageLike object compatible with agent.py -``` - -### 3. Backward Compatibility - -All existing environment variables work: -- `ANTHROPIC_API_KEY` - Still used for Direct API mode -- `USE_CLAUDE_CODE_SERVER` - Legacy mode still supported -- `CLAUDE_CODE_SERVER_URL` - Legacy server URL - -New variables: -- `USE_AGENT_SDK=true` - Enable Agent SDK (default) -- `USE_DIRECT_API=true` - Force Direct API mode - -## Installation - -### Step 1: Install Dependencies - -```bash -cd c:\Users\fam1n\projects\ajarbot -pip install -r requirements.txt -``` - -This installs: -- `claude-agent-sdk>=0.1.0` - Agent SDK -- `anyio>=4.0.0` - Async/sync bridging - -### Step 2: Configure Mode (Optional) - -Agent SDK is the default. To use a different mode: - -**For Direct API (pay-per-token):** -```bash -# Add to .env -USE_DIRECT_API=true -ANTHROPIC_API_KEY=sk-ant-... -``` - -**For Legacy Server:** -```bash -# Add to .env -USE_CLAUDE_CODE_SERVER=true -CLAUDE_CODE_SERVER_URL=http://localhost:8000 -``` - -### Step 3: Run the Bot - -```bash -python bot_runner.py -``` - -You should see: -``` -[LLM] Using Claude Agent SDK (Pro subscription) -``` - -## Files Modified - -### 1. `requirements.txt` -- Replaced `claude-code-sdk` with `claude-agent-sdk` -- Added `anyio>=4.0.0` for async bridging -- Removed FastAPI/Uvicorn (no longer needed for default mode) - -### 2. `llm_interface.py` -Major refactoring: -- Added Agent SDK import and availability check -- New mode selection logic (agent_sdk > legacy_server > direct_api) -- `_agent_sdk_chat()` - Async method for simple chat -- `_agent_sdk_chat_with_tools()` - Async method for tool chat -- `_convert_sdk_response_to_message()` - Response format converter -- Updated `chat()` and `chat_with_tools()` with Agent SDK support - -**Lines of code:** -- Before: ~250 lines -- After: ~410 lines -- Added: ~160 lines for Agent SDK support - -## Testing Checklist - -### Basic Functionality -- [ ] Bot starts successfully with Agent SDK -- [ ] Simple chat works (`agent.chat("Hello", "user")`) -- [ ] Tool execution works (file operations, Gmail, Calendar) -- [ ] Multiple tool calls in sequence work -- [ ] Error handling works (invalid requests, SDK failures) - -### Mode Switching -- [ ] Agent SDK mode works (default) -- [ ] Direct API mode works (`USE_DIRECT_API=true`) -- [ ] Legacy server mode works (`USE_CLAUDE_CODE_SERVER=true`) -- [ ] Fallback to Direct API when SDK unavailable - -### Compatibility -- [ ] All 17 tools work identically -- [ ] Scheduled tasks work -- [ ] Telegram adapter works -- [ ] Slack adapter works -- [ ] Memory system works -- [ ] Self-healing system works - -### Response Format -- [ ] `.content` attribute accessible -- [ ] `.stop_reason` attribute correct -- [ ] `.usage` attribute present -- [ ] TextBlock extraction works -- [ ] ToolUseBlock extraction works - -## Troubleshooting - -### Issue: "Agent SDK not available, falling back to Direct API" - -**Solution:** Install the SDK: -```bash -pip install claude-agent-sdk -``` - -### Issue: SDK import fails - -**Check:** -1. Is `claude-agent-sdk` installed? (`pip list | grep claude-agent-sdk`) -2. Is virtual environment activated? -3. Are there any import errors in the SDK itself? - -### Issue: Response format incompatible with agent.py - -**Check:** -- `MessageLike` class has all required attributes (`.content`, `.stop_reason`, `.usage`) -- `TextBlock` and `ToolUseBlock` are properly constructed -- `sdk_response` structure matches expected format - -### Issue: Async/sync bridge errors - -**Check:** -- `anyio` is installed (`pip list | grep anyio`) -- Thread context is available (not running in async context already) -- No event loop conflicts - -## Performance Considerations - -### Token Costs -- **Agent SDK**: $0 (uses Pro subscription) -- **Direct API**: ~$0.25-$1.25 per 1M tokens (Haiku), ~$3-$15 per 1M tokens (Sonnet) - -### Speed -- **Agent SDK**: Similar to Direct API -- **Direct API**: Baseline -- **Legacy Server**: Additional HTTP overhead - -### Memory -- **Agent SDK**: ~50MB overhead for SDK client -- **Direct API**: Minimal overhead -- **Legacy Server**: Requires separate server process - -## Future Enhancements - -### Potential Improvements -1. **Streaming Support**: Implement streaming responses via SDK -2. **Better Error Messages**: More detailed SDK error propagation -3. **Usage Tracking**: Track SDK usage separately (if SDK provides metrics) -4. **Caching**: Implement prompt caching for Agent SDK (if supported) -5. **Batch Requests**: Support batch processing via SDK - -### Migration Path -1. Phase 1: Agent SDK as default (DONE) -2. Phase 2: Remove legacy server code (after testing period) -3. Phase 3: Deprecate Direct API mode (after SDK proven stable) -4. Phase 4: SDK-only implementation - -## Version History - -### v1.0.0 (2026-02-15) -- Initial Agent SDK implementation -- Three-mode architecture (agent_sdk, direct_api, legacy_server) -- Async/sync bridge using anyio -- Response format converter -- Backward compatibility with existing env vars -- All 17 tools preserved -- Zero changes to agent.py, tools.py, adapters - -## References - -- **Agent SDK Docs**: (TBD - add when available) -- **Anthropic API Docs**: https://docs.anthropic.com/ -- **anyio Docs**: https://anyio.readthedocs.io/ - -## Credits - -- **Implementation**: Strategy A (Thin Wrapper) -- **Planning**: Based on planning agent recommendations -- **Architecture**: Minimal disruption, maximum compatibility diff --git a/HYBRID_SEARCH_SUMMARY.md b/HYBRID_SEARCH_SUMMARY.md deleted file mode 100644 index 8c5956e..0000000 --- a/HYBRID_SEARCH_SUMMARY.md +++ /dev/null @@ -1,151 +0,0 @@ -# Hybrid Search Implementation Summary - -## What Was Implemented - -Successfully upgraded Ajarbot's memory system from keyword-only search to **hybrid semantic + keyword search**. - -## Technical Details - -### Stack -- **FastEmbed** (sentence-transformers/all-MiniLM-L6-v2) - 384-dimensional embeddings -- **usearch** - Fast vector similarity search -- **SQLite FTS5** - Keyword/BM25 search (retained) - -### Scoring Algorithm -- **0.7 weight** - Vector similarity (semantic understanding) -- **0.3 weight** - BM25 score (keyword matching) -- Combined and normalized for optimal results - -### Performance -- **Query time**: ~15ms average (was 5ms keyword-only) -- **Storage overhead**: +1.5KB per memory chunk -- **Cost**: $0 (runs locally, no API calls) -- **Embeddings generated**: 59 for existing memories - -## Files Modified - -1. **memory_system.py** - - Added FastEmbed and usearch imports - - Initialize embedding model in `__init__` (line ~88) - - Added `_generate_embedding()` method - - Modified `index_file()` to generate and store embeddings - - Implemented `search_hybrid()` method - - Added database migration for `vector_id` column - - Save vector index on `close()` - -2. **agent.py** - - Line 71: Changed `search()` to `search_hybrid()` - -3. **memory_workspace/MEMORY.md** - - Updated Core Stack section - - Changed "Planned (Phase 2)" to "IMPLEMENTED" - - Added Recent Changes entry - - Updated Architecture Decisions - -## Results - Before vs After - -### Example Query: "How do I reduce costs?" - -**Keyword Search (old)**: -``` -No results found! -``` - -**Hybrid Search (new)**: -``` -1. MEMORY.md:28 (score: 0.228) - ## Cost Optimizations (2026-02-13) - Target: Minimize API costs... - -2. SOUL.md:45 (score: 0.213) - Be proactive and use tools... -``` - -### Example Query: "when was I born" - -**Keyword Search (old)**: -``` -No results found! -``` - -**Hybrid Search (new)**: -``` -1. SOUL.md:1 (score: 0.071) - # SOUL - Agent Identity... - -2. MEMORY.md:49 (score: 0.060) - ## Search Evolution... -``` - -## How It Works Automatically - -The bot now automatically uses hybrid search on **every chat message**: - -1. User sends message to bot -2. `agent.py` calls `memory.search_hybrid(user_message, max_results=2)` -3. System generates embedding for query (~10ms) -4. Searches vector index for semantic matches -5. Searches FTS5 for keyword matches -6. Combines scores (70% semantic, 30% keyword) -7. Returns top 2 results -8. Results injected into LLM context automatically - -**No user action needed** - it's completely transparent! - -## Dependencies Added - -```bash -pip install fastembed usearch -``` - -Installs: -- fastembed (0.7.4) -- usearch (2.23.0) -- numpy (2.4.2) -- onnxruntime (1.24.1) -- Plus supporting libraries - -## Files Created - -- `memory_workspace/vectors.usearch` - Vector index (~90KB for 59 vectors) -- `test_hybrid_search.py` - Test script -- `test_agent_hybrid.py` - Agent integration test -- `demo_hybrid_comparison.py` - Comparison demo - -## Memory Impact - -- **FastEmbed model**: ~50MB RAM (loaded once, persists) -- **Vector index**: ~1.5KB per memory chunk -- **59 memories**: ~90KB total vector storage - -## Benefits - -1. **10x better semantic recall** - Finds memories by meaning, not just keywords -2. **Natural language queries** - "How do I save money?" finds cost optimization -3. **Zero cost** - No API calls, runs entirely locally -4. **Fast** - Sub-20ms queries -5. **Automatic** - Works transparently in all bot interactions -6. **Maintains keyword power** - Still finds exact technical terms - -## Next Steps (Optional Future Enhancements) - -- Add `search_user_hybrid()` for per-user semantic search -- Tune weights (currently 0.7/0.3) based on query patterns -- Add query expansion for better recall -- Pre-compute common query embeddings for speed - -## Verification - -Run comparison test: -```bash -python demo_hybrid_comparison.py -``` - -Output shows keyword search finding 0 results, hybrid finding relevant matches for all queries. - ---- - -**Implementation Status**: ✅ COMPLETE -**Date**: 2026-02-13 -**Lines of Code**: ~150 added to memory_system.py -**Breaking Changes**: None (backward compatible) diff --git a/IMPLEMENTATION_SUMMARY.md b/IMPLEMENTATION_SUMMARY.md deleted file mode 100644 index efadaa0..0000000 --- a/IMPLEMENTATION_SUMMARY.md +++ /dev/null @@ -1,297 +0,0 @@ -# Claude Agent SDK Implementation - Summary - -## Status: ✅ COMPLETE - -The Claude Agent SDK backend has been successfully implemented in `llm_interface.py` following the "Strategy A - Thin Wrapper" approach from the planning phase. - -## Implementation Overview - -### Files Modified - -1. **`llm_interface.py`** (408 lines, +160 lines) - - Added Agent SDK import and availability check - - Implemented three-mode architecture (agent_sdk, direct_api, legacy_server) - - Added async/sync bridge using `anyio.from_thread.run()` - - Implemented SDK response to Message format converter - - Preserved all existing functionality - -2. **`requirements.txt`** (31 lines) - - Replaced `claude-code-sdk` with `claude-agent-sdk>=0.1.0` - - Added `anyio>=4.0.0` for async/sync bridging - - Removed FastAPI/Uvicorn (no longer needed for default mode) - -### Files Created - -3. **`AGENT_SDK_IMPLEMENTATION.md`** (Documentation) - - Architecture overview - - Installation instructions - - Testing checklist - - Troubleshooting guide - - Performance considerations - -4. **`MIGRATION_GUIDE_AGENT_SDK.md`** (User guide) - - Step-by-step migration instructions - - Environment variable reference - - Troubleshooting common issues - - Rollback plan - - FAQ section - -5. **`test_agent_sdk.py`** (Test suite) - - 5 comprehensive tests - - Mode selection verification - - Response format compatibility - - Simple chat and tool chat tests - - Initialization tests - -6. **`IMPLEMENTATION_SUMMARY.md`** (This file) - - Quick reference summary - - Key features - - Verification steps - -## Key Features Implemented - -### ✅ Three-Mode Architecture - -**Agent SDK Mode (DEFAULT)** -- Uses Claude Pro subscription (zero API costs) -- Automatic async/sync bridging -- Full tool support (all 17 tools) -- Enabled by default when SDK installed - -**Direct API Mode** -- Pay-per-token using Anthropic API -- Usage tracking enabled -- Prompt caching support -- Fallback when SDK unavailable - -**Legacy Server Mode (Deprecated)** -- Backward compatible with old setup -- Still functional but not recommended - -### ✅ Async/Sync Bridge - -```python -# Synchronous interface calls async SDK methods -response = anyio.from_thread.run( - self._agent_sdk_chat_with_tools, - messages, tools, system, max_tokens -) -``` - -### ✅ Response Format Conversion - -Converts Agent SDK responses to `anthropic.types.Message` format: -- TextBlock for text content -- ToolUseBlock for tool calls -- Usage information -- Full compatibility with agent.py - -### ✅ Backward Compatibility - -All existing features preserved: -- Environment variables (ANTHROPIC_API_KEY, etc.) -- Usage tracking (for Direct API mode) -- Model switching (/sonnet, /haiku commands) -- Prompt caching (for Direct API mode) -- All 17 tools (file ops, Gmail, Calendar, etc.) -- Scheduled tasks -- Memory system -- Self-healing system -- All adapters (Telegram, Slack, etc.) - -### ✅ Zero Changes Required - -No modifications needed to: -- `agent.py` - Tool execution loop unchanged -- `tools.py` - All 17 tools work identically -- `adapters/` - Telegram, Slack adapters unchanged -- `memory_system.py` - Memory system unchanged -- `self_healing.py` - Self-healing unchanged -- `scheduled_tasks.py` - Scheduler unchanged - -## Mode Selection Logic - -``` -Priority Order: -1. USE_DIRECT_API=true → Direct API mode -2. USE_CLAUDE_CODE_SERVER=true → Legacy server mode -3. USE_AGENT_SDK=true (default) → Agent SDK mode -4. Agent SDK unavailable → Fallback to Direct API -``` - -## Environment Variables - -### New Variables -- `USE_AGENT_SDK=true` (default) - Enable Agent SDK -- `USE_DIRECT_API=true` - Force Direct API mode - -### Preserved Variables -- `ANTHROPIC_API_KEY` - For Direct API mode -- `USE_CLAUDE_CODE_SERVER` - For legacy server mode -- `CLAUDE_CODE_SERVER_URL` - Legacy server URL - -## Code Statistics - -### Lines of Code Added -- `llm_interface.py`: ~160 lines -- `test_agent_sdk.py`: ~450 lines -- Documentation: ~800 lines -- **Total: ~1,410 lines** - -### Test Coverage -- 5 automated tests -- All test scenarios pass -- Response format validated -- Mode selection verified - -## Installation & Usage - -### Quick Start -```bash -# 1. Install dependencies -pip install -r requirements.txt - -# 2. Run the bot (Agent SDK is default) -python bot_runner.py - -# Expected output: -# [LLM] Using Claude Agent SDK (Pro subscription) -``` - -### Run Tests -```bash -python test_agent_sdk.py - -# Expected: 5/5 tests pass -``` - -## Verification Checklist - -### ✅ Implementation -- [x] Agent SDK backend class implemented -- [x] Async/sync bridge using anyio -- [x] Response format converter -- [x] Three-mode architecture -- [x] Backward compatibility maintained -- [x] Usage tracking preserved (for Direct API) -- [x] Error handling implemented - -### ✅ Testing -- [x] Initialization test -- [x] Simple chat test -- [x] Chat with tools test -- [x] Response format test -- [x] Mode selection test - -### ✅ Documentation -- [x] Implementation guide created -- [x] Migration guide created -- [x] Test suite created -- [x] Inline code comments -- [x] Summary document created - -### ✅ Compatibility -- [x] agent.py unchanged -- [x] tools.py unchanged -- [x] adapters unchanged -- [x] All 17 tools work -- [x] Scheduled tasks work -- [x] Memory system works -- [x] Self-healing works - -## Known Limitations - -### Current Limitations -1. **No streaming support** - SDK responses are not streamed (future enhancement) -2. **No usage tracking for Agent SDK** - Only Direct API mode tracks usage -3. **No prompt caching for Agent SDK** - Only Direct API mode supports caching -4. **Mode changes require restart** - Cannot switch modes dynamically - -### Future Enhancements -1. Implement streaming responses via SDK -2. Add SDK-specific usage metrics (if SDK provides them) -3. Implement dynamic mode switching -4. Add prompt caching support for Agent SDK -5. Optimize response format conversion -6. Add batch request support - -## Performance Comparison - -### Cost (per 1M tokens) -| Mode | Input | Output | Notes | -|------|-------|--------|-------| -| Agent SDK | $0 | $0 | Uses Pro subscription | -| Direct API (Haiku) | $0.25 | $1.25 | Pay-per-token | -| Direct API (Sonnet) | $3.00 | $15.00 | Pay-per-token | - -### Speed -- **Agent SDK**: Similar to Direct API -- **Direct API**: Baseline -- **Legacy Server**: Slower (HTTP overhead) - -### Memory -- **Agent SDK**: ~50MB overhead for SDK client -- **Direct API**: Minimal overhead -- **Legacy Server**: Requires separate process - -## Migration Impact - -### Zero Disruption -- Existing users can keep using Direct API mode -- Legacy server mode still works -- No breaking changes -- Smooth migration path - -### Recommended Migration -1. Install new dependencies -2. Let bot default to Agent SDK mode -3. Verify all features work -4. Remove old server code (optional) - -### Rollback Plan -If issues occur: -1. Set `USE_DIRECT_API=true` in `.env` -2. Restart bot -3. Report issues for investigation - -## Success Criteria - -### ✅ All Met -- [x] Agent SDK is the default backend -- [x] API mode still works (not Agent SDK default) -- [x] Async/sync bridge functional -- [x] Response format compatible with agent.py -- [x] Backward compatibility with old env vars -- [x] All existing functionality preserved -- [x] Zero changes to agent.py, tools.py, adapters -- [x] Test suite passes -- [x] Documentation complete - -## Conclusion - -The Claude Agent SDK implementation is **complete and production-ready**. The implementation follows the "Strategy A - Thin Wrapper" approach, making the SDK a pure LLM backend replacement while preserving all existing functionality. - -### Key Achievements -1. ✅ Agent SDK is the default mode -2. ✅ Zero breaking changes -3. ✅ All 17 tools work identically -4. ✅ Comprehensive testing and documentation -5. ✅ Smooth migration path with rollback option - -### Next Steps -1. Test in production environment -2. Monitor for issues -3. Gather user feedback -4. Plan future enhancements (streaming, caching, etc.) -5. Consider deprecating legacy server mode - ---- - -**Implementation Date**: 2026-02-15 -**Strategy Used**: Strategy A - Thin Wrapper -**Files Modified**: 2 (llm_interface.py, requirements.txt) -**Files Created**: 4 (docs + tests) -**Total Lines Added**: ~1,410 lines -**Breaking Changes**: 0 -**Tests Passing**: 5/5 -**Status**: ✅ PRODUCTION READY diff --git a/JARVIS_VOICE_INTEGRATION_PLAN.md b/JARVIS_VOICE_INTEGRATION_PLAN.md new file mode 100644 index 0000000..6c387dd --- /dev/null +++ b/JARVIS_VOICE_INTEGRATION_PLAN.md @@ -0,0 +1,1304 @@ +# Jarvis Voice Integration Plan + +## Executive Summary + +This document provides a comprehensive plan for adding ElevenLabs text-to-speech capabilities to Ajarbot, enabling Garvis to deliver occasional voice responses with a British AI assistant personality (the "Jarvis - Robot" voice). The integration follows the existing codebase patterns: an MCP tool for zero-cost routing when unused, lazy-loaded client for the ElevenLabs API, platform-specific audio delivery via Telegram voice notes and Slack file uploads, and a character budget tracker to stay within the free tier's 10,000 characters per month. + +**Key decisions:** +- **Architecture**: Hybrid MCP tool (voice generation) + adapter-level audio delivery +- **Voice**: ElevenLabs pre-made "Jarvis - Robot" (ID: `WWtyH2oxeOp9yZwK8ERD`) +- **Trigger model**: Explicit user commands and optional LLM-driven autonomous voice for high-impact moments +- **Cost**: Free tier (10,000 chars/month) -- sufficient for casual use (roughly 40-50 short voice messages) + +--- + +## Table of Contents + +1. [Architecture Design](#1-architecture-design) +2. [Implementation Plan](#2-implementation-plan) +3. [ElevenLabs Setup Guide](#3-elevenlabs-setup-guide) +4. [Configuration](#4-configuration) +5. [File-by-File Changes](#5-file-by-file-changes) +6. [Voice Trigger Logic](#6-voice-trigger-logic) +7. [Platform Delivery](#7-platform-delivery) +8. [Cost Monitoring](#8-cost-monitoring) +9. [Testing Strategy](#9-testing-strategy) +10. [Edge Cases and Error Handling](#10-edge-cases-and-error-handling) +11. [Troubleshooting](#11-troubleshooting) +12. [Future Enhancements](#12-future-enhancements) + +--- + +## 1. Architecture Design + +### 1.1 Why Hybrid (MCP Tool + Adapter Extension) + +The existing codebase uses two tool paradigms: +- **MCP tools** (`mcp_tools.py`): Zero API cost when unused, registered via `@tool` decorator, run in-process +- **Traditional tools** (`tools.py`): Google/weather tools requiring external API calls + +Voice generation naturally splits into two concerns: + +| Concern | Component | Rationale | +|---------|-----------|-----------| +| **Text-to-Speech generation** | MCP tool in `mcp_tools.py` | Follows the pattern of `web_fetch` -- makes an external HTTP call but runs as an MCP tool. Zero cost when the tool is not invoked. Lazy-loads the ElevenLabs client. | +| **Audio delivery to platform** | Adapter-level method on `BaseAdapter` | Telegram needs `send_voice()` (OGG/Opus), Slack needs `files_upload_v2()` (MP3). The adapter already owns the platform connection. Adding a `send_voice_message()` method is the cleanest separation. | + +### 1.2 Component Diagram + +``` +User says: "Garvis, say that in your voice" + | + v + [Agent / LLM] -----> decides to use speak_text tool + | + v + [MCP Tool: speak_text] + | 1. Validates character budget + | 2. Calls ElevenLabs TTS API + | 3. Returns audio bytes + metadata + v + [AdapterRuntime._process_message] + | Detects voice attachment in response metadata + | Routes to adapter.send_voice_message() + v + [TelegramAdapter.send_voice_message] or [SlackAdapter.send_voice_message] + | Sends OGG voice note Uploads MP3 file snippet + v + User receives voice message in chat +``` + +### 1.3 Why NOT a Standalone Traditional Tool + +Traditional tools in `tools.py` return plain strings. Voice requires returning binary audio data plus metadata (format, duration, character count). The MCP tool pattern supports structured return values and integrates naturally with the Agent SDK's tool execution pipeline. Additionally, the MCP tool is never loaded or called unless the LLM decides to use it, matching the "zero cost when unused" principle from SOUL.md. + +### 1.4 Data Flow for Voice Responses + +The voice tool follows a **two-phase** approach: + +**Phase 1 - Generation (MCP Tool):** +1. LLM calls `speak_text` tool with the text to speak +2. Tool checks character budget (reject if would exceed monthly limit) +3. Tool calls ElevenLabs API, receives MP3 audio bytes +4. Tool saves audio to a temporary file (`temp/voice_{timestamp}.mp3`) +5. Tool returns success message with file path and metadata + +**Phase 2 - Delivery (Adapter Runtime):** +1. Agent's text response includes a voice marker: `[VOICE: temp/voice_12345.mp3]` +2. Runtime postprocessor detects the marker +3. Runtime calls `adapter.send_voice_message(channel_id, audio_path)` +4. Adapter sends platform-native voice message +5. Temporary file is cleaned up + +This two-phase approach avoids passing binary data through the LLM response chain and uses the existing postprocessor pattern from `adapters/runtime.py`. + +--- + +## 2. Implementation Plan + +### 2.1 Overview of Changes + +| File | Change Type | Description | +|------|-------------|-------------| +| `elevenlabs_client.py` | **NEW** | ElevenLabs API client (TTS, usage tracking) | +| `mcp_tools.py` | MODIFY | Add `speak_text` MCP tool | +| `adapters/base.py` | MODIFY | Add `send_voice_message()` to `BaseAdapter` | +| `adapters/telegram/adapter.py` | MODIFY | Implement `send_voice_message()` using `send_voice()` | +| `adapters/slack/adapter.py` | MODIFY | Implement `send_voice_message()` using `files_upload_v2()` | +| `adapters/runtime.py` | MODIFY | Add voice postprocessor to detect and deliver voice messages | +| `memory_workspace/SOUL.md` | MODIFY | Add `speak_text` tool documentation and voice personality notes | +| `llm_interface.py` | MODIFY | Add `speak_text` to allowed_tools list | +| `.env.example` | MODIFY | Add ElevenLabs configuration variables | +| `.gitignore` | MODIFY | Add temp voice files | +| `config/voice_preferences.yaml` | **NEW** | Per-user voice preferences (optional) | + +### 2.2 Dependencies + +```bash +pip install elevenlabs # Official Python SDK +pip install pydub # Audio format conversion (MP3 -> OGG/Opus for Telegram) +``` + +Note: `pydub` requires `ffmpeg` installed on the system for OGG/Opus conversion. On Windows: `choco install ffmpeg` or download from https://ffmpeg.org/download.html. + +--- + +## 3. ElevenLabs Setup Guide + +### 3.1 Account Creation + +1. Go to https://elevenlabs.io and sign up for a free account +2. Verify your email address +3. Navigate to **Profile + API Key** (click your avatar, top right) +4. Copy your API key + +### 3.2 Voice Selection + +The pre-made "Jarvis - Robot" voice is ideal for this use case: +- **Voice ID**: `WWtyH2oxeOp9yZwK8ERD` +- **Character**: British, robotic, AI assistant personality +- **Quality**: High quality even on free tier +- **No voice cloning needed**: Pre-made voices are available immediately + +To verify the voice ID or browse alternatives: +1. Go to https://elevenlabs.io/voice-library +2. Search for "Jarvis" +3. Click the voice to preview it +4. The voice ID is in the URL or available via API + +### 3.3 Free Tier Limits + +| Limit | Value | +|-------|-------| +| Characters per month | 10,000 | +| Max characters per request | 2,500 | +| Custom voices | 3 | +| Commercial use | No | +| Audio quality | Standard | +| Concurrent requests | 2 | + +**Budget math**: At ~200 characters per voice message (average sentence), 10,000 chars allows roughly **50 voice messages per month** -- more than enough for "here and there" casual use. + +### 3.4 API Key Configuration + +Add to `.env`: +```bash +# ElevenLabs Voice (Optional) +ELEVENLABS_API_KEY=your-api-key-here +ELEVENLABS_VOICE_ID=WWtyH2oxeOp9yZwK8ERD +``` + +--- + +## 4. Configuration + +### 4.1 Environment Variables + +```bash +# === ElevenLabs Voice Configuration === + +# API key from https://elevenlabs.io (required for voice features) +ELEVENLABS_API_KEY=your-api-key-here + +# Voice ID - default: Jarvis - Robot (British AI assistant) +ELEVENLABS_VOICE_ID=WWtyH2oxeOp9yZwK8ERD + +# Model ID - default: eleven_multilingual_v2 (best quality) +# Options: eleven_multilingual_v2, eleven_turbo_v2_5 (faster, lower quality) +ELEVENLABS_MODEL_ID=eleven_multilingual_v2 + +# Monthly character budget (default: 9000, leaves 1000 char buffer from 10k limit) +ELEVENLABS_MONTHLY_BUDGET=9000 + +# Output format (default: mp3_44100_128 - good quality, reasonable size) +ELEVENLABS_OUTPUT_FORMAT=mp3_44100_128 + +# Enable/disable voice features globally +ELEVENLABS_ENABLED=true +``` + +### 4.2 Per-User Voice Preferences (Optional) + +File: `config/voice_preferences.yaml` + +```yaml +# Voice preferences per user +# Users can enable/disable voice responses and set preferences + +defaults: + voice_enabled: true + voice_mode: "explicit" # "explicit" = only on request, "auto" = LLM decides + max_chars_per_message: 500 # Limit text length for voice to save budget + +users: + jordan: + voice_enabled: true + voice_mode: "auto" # Garvis can decide when to use voice + preferred_voice: "WWtyH2oxeOp9yZwK8ERD" # Jarvis - Robot +``` + +### 4.3 Voice Trigger Modes + +| Mode | Behavior | Configuration | +|------|----------|---------------| +| `explicit` | Voice only when user explicitly requests (e.g., "say that out loud", "voice response") | Default, safest for budget | +| `auto` | LLM decides when voice adds value (greetings, dramatic moments, short quips) | Set `voice_mode: auto` for user | +| `disabled` | No voice at all | Set `voice_enabled: false` | + +--- + +## 5. File-by-File Changes + +### 5.1 NEW: `elevenlabs_client.py` + +This module handles all ElevenLabs API interaction, usage tracking, and audio format management. + +```python +"""ElevenLabs TTS client for Garvis voice capabilities. + +Handles text-to-speech generation, character budget tracking, +and audio format management. Lazy-loaded -- zero cost when unused. +""" + +import json +import os +import time +from datetime import datetime +from pathlib import Path +from typing import Any, Dict, Optional + +import httpx + +# Budget tracking file +_USAGE_FILE = Path("config/elevenlabs_usage.json") +_TEMP_DIR = Path("temp/voice") + + +class ElevenLabsClient: + """ElevenLabs TTS client with budget tracking.""" + + def __init__(self) -> None: + self.api_key = os.getenv("ELEVENLABS_API_KEY", "") + self.voice_id = os.getenv("ELEVENLABS_VOICE_ID", "WWtyH2oxeOp9yZwK8ERD") + self.model_id = os.getenv("ELEVENLABS_MODEL_ID", "eleven_multilingual_v2") + self.output_format = os.getenv("ELEVENLABS_OUTPUT_FORMAT", "mp3_44100_128") + self.monthly_budget = int(os.getenv("ELEVENLABS_MONTHLY_BUDGET", "9000")) + self.enabled = os.getenv("ELEVENLABS_ENABLED", "true").lower() == "true" + + self._base_url = "https://api.elevenlabs.io/v1" + _TEMP_DIR.mkdir(parents=True, exist_ok=True) + + def is_available(self) -> bool: + """Check if ElevenLabs is configured and enabled.""" + return bool(self.enabled and self.api_key) + + def get_remaining_budget(self) -> int: + """Get remaining character budget for this month.""" + usage = self._load_usage() + current_month = datetime.now().strftime("%Y-%m") + + if usage.get("month") != current_month: + # New month, reset counter + return self.monthly_budget + + return max(0, self.monthly_budget - usage.get("chars_used", 0)) + + def text_to_speech( + self, + text: str, + voice_id: Optional[str] = None, + stability: float = 0.5, + similarity_boost: float = 0.75, + style: float = 0.0, + speed: float = 1.0, + ) -> Dict[str, Any]: + """Convert text to speech using ElevenLabs API. + + Args: + text: Text to convert (max 2500 chars on free tier) + voice_id: Override default voice ID + stability: Voice stability (0.0-1.0) + similarity_boost: Voice clarity (0.0-1.0) + style: Style exaggeration (0.0-1.0, costs more latency) + speed: Speech speed (0.7-1.2) + + Returns: + Dict with: success, audio_path, chars_used, remaining_budget, duration_ms + """ + if not self.is_available(): + return {"success": False, "error": "ElevenLabs not configured or disabled"} + + # Budget check + char_count = len(text) + remaining = self.get_remaining_budget() + + if char_count > remaining: + return { + "success": False, + "error": ( + f"Insufficient character budget. " + f"Need {char_count} chars, only {remaining} remaining this month. " + f"Budget resets on the 1st." + ), + } + + if char_count > 2500: + return { + "success": False, + "error": ( + f"Text too long ({char_count} chars). " + f"Free tier limit is 2500 chars per request. " + f"Shorten the text or split into multiple requests." + ), + } + + if char_count == 0: + return {"success": False, "error": "No text provided"} + + # Call ElevenLabs API + vid = voice_id or self.voice_id + url = f"{self._base_url}/text-to-speech/{vid}" + + headers = { + "xi-api-key": self.api_key, + "Content-Type": "application/json", + "Accept": "audio/mpeg", + } + + payload = { + "text": text, + "model_id": self.model_id, + "voice_settings": { + "stability": stability, + "similarity_boost": similarity_boost, + "style": style, + "speed": speed, + "use_speaker_boost": True, + }, + } + + try: + start_time = time.time() + + with httpx.Client(timeout=30.0) as client: + response = client.post( + url, + headers=headers, + json=payload, + params={"output_format": self.output_format}, + ) + response.raise_for_status() + + duration_ms = (time.time() - start_time) * 1000 + + # Save audio to temp file + timestamp = datetime.now().strftime("%Y%m%d_%H%M%S") + audio_path = _TEMP_DIR / f"voice_{timestamp}.mp3" + audio_path.write_bytes(response.content) + + # Update usage tracking + self._track_usage(char_count) + + return { + "success": True, + "audio_path": str(audio_path), + "chars_used": char_count, + "remaining_budget": self.get_remaining_budget(), + "duration_ms": round(duration_ms), + "audio_size_bytes": len(response.content), + } + + except httpx.HTTPStatusError as e: + if e.response.status_code == 401: + return {"success": False, "error": "Invalid ElevenLabs API key"} + elif e.response.status_code == 429: + return {"success": False, "error": "Rate limited. Wait a moment and try again."} + else: + return {"success": False, "error": f"ElevenLabs API error: {e.response.status_code}"} + except httpx.TimeoutException: + return {"success": False, "error": "ElevenLabs API request timed out (30s)"} + except Exception as e: + return {"success": False, "error": f"Voice generation failed: {str(e)}"} + + def _load_usage(self) -> Dict: + """Load usage data from file.""" + if not _USAGE_FILE.exists(): + return {"month": "", "chars_used": 0, "requests": 0, "history": []} + + try: + return json.loads(_USAGE_FILE.read_text(encoding="utf-8")) + except Exception: + return {"month": "", "chars_used": 0, "requests": 0, "history": []} + + def _track_usage(self, chars: int) -> None: + """Track character usage for budget monitoring.""" + usage = self._load_usage() + current_month = datetime.now().strftime("%Y-%m") + + if usage.get("month") != current_month: + # New month, archive old data and reset + usage = { + "month": current_month, + "chars_used": 0, + "requests": 0, + "history": [], + } + + usage["chars_used"] = usage.get("chars_used", 0) + chars + usage["requests"] = usage.get("requests", 0) + 1 + usage["history"].append({ + "timestamp": datetime.now().isoformat(), + "chars": chars, + }) + + # Keep history manageable (last 100 entries) + if len(usage["history"]) > 100: + usage["history"] = usage["history"][-100:] + + _USAGE_FILE.parent.mkdir(parents=True, exist_ok=True) + _USAGE_FILE.write_text( + json.dumps(usage, indent=2), + encoding="utf-8", + ) + + def get_usage_report(self) -> str: + """Get a formatted usage report.""" + usage = self._load_usage() + current_month = datetime.now().strftime("%Y-%m") + + if usage.get("month") != current_month: + return ( + f"Voice Usage ({current_month}):\n" + f" Characters used: 0 / {self.monthly_budget}\n" + f" Requests: 0\n" + f" Budget remaining: {self.monthly_budget} chars" + ) + + chars_used = usage.get("chars_used", 0) + remaining = max(0, self.monthly_budget - chars_used) + pct = (chars_used / self.monthly_budget * 100) if self.monthly_budget > 0 else 0 + + report = ( + f"Voice Usage ({current_month}):\n" + f" Characters used: {chars_used:,} / {self.monthly_budget:,} ({pct:.1f}%)\n" + f" Requests: {usage.get('requests', 0)}\n" + f" Budget remaining: {remaining:,} chars" + ) + + if pct >= 80: + report += "\n WARNING: Approaching monthly limit!" + elif pct >= 100: + report += "\n BUDGET EXHAUSTED: Voice disabled until next month." + + return report + + @staticmethod + def cleanup_temp_files(max_age_hours: int = 1) -> int: + """Remove temporary voice files older than max_age_hours. + + Returns number of files cleaned up. + """ + if not _TEMP_DIR.exists(): + return 0 + + cutoff = time.time() - (max_age_hours * 3600) + cleaned = 0 + + for audio_file in _TEMP_DIR.glob("voice_*.mp3"): + try: + if audio_file.stat().st_mtime < cutoff: + audio_file.unlink() + cleaned += 1 + except Exception: + continue + + return cleaned +``` + +### 5.2 MODIFY: `mcp_tools.py` -- Add `speak_text` Tool + +Add after the existing tool definitions, before the `file_system_server` creation: + +```python +# ============================================ +# ElevenLabs Voice Tool (MCP) +# ============================================ +# Lazy-loaded ElevenLabs client +_elevenlabs_client: Optional[Any] = None + + +def _get_elevenlabs_client(): + """Lazy-load ElevenLabs client when first needed.""" + global _elevenlabs_client + if _elevenlabs_client is None: + try: + from elevenlabs_client import ElevenLabsClient + _elevenlabs_client = ElevenLabsClient() + except ImportError: + return None + return _elevenlabs_client + + +@tool( + name="speak_text", + description=( + "Convert text to speech using Garvis's voice (British AI assistant). " + "Use this to deliver important messages, greetings, witty remarks, or " + "when the user explicitly asks for a voice response. The audio will be " + "sent as a voice message on the user's platform (Telegram voice note " + "or Slack audio). Keep text concise -- budget is limited to ~9000 " + "chars/month (free tier). Returns a voice marker that the runtime " + "will convert to an audio message." + ), + input_schema={ + "text": str, # The text to speak (max 2500 chars) + }, +) +async def speak_text_tool(args: Dict[str, Any]) -> Dict[str, Any]: + """Generate speech from text using ElevenLabs. + + Zero-cost MCP tool when not invoked. Calls ElevenLabs API only on use. + Returns a voice marker for the runtime to process. + """ + text = args.get("text", "").strip() + + if not text: + return { + "content": [{"type": "text", "text": "Error: No text provided for speech"}], + "isError": True, + } + + client = _get_elevenlabs_client() + if not client: + return { + "content": [{ + "type": "text", + "text": "Error: ElevenLabs not configured. Add ELEVENLABS_API_KEY to .env", + }], + "isError": True, + } + + if not client.is_available(): + return { + "content": [{ + "type": "text", + "text": "Error: ElevenLabs voice is disabled or not configured", + }], + "isError": True, + } + + # Check budget before calling API + remaining = client.get_remaining_budget() + if len(text) > remaining: + return { + "content": [{ + "type": "text", + "text": ( + f"Voice budget insufficient: need {len(text)} chars, " + f"only {remaining} remaining this month. " + f"Respond with text instead." + ), + }], + "isError": True, + } + + # Generate speech + result = client.text_to_speech(text) + + if result["success"]: + audio_path = result["audio_path"] + chars_used = result["chars_used"] + remaining_budget = result["remaining_budget"] + + # Return a voice marker that the runtime postprocessor will detect. + # The marker embeds the audio file path so the runtime can find it. + return { + "content": [{ + "type": "text", + "text": ( + f"[VOICE:{audio_path}]\n\n" + f"Voice message generated ({chars_used} chars, " + f"{remaining_budget} remaining this month). " + f"The audio will be delivered as a voice message." + ), + }], + } + else: + return { + "content": [{ + "type": "text", + "text": f"Voice generation failed: {result['error']}. Responding with text instead.", + }], + "isError": True, + } +``` + +Then add `speak_text_tool` to the `file_system_server` tools list: + +```python +file_system_server = create_sdk_mcp_server( + name="file_system", + version="2.1.0", # bump version + tools=[ + # ... existing tools ... + # Voice tool + speak_text_tool, + ] +) +``` + +### 5.3 MODIFY: `llm_interface.py` -- Add to Allowed Tools + +In `_build_agent_sdk_options()`, add `"speak_text"` to the `allowed_tools` list: + +```python +allowed_tools = [ + # ... existing tools ... + # Voice + "speak_text", +] +``` + +### 5.4 MODIFY: `adapters/base.py` -- Add Voice Support + +Add to `AdapterCapabilities`: + +```python +@dataclass +class AdapterCapabilities: + supports_threads: bool = False + supports_reactions: bool = False + supports_media: bool = False + supports_files: bool = False + supports_markdown: bool = False + supports_voice: bool = False # NEW + max_message_length: int = 2000 + chunking_strategy: Optional[str] = None +``` + +Add default method to `BaseAdapter`: + +```python +async def send_voice_message( + self, + channel_id: str, + audio_path: str, + reply_to_id: Optional[str] = None, + thread_id: Optional[str] = None, + caption: Optional[str] = None, +) -> Dict[str, Any]: + """Send a voice/audio message to the platform. Optional. + + Args: + channel_id: Target channel/chat ID + audio_path: Path to the audio file (MP3) + reply_to_id: Optional message to reply to + thread_id: Optional thread to post in + caption: Optional text caption with the voice message + + Returns: + Dict with at least {"success": bool} + """ + return {"success": False, "error": "Voice not supported on this platform"} +``` + +### 5.5 MODIFY: `adapters/telegram/adapter.py` -- Implement Voice Sending + +Add to capabilities: + +```python +@property +def capabilities(self) -> AdapterCapabilities: + return AdapterCapabilities( + supports_threads=False, + supports_reactions=True, + supports_media=True, + supports_files=True, + supports_markdown=True, + supports_voice=True, # NEW + max_message_length=4096, + chunking_strategy="markdown", + ) +``` + +Add voice sending method: + +```python +async def send_voice_message( + self, + channel_id: str, + audio_path: str, + reply_to_id: Optional[str] = None, + thread_id: Optional[str] = None, + caption: Optional[str] = None, +) -> Dict[str, Any]: + """Send a voice message to Telegram. + + Telegram voice notes require OGG/Opus format. If the source is MP3, + we convert it using pydub + ffmpeg. Telegram also accepts MP3 directly + via send_voice() since Bot API 6.0+. + """ + if not self.bot: + return {"success": False, "error": "Bot not started"} + + try: + from pathlib import Path + + audio_file = Path(audio_path) + if not audio_file.exists(): + return {"success": False, "error": f"Audio file not found: {audio_path}"} + + chat_id = int(channel_id) + reply_id = int(reply_to_id) if reply_to_id else None + + # Attempt OGG/Opus conversion for native voice note display. + # Falls back to sending MP3 directly if pydub/ffmpeg not available. + ogg_path = None + try: + from pydub import AudioSegment + audio = AudioSegment.from_mp3(str(audio_file)) + ogg_path = audio_file.with_suffix(".ogg") + audio.export(str(ogg_path), format="ogg", codec="libopus") + voice_file = ogg_path + except Exception: + # pydub or ffmpeg not available; send MP3 directly + voice_file = audio_file + + with open(voice_file, "rb") as f: + sent = await self.bot.send_voice( + chat_id=chat_id, + voice=f, + caption=caption, + reply_to_message_id=reply_id, + ) + + # Clean up temporary OGG file + if ogg_path and ogg_path.exists(): + try: + ogg_path.unlink() + except Exception: + pass + + return { + "success": True, + "message_id": sent.message_id, + "chat_id": sent.chat_id, + } + + except TelegramError as e: + print(f"[Telegram] Error sending voice: {e}") + return {"success": False, "error": str(e)} + except Exception as e: + print(f"[Telegram] Voice send error: {e}") + return {"success": False, "error": str(e)} +``` + +### 5.6 MODIFY: `adapters/slack/adapter.py` -- Implement Voice Sending + +Add to capabilities: + +```python +@property +def capabilities(self) -> AdapterCapabilities: + return AdapterCapabilities( + supports_threads=True, + supports_reactions=True, + supports_media=True, + supports_files=True, + supports_markdown=True, + supports_voice=True, # NEW + max_message_length=4000, + chunking_strategy="word", + ) +``` + +Add voice sending method: + +```python +async def send_voice_message( + self, + channel_id: str, + audio_path: str, + reply_to_id: Optional[str] = None, + thread_id: Optional[str] = None, + caption: Optional[str] = None, +) -> Dict[str, Any]: + """Send a voice/audio file to Slack. + + Uses files_upload_v2 (the modern file upload method). + Slack displays MP3 files with an inline audio player. + """ + if not self.app: + return {"success": False, "error": "Adapter not started"} + + try: + from pathlib import Path + + audio_file = Path(audio_path) + if not audio_file.exists(): + return {"success": False, "error": f"Audio file not found: {audio_path}"} + + result = await self.app.client.files_upload_v2( + channel=channel_id, + file=str(audio_file), + filename=f"garvis_voice_{audio_file.stem}.mp3", + title="Garvis Voice Message", + initial_comment=caption or "", + thread_ts=thread_id, + ) + + return { + "success": True, + "file_id": result.get("file", {}).get("id", "unknown"), + } + + except SlackApiError as e: + error_msg = e.response["error"] + print(f"[Slack] Error sending voice: {error_msg}") + return {"success": False, "error": error_msg} + except Exception as e: + print(f"[Slack] Voice send error: {e}") + return {"success": False, "error": str(e)} +``` + +### 5.7 MODIFY: `adapters/runtime.py` -- Voice Postprocessor + +Add a voice postprocessor that detects `[VOICE:path]` markers in the agent's response and triggers audio delivery. Add this function and register it: + +```python +import re +from pathlib import Path + + +def _extract_voice_markers(text: str) -> list: + """Extract [VOICE:path] markers from text. + + Returns list of (marker_string, audio_path) tuples. + """ + pattern = r'\[VOICE:(.*?)\]' + matches = re.findall(pattern, text) + return [(f"[VOICE:{path}]", path.strip()) for path in matches] + + +# In AdapterRuntime._process_message(), after the agent response is received +# and before sending the text response, add voice handling: + +# --- Inside _process_message, after getting `response` from agent.chat() --- + +# Handle voice markers in response +voice_markers = _extract_voice_markers(response) +if voice_markers and adapter and adapter.capabilities.supports_voice: + for marker, audio_path in voice_markers: + # Remove the marker from the text response + response = response.replace(marker, "").strip() + + # Send the voice message + voice_result = await adapter.send_voice_message( + channel_id=message.channel_id, + audio_path=audio_path, + reply_to_id=( + message.metadata.get("ts") + or str(message.metadata.get("message_id", "")) + ), + thread_id=message.thread_id, + ) + + if voice_result.get("success"): + print( + f"[{message.platform.upper()}] Voice message sent " + f"({Path(audio_path).stat().st_size} bytes)" + ) + else: + print( + f"[{message.platform.upper()}] Voice send failed: " + f"{voice_result.get('error')}" + ) + + # Clean up temp audio file + try: + Path(audio_path).unlink(missing_ok=True) + except Exception: + pass + +# If the text response is now empty (was voice-only), add a minimal text fallback +if not response.strip() and voice_markers: + response = "" # Don't send empty text; voice was the response + +# Continue with existing text send logic (only if response is non-empty) +``` + +### 5.8 MODIFY: `memory_workspace/SOUL.md` -- Document Voice Tool + +Add to the "Available Tools" section: + +```markdown +### Voice (ElevenLabs - API Cost) +- speak_text (convert text to Garvis voice message, delivered as platform voice note) + +**Voice Guidelines**: +- Use voice for: greetings, witty quips, important announcements, when user asks +- Keep voice messages SHORT (1-3 sentences, under 500 chars) +- Budget: ~9,000 chars/month -- be selective +- Always provide text alongside voice (accessibility) +- Voice personality: British AI assistant, dry wit, composed confidence +- Signature phrases in voice: "Right then", "Very good, sir", "I've taken the liberty of..." +- DO NOT use voice for: long explanations, code, lists, weather reports (waste of budget) +``` + +### 5.9 MODIFY: `.env.example` -- Add ElevenLabs Section + +```bash +# ======================================== +# ElevenLabs Voice (Optional) +# ======================================== +# Enables Jarvis-style voice responses +# Sign up: https://elevenlabs.io (free tier: 10,000 chars/month) + +# API Key (from Profile + API Key page) +ELEVENLABS_API_KEY=your-api-key-here + +# Voice ID - Jarvis Robot (British AI assistant) +ELEVENLABS_VOICE_ID=WWtyH2oxeOp9yZwK8ERD + +# Model - eleven_multilingual_v2 (best) or eleven_turbo_v2_5 (faster) +# ELEVENLABS_MODEL_ID=eleven_multilingual_v2 + +# Monthly character budget (default: 9000, buffer below 10k free limit) +# ELEVENLABS_MONTHLY_BUDGET=9000 + +# Enable/disable voice globally +# ELEVENLABS_ENABLED=true +``` + +### 5.10 MODIFY: `.gitignore` -- Add Voice Temp Files + +``` +# Voice temp files +temp/voice/ +config/elevenlabs_usage.json +``` + +--- + +## 6. Voice Trigger Logic + +### 6.1 When Garvis Should Use Voice + +The LLM decides when to use the `speak_text` tool based on SOUL.md instructions. The key triggers: + +**Explicit triggers** (always use voice): +- "Say that out loud" +- "Voice response please" +- "Tell me in your voice" +- "Speak to me, Garvis" +- Any message containing "voice" + "say/tell/speak/read" + +**Auto triggers** (when `voice_mode: auto`, LLM decides): +- Morning greetings: "Good morning, sir. I trust you slept well." +- Task completion announcements: "All done. Your calendar is updated." +- Witty remarks / personality moments +- Important alerts: "Sir, your budget has exceeded 75%." + +**Never voice** (even in auto mode): +- Code blocks or technical output +- Long responses (> 500 chars) +- Lists, tables, structured data +- Weather reports (text is more useful) +- When budget is low (< 1000 chars remaining) + +### 6.2 Dual Response Pattern + +When using voice, Garvis should ALWAYS send both: +1. **Voice message**: The spoken audio +2. **Text message**: The same (or slightly different) text version + +This ensures accessibility, searchability, and works even if voice delivery fails. The text accompanies the voice naturally -- Telegram shows it as a caption, Slack shows it as a message with the audio file. + +Example agent response with voice: +``` +[VOICE:temp/voice/voice_20260217_143022.mp3] + +Good morning, sir. The weather in Centennial looks rather agreeable today -- 72 degrees with clear skies. I'd recommend that light jacket you've been neglecting. +``` + +The runtime strips the `[VOICE:...]` marker, sends the audio, then sends the remaining text as a regular message. + +--- + +## 7. Platform Delivery + +### 7.1 Telegram + +| Aspect | Details | +|--------|---------| +| API Method | `bot.send_voice()` | +| Format | OGG/Opus (converted from MP3 via pydub) or MP3 directly | +| Display | Native voice note player (waveform visualization) | +| Max Size | 50 MB | +| Caption | Supported (text alongside voice note) | +| Duration | Auto-detected from audio metadata | + +**User experience**: The voice message appears as a playable waveform bubble in chat. The user taps to listen. It looks and feels like a standard Telegram voice message. + +### 7.2 Slack + +| Aspect | Details | +|--------|---------| +| API Method | `files_upload_v2()` | +| Format | MP3 (no conversion needed) | +| Display | Inline audio player with play button | +| Max Size | Determined by workspace plan | +| Caption | Via `initial_comment` parameter | +| Thread | Supported via `thread_ts` | + +**User experience**: The audio appears as an uploaded file with Slack's inline audio player. The user clicks play to listen. Less native-feeling than Telegram but functional. + +--- + +## 8. Cost Monitoring + +### 8.1 Character Budget System + +The `ElevenLabsClient` tracks usage in `config/elevenlabs_usage.json`: + +```json +{ + "month": "2026-02", + "chars_used": 2340, + "requests": 12, + "history": [ + {"timestamp": "2026-02-17T14:30:22", "chars": 180}, + {"timestamp": "2026-02-17T15:45:10", "chars": 220} + ] +} +``` + +### 8.2 Budget Enforcement + +| Chars Remaining | Behavior | +|----------------|----------| +| > 3000 | Normal operation | +| 1000 - 3000 | LLM gets usage warning in tool response | +| 100 - 1000 | Only explicit voice requests honored | +| 0 | Voice tool returns error, LLM responds with text only | + +### 8.3 Monthly Budget Reset + +The budget resets automatically on the 1st of each month (detected by comparing `usage.month` with current month string). + +### 8.4 Integration with Daily Cost Report + +The existing daily cost report scheduled task (`daily-cost-report` in `scheduled_tasks.yaml`) can be extended to include voice usage. The agent can read `config/elevenlabs_usage.json` using the `read_file` tool and include voice stats in the report. + +--- + +## 9. Testing Strategy + +### 9.1 Unit Tests + +```python +# test_elevenlabs.py + +def test_budget_tracking(): + """Ensure character budget is tracked correctly.""" + client = ElevenLabsClient() + # Reset usage file + # Track 100 chars + # Assert remaining = budget - 100 + +def test_budget_rejection(): + """Ensure over-budget requests are rejected.""" + # Set budget to 50 + # Attempt to speak 100 chars + # Assert error returned + +def test_monthly_reset(): + """Ensure budget resets on new month.""" + # Write usage with month = "2026-01" + # Check remaining in "2026-02" + # Assert full budget available + +def test_text_too_long(): + """Ensure 2500 char per-request limit is enforced.""" + # Attempt to speak 3000 chars + # Assert error about per-request limit + +def test_empty_text(): + """Ensure empty text is rejected.""" + +def test_temp_file_cleanup(): + """Ensure old temp files are cleaned up.""" +``` + +### 9.2 Integration Tests + +```python +def test_voice_marker_extraction(): + """Test [VOICE:path] marker parsing.""" + text = "Hello [VOICE:temp/voice/v1.mp3] world" + markers = _extract_voice_markers(text) + assert len(markers) == 1 + assert markers[0][1] == "temp/voice/v1.mp3" + +def test_voice_marker_removal(): + """Test that markers are cleanly removed from text.""" + text = "[VOICE:temp/v.mp3]\n\nHello, sir." + markers = _extract_voice_markers(text) + clean = text.replace(markers[0][0], "").strip() + assert clean == "Hello, sir." + +def test_telegram_voice_send(): + """Test Telegram voice message delivery (mock).""" + # Mock bot.send_voice + # Call adapter.send_voice_message + # Assert send_voice called with correct params + +def test_slack_voice_send(): + """Test Slack audio file upload (mock).""" + # Mock app.client.files_upload_v2 + # Call adapter.send_voice_message + # Assert upload called with correct params +``` + +### 9.3 Manual Testing Checklist + +- [ ] Set `ELEVENLABS_API_KEY` in `.env` +- [ ] Send "Garvis, say hello in your voice" via Telegram +- [ ] Verify voice note appears in Telegram chat +- [ ] Verify voice waveform is playable +- [ ] Verify text response also appears alongside voice +- [ ] Send "Speak to me" via Slack (if configured) +- [ ] Verify audio file appears in Slack with player +- [ ] Check `config/elevenlabs_usage.json` for correct tracking +- [ ] Test budget exhaustion (set budget to 10, speak > 10 chars) +- [ ] Verify graceful fallback to text when voice fails +- [ ] Test with `ELEVENLABS_ENABLED=false` -- voice tool should return error +- [ ] Test with missing API key -- voice tool should return error +- [ ] Test text > 2500 chars -- should reject with clear message +- [ ] Verify temp files are cleaned up after delivery +- [ ] Test on slow network (API timeout handling) + +--- + +## 10. Edge Cases and Error Handling + +### 10.1 Error Scenarios + +| Scenario | Handling | +|----------|----------| +| No API key configured | Tool returns error, LLM responds with text | +| Invalid API key | Tool returns clear error message | +| API rate limit (429) | Tool returns "wait and retry" message | +| API timeout | Tool returns timeout error after 30s | +| Audio conversion fails (no ffmpeg) | Send MP3 directly (Telegram supports it since Bot API 6.0) | +| Budget exhausted | Tool rejects with remaining chars info | +| Temp file missing at send time | Log error, send text-only response | +| Platform doesn't support voice | Voice marker stays in text (removed by marker cleanup), text still sent | +| Large text (> 2500 chars) | Tool rejects, suggests shortening | +| Empty text | Tool rejects immediately | +| Network error during API call | Tool returns error, LLM falls back to text | +| Concurrent voice requests | Each gets its own timestamp-based temp file | +| Bot restart mid-voice | Orphaned temp files cleaned up by periodic cleanup | + +### 10.2 Graceful Degradation + +The system degrades gracefully at every level: + +1. **No ElevenLabs configured**: Tool returns error -> LLM uses text only +2. **Budget exhausted**: Tool rejects -> LLM uses text only +3. **API failure**: Tool returns error -> LLM uses text only +4. **Audio conversion fails**: Send MP3 instead of OGG +5. **Platform doesn't support voice**: Text response still delivered +6. **Voice file cleanup fails**: No impact on user; files are small + +### 10.3 Security Considerations + +- **API key**: Stored in `.env` (gitignored), never logged +- **Audio files**: Temporary, auto-cleaned, stored in `temp/voice/` (gitignored) +- **Usage data**: `config/elevenlabs_usage.json` (gitignored) -- no sensitive data +- **User content**: Text sent to ElevenLabs API for synthesis -- same privacy model as sending text to any external API. ElevenLabs has a zero-retention mode (`enable_logging: false`) that can be enabled for additional privacy. + +--- + +## 11. Troubleshooting + +### 11.1 Common Issues + +**"ElevenLabs not configured"** +- Ensure `ELEVENLABS_API_KEY` is set in `.env` +- Ensure `ELEVENLABS_ENABLED` is not set to `false` +- Restart the bot after changing `.env` + +**"Invalid ElevenLabs API key"** +- Check key at https://elevenlabs.io (Profile + API Key) +- Ensure no trailing whitespace in `.env` +- Free tier keys work fine; no paid plan needed + +**Voice note shows as file instead of voice player (Telegram)** +- Install `ffmpeg`: `choco install ffmpeg` (Windows) or `apt install ffmpeg` (Linux) +- Install `pydub`: `pip install pydub` +- Without these, MP3 is sent directly; Telegram may display it as audio file instead of voice note + +**No sound / corrupted audio** +- Check ElevenLabs dashboard for the request in usage history +- Try changing `ELEVENLABS_OUTPUT_FORMAT` to `mp3_22050_32` (smaller, more compatible) +- Verify the voice ID is correct: `WWtyH2oxeOp9yZwK8ERD` + +**Budget shows exhausted but it's a new month** +- Delete `config/elevenlabs_usage.json` -- it will be recreated +- The auto-reset checks the month string; manual deletion is safe + +**Voice works in Telegram but not Slack** +- Ensure Slack bot has `files:write` scope +- Check Slack workspace file upload limits + +### 11.2 Diagnostic Commands + +You can ask Garvis directly: +- "What's your voice budget status?" -- reads `elevenlabs_usage.json` +- "Test your voice" -- triggers a short speak_text +- "Disable voice" -- edit preferences + +--- + +## 12. Future Enhancements + +### 12.1 Short Term (After Initial Integration) + +- **Voice-to-text (STT)**: Accept Telegram voice messages as input using ElevenLabs Speech-to-Text API or Whisper. User sends voice -> Garvis transcribes -> processes as text. +- **Voice preference commands**: `/voice on`, `/voice off`, `/voice status` as Telegram commands. +- **Smart budget allocation**: Reserve 20% of budget for the last week of the month. +- **Audio caching**: Cache frequently spoken phrases (greetings, confirmations) to save API calls. + +### 12.2 Medium Term + +- **Custom voice cloning**: Clone a custom Jarvis-like voice using ElevenLabs voice cloning (requires Starter plan at $5/month). Train on MCU JARVIS audio clips for closer personality match. +- **Scheduled voice messages**: Morning briefing delivered as voice note instead of text. "Good morning, sir. Today's forecast calls for..." +- **Emotional voice modulation**: Adjust `stability` and `style` parameters based on message tone (urgent = higher stability, witty = lower stability + more style). +- **Multi-language support**: Use `language_code` parameter for occasional non-English responses. + +### 12.3 Long Term + +- **Real-time voice conversations**: ElevenLabs Conversational AI SDK for live voice chat via Telegram voice calls. +- **Voice-based authentication**: Recognize Jordan's voice vs. other users. +- **Ambient audio**: Background music or sound effects for dramatic effect (Iron Man suit sounds). +- **Voice journal**: Daily summary delivered as a podcast-style voice recording. + +--- + +## Implementation Order + +For a smooth rollout, implement in this order: + +1. **Create `elevenlabs_client.py`** -- standalone, testable, no dependencies on existing code +2. **Add `speak_text` MCP tool to `mcp_tools.py`** -- register the tool +3. **Add `speak_text` to `llm_interface.py` allowed tools** -- make it discoverable +4. **Add `send_voice_message()` to `adapters/base.py`** -- base interface +5. **Implement Telegram voice** in `adapters/telegram/adapter.py` -- primary platform +6. **Add voice postprocessor** to `adapters/runtime.py` -- wire up delivery +7. **Update `SOUL.md`** -- teach Garvis when/how to use voice +8. **Update `.env.example` and `.gitignore`** -- configuration +9. **Test end-to-end** on Telegram +10. **Implement Slack voice** in `adapters/slack/adapter.py` -- secondary platform +11. **Add budget monitoring to daily report** -- observability + +**Estimated effort**: 3-4 hours for core implementation, 1-2 hours for testing. + +--- + +## Quick Reference Card + +``` +Tool: speak_text +Input: { "text": "Hello, sir." } +API: ElevenLabs TTS v1 +Voice: Jarvis - Robot (WWtyH2oxeOp9yZwK8ERD) +Model: eleven_multilingual_v2 +Format: MP3 -> OGG/Opus (Telegram) or MP3 (Slack) +Budget: 9,000 chars/month (free tier: 10,000) +Max/request: 2,500 chars +Temp files: temp/voice/voice_*.mp3 +Usage file: config/elevenlabs_usage.json +Delivery: [VOICE:path] marker -> runtime postprocessor -> adapter.send_voice_message() +Fallback: Always text, voice is enhancement +``` diff --git a/MIGRATION.md b/MIGRATION.md deleted file mode 100644 index a9c709e..0000000 --- a/MIGRATION.md +++ /dev/null @@ -1,325 +0,0 @@ -# Migration Guide: FastAPI Server to Agent SDK - -This guide helps you upgrade from the old FastAPI server setup to the new Claude Agent SDK integration. - -## What Changed? - -### Old Architecture (Deprecated) -``` -Bot → FastAPI Server (localhost:8000) → Claude Code SDK → Claude -``` -- Required running `claude_code_server.py` in separate terminal -- Only worked with Pro subscription -- More complex setup with multiple processes - -### New Architecture (Current) -``` -Bot → Claude Agent SDK → Claude (Pro OR API) -``` -- Single process (no separate server) -- Works with Pro subscription OR API key -- Simpler setup and operation -- Same functionality, less complexity - -## Migration Steps - -### Step 1: Update Dependencies - -Pull latest code and reinstall dependencies: - -```bash -git pull -pip install -r requirements.txt -``` - -This installs `claude-agent-sdk` and removes deprecated dependencies. - -### Step 2: Update Environment Configuration - -Edit your `.env` file: - -**Remove these deprecated variables:** -```bash -# DELETE THESE -USE_CLAUDE_CODE_SERVER=true -CLAUDE_CODE_SERVER_URL=http://localhost:8000 -``` - -**Add new mode selection:** -```bash -# ADD THIS -AJARBOT_LLM_MODE=agent-sdk # Use Pro subscription (default) -# OR -AJARBOT_LLM_MODE=api # Use pay-per-token API -``` - -**If using API mode**, ensure you have: -```bash -ANTHROPIC_API_KEY=sk-ant-... -``` - -**If using agent-sdk mode**, authenticate once: -```bash -claude auth login -``` - -### Step 3: Stop Old Server - -The FastAPI server is no longer needed: - -1. Stop any running `claude_code_server.py` processes -2. Remove it from startup scripts/systemd services -3. Optionally archive or delete `claude_code_server.py` (kept for reference) - -### Step 4: Use New Launcher - -**Old way:** -```bash -# Terminal 1 -python claude_code_server.py - -# Terminal 2 -python bot_runner.py -``` - -**New way:** -```bash -# Single command -run.bat # Windows -python ajarbot.py # Linux/Mac -``` - -The new launcher: -- Runs pre-flight checks (Node.js, authentication, config) -- Sets sensible defaults (agent-sdk mode) -- Starts bot in single process -- No separate server needed - -### Step 5: Test Your Setup - -Run health check: -```bash -python ajarbot.py --health -``` - -Expected output (agent-sdk mode): -``` -============================================================ -Ajarbot Pre-Flight Checks -============================================================ - -✓ Python 3.10.x -✓ Node.js found: v18.x.x -✓ Claude CLI authenticated - -[Configuration Checks] -✓ Config file found: config/adapters.local.yaml - -Pre-flight checks complete! -============================================================ -``` - -### Step 6: Verify Functionality - -Test that everything works: - -1. **Start the bot:** - ```bash - run.bat # or python ajarbot.py - ``` - -2. **Send a test message** via Slack/Telegram - -3. **Verify tools work:** - - Ask bot to read a file - - Request calendar events - - Test scheduled tasks - -All features are preserved: -- 15 tools (file ops, Gmail, Calendar, Contacts) -- Memory system with hybrid search -- Multi-platform adapters -- Task scheduling - -## Mode Comparison - -Choose the mode that fits your use case: - -| Feature | Agent SDK Mode | API Mode | -|---------|---------------|----------| -| **Cost** | $20/month (Pro) | ~$0.25-$3/M tokens | -| **Setup** | `claude auth login` | API key in `.env` | -| **Requirements** | Node.js + Claude CLI | Just Python | -| **Best For** | Personal heavy use | Light use, production | -| **Rate Limits** | Pro subscription limits | API rate limits | - -### Switching Between Modes - -You can switch anytime by editing `.env`: - -```bash -# Switch to agent-sdk -AJARBOT_LLM_MODE=agent-sdk - -# Switch to API -AJARBOT_LLM_MODE=api -ANTHROPIC_API_KEY=sk-ant-... -``` - -No code changes needed - just restart the bot. - -## Troubleshooting - -### "Node.js not found" (Agent SDK mode) - -**Option 1: Install Node.js** -```bash -# Download from https://nodejs.org -# Or via package manager: -winget install OpenJS.NodeJS # Windows -brew install node # Mac -sudo apt install nodejs # Ubuntu/Debian -``` - -**Option 2: Switch to API mode** -```bash -# In .env -AJARBOT_LLM_MODE=api -ANTHROPIC_API_KEY=sk-ant-... -``` - -### "Claude CLI not authenticated" - -```bash -# Check status -claude auth status - -# Re-authenticate -claude auth logout -claude auth login -``` - -If Claude CLI isn't installed, download from: https://claude.ai/download - -### "Agent SDK not available" - -```bash -pip install claude-agent-sdk -``` - -If installation fails, use API mode instead. - -### Old Environment Variables Still Set - -Check your `.env` file for deprecated variables: - -```bash -# These should NOT be in your .env: -USE_CLAUDE_CODE_SERVER=true -CLAUDE_CODE_SERVER_URL=http://localhost:8000 -USE_AGENT_SDK=true -USE_DIRECT_API=true -``` - -Delete them and use `AJARBOT_LLM_MODE` instead. - -### Bot Works But Features Missing - -Ensure you have latest code: -```bash -git pull -pip install -r requirements.txt --upgrade -``` - -All features from the old setup are preserved: -- Tools system (15 tools) -- Memory with hybrid search -- Scheduled tasks -- Google integration -- Multi-platform adapters - -### Performance Issues - -**Agent SDK mode:** -- May hit Pro subscription rate limits -- Temporary solution: Switch to API mode -- Long-term: Wait for limit reset (usually 24 hours) - -**API mode:** -- Check usage with: `python -c "from usage_tracker import UsageTracker; UsageTracker().print_summary()"` -- Costs shown in usage_data.json -- Default Haiku model is very cheap (~$0.04/day moderate use) - -## Rollback Plan - -If you need to rollback to the old setup: - -1. **Restore old .env settings:** - ```bash - USE_CLAUDE_CODE_SERVER=true - CLAUDE_CODE_SERVER_URL=http://localhost:8000 - ``` - -2. **Start the old server:** - ```bash - python claude_code_server.py - ``` - -3. **Run bot with old method:** - ```bash - python bot_runner.py - ``` - -However, the new setup is recommended - same functionality with less complexity. - -## What's Backward Compatible? - -All existing functionality is preserved: - -- Configuration files (`config/adapters.local.yaml`, `config/scheduled_tasks.yaml`) -- Memory database (`memory_workspace/memory.db`) -- User profiles (`memory_workspace/users/`) -- Google OAuth tokens (`config/google_oauth_token.json`) -- Tool definitions and capabilities -- Adapter integrations - -You can safely migrate without losing data or functionality. - -## Benefits of New Setup - -1. **Simpler operation**: Single command to start -2. **Flexible modes**: Choose Pro subscription OR API -3. **Automatic checks**: Pre-flight validation before starting -4. **Better errors**: Clear messages about missing requirements -5. **Less complexity**: No multi-process coordination -6. **Same features**: All 15 tools, adapters, scheduling preserved - -## Need Help? - -- Review [CLAUDE_CODE_SETUP.md](CLAUDE_CODE_SETUP.md) for detailed mode documentation -- Check [README.md](README.md) for quick start guides -- Run `python ajarbot.py --health` to diagnose issues -- Open an issue if you encounter problems - -## Summary - -**Before:** -```bash -# Terminal 1 -python claude_code_server.py - -# Terminal 2 -python bot_runner.py -``` - -**After:** -```bash -# .env -AJARBOT_LLM_MODE=agent-sdk # or "api" - -# Single command -run.bat # Windows -python ajarbot.py # Linux/Mac -``` - -Same features, less complexity, more flexibility. diff --git a/MIGRATION_GUIDE_AGENT_SDK.md b/MIGRATION_GUIDE_AGENT_SDK.md deleted file mode 100644 index 2c989f7..0000000 --- a/MIGRATION_GUIDE_AGENT_SDK.md +++ /dev/null @@ -1,401 +0,0 @@ -# Migration Guide: Agent SDK Implementation - -## Quick Start (TL;DR) - -### For New Users -```bash -# 1. Install dependencies -pip install -r requirements.txt - -# 2. Run the bot (Agent SDK is the default) -python bot_runner.py -``` - -### For Existing Users -```bash -# 1. Update dependencies -pip install -r requirements.txt - -# 2. That's it! The bot will automatically use Agent SDK -# Your existing .env settings are preserved -``` - -## Detailed Migration Steps - -### Step 1: Understand Your Current Setup - -Check your `.env` file: - -```bash -cat .env -``` - -**Scenario A: Using Direct API (Pay-per-token)** -```env -ANTHROPIC_API_KEY=sk-ant-... -# No USE_CLAUDE_CODE_SERVER variable, or it's set to false -``` - -**Scenario B: Using Legacy Claude Code Server** -```env -USE_CLAUDE_CODE_SERVER=true -CLAUDE_CODE_SERVER_URL=http://localhost:8000 -``` - -### Step 2: Choose Your Migration Path - -#### Option 1: Migrate to Agent SDK (Recommended) - -**Benefits:** -- Uses Claude Pro subscription (no per-token costs) -- Same speed as Direct API -- No separate server process required -- All features work identically - -**Steps:** -1. Install dependencies: - ```bash - pip install -r requirements.txt - ``` - -2. Update `.env` (optional - SDK is default): - ```env - # Remove or comment out old settings - # USE_CLAUDE_CODE_SERVER=false - # CLAUDE_CODE_SERVER_URL=http://localhost:8000 - - # Agent SDK is enabled by default, but you can be explicit: - USE_AGENT_SDK=true - - # Keep your API key for fallback (optional) - ANTHROPIC_API_KEY=sk-ant-... - ``` - -3. Run the bot: - ```bash - python bot_runner.py - ``` - -4. Verify Agent SDK is active: - ``` - [LLM] Using Claude Agent SDK (Pro subscription) - ``` - -#### Option 2: Keep Using Direct API - -**When to use:** -- You don't have Claude Pro subscription -- You prefer pay-per-token billing -- You need to track exact API usage costs - -**Steps:** -1. Install dependencies: - ```bash - pip install -r requirements.txt - ``` - -2. Update `.env`: - ```env - USE_DIRECT_API=true - ANTHROPIC_API_KEY=sk-ant-... - ``` - -3. Run the bot: - ```bash - python bot_runner.py - ``` - -4. Verify Direct API is active: - ``` - [LLM] Using Direct API (pay-per-token) - ``` - -#### Option 3: Keep Using Legacy Server (Not Recommended) - -**Only use if:** -- You have a custom modified `claude_code_server.py` -- You need the server for other tools/integrations - -**Steps:** -1. Keep your current setup -2. The legacy server mode still works -3. No changes required - -### Step 3: Test the Migration - -Run the test suite: - -```bash -python test_agent_sdk.py -``` - -Expected output: -``` -=== Test 1: LLMInterface Initialization === -✓ LLMInterface created successfully - - Mode: agent_sdk -... -Total: 5/5 tests passed -🎉 All tests passed! -``` - -### Step 4: Verify Bot Functionality - -Test all critical features: - -1. **Simple Chat:** - ``` - User: Hello! - Bot: [Should respond normally] - ``` - -2. **Tool Usage:** - ``` - User: What files are in the current directory? - Bot: [Should use list_directory tool] - ``` - -3. **Gmail Integration:** - ``` - User: Check my recent emails - Bot: [Should use read_emails tool] - ``` - -4. **Calendar Integration:** - ``` - User: What's on my calendar today? - Bot: [Should use read_calendar tool] - ``` - -5. **Scheduled Tasks:** - - Verify scheduled tasks still run - - Check `config/scheduled_tasks.yaml` - -## Environment Variables Reference - -### New Variables - -| Variable | Default | Description | -|----------|---------|-------------| -| `USE_AGENT_SDK` | `true` | Enable Agent SDK mode (default) | -| `USE_DIRECT_API` | `false` | Force Direct API mode | - -### Existing Variables (Still Supported) - -| Variable | Default | Description | -|----------|---------|-------------| -| `ANTHROPIC_API_KEY` | - | API key for Direct API mode | -| `USE_CLAUDE_CODE_SERVER` | `false` | Enable legacy server mode | -| `CLAUDE_CODE_SERVER_URL` | `http://localhost:8000` | Legacy server URL | - -### Priority Order - -If multiple modes are enabled, the priority is: -1. `USE_DIRECT_API=true` → Direct API mode -2. `USE_CLAUDE_CODE_SERVER=true` → Legacy server mode -3. `USE_AGENT_SDK=true` (default) → Agent SDK mode -4. Agent SDK unavailable → Fallback to Direct API mode - -## Troubleshooting - -### Issue: "Agent SDK not available, falling back to Direct API" - -**Cause:** `claude-agent-sdk` is not installed - -**Solution:** -```bash -pip install claude-agent-sdk -``` - -### Issue: "ModuleNotFoundError: No module named 'claude_agent_sdk'" - -**Cause:** Package not in requirements.txt or not installed - -**Solution:** -```bash -# Verify requirements.txt has claude-agent-sdk -grep claude-agent-sdk requirements.txt - -# If missing, update requirements.txt -pip install claude-agent-sdk anyio - -# Or reinstall all dependencies -pip install -r requirements.txt -``` - -### Issue: Bot still using Direct API after migration - -**Cause:** Explicit `USE_DIRECT_API=true` in `.env` - -**Solution:** -```bash -# Edit .env and remove or change to false -USE_DIRECT_API=false - -# Or comment out the line -# USE_DIRECT_API=true -``` - -### Issue: "anyio" import error - -**Cause:** `anyio` package not installed (required for async/sync bridge) - -**Solution:** -```bash -pip install anyio>=4.0.0 -``` - -### Issue: Response format errors in agent.py - -**Cause:** SDK response not properly converted to Message format - -**Solution:** -1. Check `_convert_sdk_response_to_message()` implementation -2. Verify `TextBlock` and `ToolUseBlock` are imported -3. Run `python test_agent_sdk.py` to verify format compatibility - -### Issue: Tool execution fails with Agent SDK - -**Cause:** Agent SDK might not be returning expected tool format - -**Solution:** -1. Check `_agent_sdk_chat_with_tools()` method -2. Verify tool definitions are passed correctly -3. Add debug logging: - ```python - print(f"SDK Response: {sdk_response}") - ``` - -## Rollback Plan - -If you need to rollback to the old system: - -### Rollback to Direct API - -```env -# In .env -USE_DIRECT_API=true -USE_AGENT_SDK=false -ANTHROPIC_API_KEY=sk-ant-... -``` - -### Rollback to Legacy Server - -```env -# In .env -USE_CLAUDE_CODE_SERVER=true -CLAUDE_CODE_SERVER_URL=http://localhost:8000 - -# Start the server -python claude_code_server.py -``` - -### Rollback Code (if needed) - -```bash -# Reinstall old dependencies (FastAPI/Uvicorn) -pip install fastapi>=0.109.0 uvicorn>=0.27.0 - -# Revert to old requirements.txt (backup needed) -git checkout HEAD~1 requirements.txt -pip install -r requirements.txt -``` - -## Frequently Asked Questions - -### Q: Will this increase my costs? - -**A:** If you have Claude Pro, **costs will decrease to $0** for LLM calls. If you don't have Pro, you can keep using Direct API mode. - -### Q: Will this break my existing bot setup? - -**A:** No. All functionality is preserved: -- All 17 tools work identically -- Scheduled tasks unchanged -- Adapters (Telegram, Slack) unchanged -- Memory system unchanged -- Self-healing system unchanged - -### Q: Can I switch modes dynamically? - -**A:** Not currently. You need to set the mode in `.env` and restart the bot. - -### Q: Will usage tracking still work? - -**A:** Usage tracking is disabled for Agent SDK mode (no costs to track). It still works for Direct API mode. - -### Q: What about prompt caching? - -**A:** Prompt caching currently works only in Direct API mode. Agent SDK support may be added in the future. - -### Q: Can I use different modes for different bot instances? - -**A:** Yes! Each bot instance reads `.env` independently. You can run multiple bots with different modes. - -## Migration Checklist - -Use this checklist to ensure a smooth migration: - -### Pre-Migration -- [ ] Backup `.env` file -- [ ] Backup `requirements.txt` -- [ ] Note current mode (Direct API or Legacy Server) -- [ ] Verify bot is working correctly -- [ ] Document any custom configurations - -### Migration -- [ ] Update `requirements.txt` (or `git pull` latest) -- [ ] Install new dependencies (`pip install -r requirements.txt`) -- [ ] Update `.env` with new variables (if needed) -- [ ] Remove old variables (if migrating from legacy server) - -### Testing -- [ ] Run `python test_agent_sdk.py` -- [ ] Test simple chat -- [ ] Test tool usage (file operations) -- [ ] Test Gmail integration (if using) -- [ ] Test Calendar integration (if using) -- [ ] Test scheduled tasks -- [ ] Test with Telegram adapter (if using) -- [ ] Test with Slack adapter (if using) - -### Post-Migration -- [ ] Verify mode in startup logs (`[LLM] Using Claude Agent SDK...`) -- [ ] Monitor for errors in first 24 hours -- [ ] Verify scheduled tasks still run -- [ ] Check memory system working correctly -- [ ] Document any issues or edge cases - -### Cleanup (Optional) -- [ ] Remove unused legacy server code (if not needed) -- [ ] Remove `USE_CLAUDE_CODE_SERVER` from `.env` -- [ ] Uninstall FastAPI/Uvicorn (if not used elsewhere) -- [ ] Update documentation with new setup - -## Support - -If you encounter issues: - -1. **Check logs:** Look for `[LLM]` and `[Agent]` prefixed messages -2. **Run tests:** `python test_agent_sdk.py` -3. **Check mode:** Verify startup message shows correct mode -4. **Verify dependencies:** `pip list | grep claude-agent-sdk` -5. **Check .env:** Ensure no conflicting variables - -## Next Steps - -After successful migration: - -1. **Monitor performance:** Compare speed and response quality -2. **Track savings:** Calculate cost savings vs Direct API -3. **Report issues:** Document any bugs or edge cases -4. **Optimize:** Look for opportunities to leverage SDK features -5. **Share feedback:** Help improve the implementation - -## Version History - -### v1.0.0 (2026-02-15) -- Initial Agent SDK implementation -- Three-mode architecture -- Backward compatibility maintained -- Zero changes to agent.py, tools.py, adapters diff --git a/OBSIDIAN_MCP_SETUP_INSTRUCTIONS.md b/OBSIDIAN_MCP_SETUP_INSTRUCTIONS.md new file mode 100644 index 0000000..fb63fb7 --- /dev/null +++ b/OBSIDIAN_MCP_SETUP_INSTRUCTIONS.md @@ -0,0 +1,103 @@ +# Quick Setup: Obsidian Local REST API Plugin + +## Your Current Status +- ✅ Obsidian is running +- ✅ Config file is ready (`config/obsidian_mcp.yaml`) +- ❌ Local REST API plugin not responding on port 27123 + +## Setup Steps + +### 1. Install the Local REST API Plugin in Obsidian + +1. Open **Obsidian** +2. Go to **Settings** (gear icon) → **Community Plugins** +3. If you see "Safe mode is on", click **Turn off Safe Mode** +4. Click **Browse** button +5. Search for: **"Local REST API"** +6. Click **Install** on the "Local REST API" plugin by coddingtonbear +7. After installation, click **Enable** + +### 2. Configure the Plugin + +1. In Obsidian Settings, scroll down to **Plugin Options** +2. Find **Local REST API** in the left sidebar +3. Copy your API key shown in the plugin settings +4. Compare it with the key in your `config/obsidian_mcp.yaml`: + ``` + api_key: "ee625f06a778e3267a9219f9b8c1065a039375ea270e414a34436c6a3027f2da" + ``` +5. If they don't match, update the config file with the correct key + +### 3. Verify the Plugin is Running + +1. Check that the plugin shows as **enabled** in Obsidian +2. The plugin should show: "Server running on http://127.0.0.1:27123" +3. Restart Obsidian if needed + +### 4. Test the Connection + +Run this command in your project directory: + +```powershell +python -c "from obsidian_mcp import check_obsidian_health; print('Health Check:', check_obsidian_health(force=True))" +``` + +**Expected output**: `Health Check: True` + +### 5. Restart the Bot + +```powershell +venv\Scripts\activate +python bot_runner.py +``` + +Look for this line in the startup logs: +``` +[LLM] Obsidian MCP server registered (8 tools) +``` + +If you see this instead, the plugin isn't working yet: +``` +[LLM] Obsidian MCP enabled but health check failed - using custom tools only +``` + +## Alternative: File-Based Access (Already Working) + +If you don't want to use the Local REST API plugin, your bot can **already** access your Obsidian vault via the filesystem using these tools: + +- `fleeting_note` - Quick capture with auto-ID +- `daily_note` - Timestamped journal entries +- `literature_note` - Save web articles +- `permanent_note` - Create refined notes with auto-linking +- `search_vault` - Hybrid semantic search +- `search_by_tags` - Find notes by tags +- `read_file` / `write_file` / `edit_file` - Direct file access + +The **Obsidian MCP tools** add these extra capabilities: +- `obsidian_update_note` - Frontmatter-aware editing +- `obsidian_global_search` - Native Obsidian search +- `obsidian_manage_frontmatter` - Advanced metadata management +- `obsidian_manage_tags` - Bulk tag operations +- `obsidian_delete_note` - Safe deletion + +## Troubleshooting + +### Plugin shows "Server not running" +- Click the **Restart Server** button in the plugin settings +- Check Windows Firewall isn't blocking port 27123 + +### API key mismatch +- Copy the EXACT key from Obsidian plugin settings +- Update `config/obsidian_mcp.yaml` → `connection.api_key` + +### Wrong vault path +- Your current vault path: `C:/Users/fam1n/OneDrive/Documents/Remote-Mind-Vault` +- Verify this path exists and contains a `.obsidian` folder + +### Health check still fails after setup +- Restart Obsidian +- Restart the bot +- Check port 27123 isn't used by another program: + ```powershell + netstat -ano | findstr :27123 + ``` diff --git a/adapters/runtime.py b/adapters/runtime.py index 91a212d..f2b86fe 100644 --- a/adapters/runtime.py +++ b/adapters/runtime.py @@ -139,11 +139,37 @@ class AdapterRuntime: if adapter: await adapter.send_typing_indicator(message.channel_id) + # Capture the event loop for thread-safe progress updates + event_loop = asyncio.get_running_loop() + + # Create progress callback to send updates to the user + def progress_callback(update_message: str): + """Send progress updates to the user during long operations.""" + if adapter: + try: + # Create outbound message for progress update + progress_msg = OutboundMessage( + platform=message.platform, + channel_id=message.channel_id, + text=update_message, + thread_id=message.thread_id, + ) + # Run async send in a thread-safe way + # Use the captured event loop instead of get_running_loop() + # since this callback runs from a thread (agent.chat via to_thread) + asyncio.run_coroutine_threadsafe( + adapter.send_message(progress_msg), + event_loop + ) + except Exception as e: + print(f"[Runtime] Failed to send progress update: {e}") + # Get response from agent (synchronous call in thread) response = await asyncio.to_thread( self.agent.chat, user_message=processed_message.text, username=username, + progress_callback=progress_callback, ) # Apply postprocessors @@ -217,6 +243,14 @@ class AdapterRuntime: print("[Runtime] Starting adapter runtime...") await self.registry.start_all() + # Pass the main event loop to the LLM interface so that Agent SDK + # async calls (from worker threads created by asyncio.to_thread) + # can be scheduled back onto this loop via run_coroutine_threadsafe. + loop = asyncio.get_running_loop() + if hasattr(self.agent, 'llm') and hasattr(self.agent.llm, 'set_event_loop'): + self.agent.llm.set_event_loop(loop) + print("[Runtime] Event loop reference passed to LLM interface") + self._is_running = True self.message_loop_task = asyncio.create_task( self._process_message_queue() diff --git a/agent.py b/agent.py index f3a3bfa..4114c05 100644 --- a/agent.py +++ b/agent.py @@ -1,7 +1,8 @@ """AI Agent with Memory and LLM Integration.""" import threading -from typing import List, Optional +import time +from typing import List, Optional, Callable from hooks import HooksSystem from llm_interface import LLMInterface @@ -35,6 +36,8 @@ class Agent: self.conversation_history: List[dict] = [] self._chat_lock = threading.Lock() self.healing_system = SelfHealingSystem(self.memory, self) + self._progress_callback: Optional[Callable[[str], None]] = None + self._progress_timer: Optional[threading.Timer] = None # Sub-agent orchestration self.is_sub_agent = is_sub_agent @@ -194,13 +197,26 @@ class Agent: self.conversation_history = self.conversation_history[start_idx:] - def chat(self, user_message: str, username: str = "default") -> str: + def chat( + self, + user_message: str, + username: str = "default", + progress_callback: Optional[Callable[[str], None]] = None + ) -> str: """Chat with context from memory and tool use. Thread-safe: uses a lock to prevent concurrent modification of conversation history from multiple threads (e.g., scheduled tasks and live messages). + + Args: + user_message: The user's message + username: The user's name (default: "default") + progress_callback: Optional callback for sending progress updates during long operations """ + # Store the callback for use during the chat + self._progress_callback = progress_callback + # Handle model switching commands (no lock needed, read-only on history) if user_message.lower().startswith("/model "): model_name = user_message[7:].strip() @@ -225,48 +241,160 @@ class Agent: ) with self._chat_lock: - return self._chat_inner(user_message, username) + try: + return self._chat_inner(user_message, username) + finally: + # Clear the callback when done + self._progress_callback = None + + def _build_system_prompt(self, user_message: str, username: str) -> str: + """Build the system prompt with SOUL, user profile, and memory context.""" + if self.specialist_prompt: + return ( + f"{self.specialist_prompt}\n\n" + f"You have access to tools for file operations, command execution, " + f"web fetching, note-taking, and Google services. " + f"Use them to accomplish your specialized task efficiently." + ) + + soul = self.memory.get_soul() + user_profile = self.memory.get_user(username) + relevant_memory = self.memory.search_hybrid(user_message, max_results=5) + + memory_lines = [f"- {mem['snippet']}" for mem in relevant_memory] + return ( + f"{soul}\n\nUser Profile:\n{user_profile}\n\n" + f"Relevant Memory:\n" + "\n".join(memory_lines) + + f"\n\nYou have access to tools for file operations, command execution, " + f"web fetching, note-taking, and Google services (Gmail, Calendar, Contacts). " + f"Use them freely to help the user." + ) def _chat_inner(self, user_message: str, username: str) -> str: """Inner chat logic, called while holding _chat_lock.""" - # Use specialist prompt if this is a sub-agent, otherwise use full context - if self.specialist_prompt: - # Sub-agent: Use focused specialist prompt - system = ( - f"{self.specialist_prompt}\n\n" - f"You have access to {len(TOOL_DEFINITIONS)} tools. Use them to accomplish your specialized task. " - f"Stay focused on your specialty and complete the task efficiently." - ) - else: - # Main agent: Use full SOUL, user profile, and memory context - soul = self.memory.get_soul() - user_profile = self.memory.get_user(username) - relevant_memory = self.memory.search_hybrid(user_message, max_results=5) - - memory_lines = [f"- {mem['snippet']}" for mem in relevant_memory] - system = ( - f"{soul}\n\nUser Profile:\n{user_profile}\n\n" - f"Relevant Memory:\n" + "\n".join(memory_lines) + - f"\n\nYou have access to {len(TOOL_DEFINITIONS)} tools for file operations, " - f"command execution, and Google services. Use them freely to help the user. " - f"Note: You're running on a flat-rate Agent SDK subscription, so don't worry " - f"about API costs when making multiple tool calls or processing large contexts." - ) + system = self._build_system_prompt(user_message, username) self.conversation_history.append( {"role": "user", "content": user_message} ) - # Prune history to prevent unbounded growth self._prune_conversation_history() - # Tool execution loop + # In Agent SDK mode, query() handles tool calls automatically via MCP. + # The tool loop is only needed for Direct API mode. + if self.llm.mode == "agent_sdk": + return self._chat_agent_sdk(user_message, system) + else: + return self._chat_direct_api(user_message, system) + + def _send_progress_update(self, elapsed_seconds: int): + """Send a progress update if callback is set.""" + if self._progress_callback: + messages = [ + f"⏳ Still working... ({elapsed_seconds}s elapsed)", + f"🔄 Processing your request... ({elapsed_seconds}s)", + f"⚙️ Working on it, this might take a moment... ({elapsed_seconds}s)", + ] + # Rotate through messages + message = messages[(elapsed_seconds // 90) % len(messages)] + try: + self._progress_callback(message) + except Exception as e: + print(f"[Agent] Failed to send progress update: {e}") + + def _start_progress_updates(self): + """Start periodic progress updates (every 90 seconds).""" + def send_update(elapsed: int): + self._send_progress_update(elapsed) + # Schedule next update + self._progress_timer = threading.Timer(90.0, send_update, args=[elapsed + 90]) + self._progress_timer.daemon = True + self._progress_timer.start() + + # Send first update after 90 seconds + self._progress_timer = threading.Timer(90.0, send_update, args=[90]) + self._progress_timer.daemon = True + self._progress_timer.start() + + def _stop_progress_updates(self): + """Stop progress updates.""" + if self._progress_timer: + self._progress_timer.cancel() + self._progress_timer = None + + def _chat_agent_sdk(self, user_message: str, system: str) -> str: + """Chat using Agent SDK. Tools are handled automatically by MCP.""" + context_messages = self._get_context_messages(MAX_CONTEXT_MESSAGES) + + # Start progress updates + self._start_progress_updates() + + try: + # chat_with_tools() in Agent SDK mode returns a string directly. + # The SDK handles all tool calls via MCP servers internally. + response = self.llm.chat_with_tools( + context_messages, + tools=[], # Ignored in Agent SDK mode; tools come from MCP + system=system, + ) + except TimeoutError as e: + error_msg = "⏱️ Task timed out after 5 minutes. The task might be too complex - try breaking it into smaller steps." + print(f"[Agent] TIMEOUT: {error_msg}") + self.healing_system.capture_error( + error=e, + component="agent.py:_chat_agent_sdk", + intent="Calling Agent SDK for chat response (TIMEOUT)", + context={ + "model": self.llm.model, + "message_preview": user_message[:100], + "error_type": "timeout", + }, + ) + return error_msg + except Exception as e: + error_msg = f"Agent SDK error: {e}" + print(f"[Agent] {error_msg}") + self.healing_system.capture_error( + error=e, + component="agent.py:_chat_agent_sdk", + intent="Calling Agent SDK for chat response", + context={ + "model": self.llm.model, + "message_preview": user_message[:100], + }, + ) + return "Sorry, I encountered an error communicating with the AI model. Please try again." + finally: + # Always stop progress updates when done + self._stop_progress_updates() + + # In Agent SDK mode, response is always a string + final_response = response if isinstance(response, str) else str(response) + + if not final_response.strip(): + final_response = "(No response generated)" + + self.conversation_history.append( + {"role": "assistant", "content": final_response} + ) + + # Write compact summary to memory + compact_summary = self.memory.compact_conversation( + user_message=user_message, + assistant_response=final_response, + tools_used=None # SDK handles tools internally; we don't track them here + ) + self.memory.write_memory(compact_summary, daily=True) + + return final_response + + def _chat_direct_api(self, user_message: str, system: str) -> str: + """Chat using Direct API with manual tool execution loop.""" max_iterations = MAX_TOOL_ITERATIONS - # Enable caching for Sonnet to save 90% on repeated system prompts use_caching = "sonnet" in self.llm.model.lower() + tools_used = [] for iteration in range(max_iterations): - # Get recent messages, ensuring we don't break tool_use/tool_result pairs context_messages = self._get_context_messages(MAX_CONTEXT_MESSAGES) try: @@ -281,19 +409,17 @@ class Agent: print(f"[Agent] {error_msg}") self.healing_system.capture_error( error=e, - component="agent.py:_chat_inner", - intent="Calling LLM API for chat response", + component="agent.py:_chat_direct_api", + intent="Calling Direct API for chat response", context={ "model": self.llm.model, "message_preview": user_message[:100], "iteration": iteration, }, ) - return f"Sorry, I encountered an error communicating with the AI model. Please try again." + return "Sorry, I encountered an error communicating with the AI model. Please try again." - # Check stop reason if response.stop_reason == "end_turn": - # Extract text response text_content = [] for block in response.content: if block.type == "text": @@ -301,7 +427,6 @@ class Agent: final_response = "\n".join(text_content) - # Handle empty response if not final_response.strip(): final_response = "(No response generated)" @@ -309,17 +434,16 @@ class Agent: {"role": "assistant", "content": final_response} ) - preview = final_response[:MEMORY_RESPONSE_PREVIEW_LENGTH] - self.memory.write_memory( - f"**{username}**: {user_message}\n" - f"**Garvis**: {preview}...", - daily=True, + compact_summary = self.memory.compact_conversation( + user_message=user_message, + assistant_response=final_response, + tools_used=tools_used if tools_used else None ) + self.memory.write_memory(compact_summary, daily=True) return final_response elif response.stop_reason == "tool_use": - # Build assistant message with tool uses assistant_content = [] tool_uses = [] @@ -343,11 +467,11 @@ class Agent: "content": assistant_content }) - # Execute tools and build tool result message tool_results = [] for tool_use in tool_uses: + if tool_use.name not in tools_used: + tools_used.append(tool_use.name) result = execute_tool(tool_use.name, tool_use.input, healing_system=self.healing_system) - # Truncate large tool outputs to prevent token explosion if len(result) > 5000: result = result[:5000] + "\n... (output truncated)" print(f"[Tool] {tool_use.name}: {result[:100]}...") @@ -363,7 +487,6 @@ class Agent: }) else: - # Unexpected stop reason return f"Unexpected stop reason: {response.stop_reason}" return "Error: Maximum tool use iterations exceeded" diff --git a/config/gitea_config.example.yaml b/config/gitea_config.example.yaml new file mode 100644 index 0000000..9c309b2 --- /dev/null +++ b/config/gitea_config.example.yaml @@ -0,0 +1,22 @@ +# Gitea Configuration +# Copy to gitea_config.yaml and fill in your values +# +# cp config/gitea_config.example.yaml config/gitea_config.yaml + +# Gitea instance URL (no trailing slash) +base_url: "https://vulcan.apophisnetworking.net" + +# Personal Access Token for API authentication +# To generate a token: +# 1. Go to https://vulcan.apophisnetworking.net/user/settings/applications +# 2. Under "Manage Access Tokens", enter a token name (e.g., "garvis-bot") +# 3. Select permissions: at minimum, check "repo" (read) scope +# 4. Click "Generate Token" +# 5. Copy the token here (it is shown only once!) +token: "your_personal_access_token_here" + +# Default repository owner (used when repo is not specified in tool calls) +default_owner: "jramos" + +# Default repository name (used when repo is not specified in tool calls) +default_repo: "homelab" diff --git a/config/obsidian_mcp.example.yaml b/config/obsidian_mcp.example.yaml new file mode 100644 index 0000000..50b2cf0 --- /dev/null +++ b/config/obsidian_mcp.example.yaml @@ -0,0 +1,113 @@ +# Obsidian MCP Server Configuration +# =================================== +# This file configures the external Obsidian MCP server integration. +# +# Setup: +# 1. Copy this file: copy config\obsidian_mcp.example.yaml config\obsidian_mcp.yaml +# 2. Set your vault_path below +# 3. Ensure Node.js 20+ is installed: node --version +# 4. Restart the bot: python bot_runner.py +# +# The config file (obsidian_mcp.yaml) is gitignored to protect your vault path. +# See OBSIDIAN_MCP_INTEGRATION.md for full documentation. + +obsidian_mcp: + # ---- Core Settings ---- + + # Enable or disable the Obsidian MCP integration + # Set to false to disable without removing the config + enabled: true + + # Absolute path to your Obsidian vault directory + # This MUST be the root folder of your vault (contains .obsidian/ subfolder) + # + # Windows examples (use double backslashes OR forward slashes): + # "C:\\Users\\username\\Documents\\Obsidian\\MyVault" + # "C:/Users/username/Documents/Obsidian/MyVault" + # + # Linux/Mac example: + # "/home/username/obsidian-vault" + # + # To use the bot's built-in zettelkasten vault (same files as custom tools): + # "C:\\Users\\username\\projects\\ajarbot\\memory_workspace\\obsidian" + vault_path: "C:\\Users\\YOUR_USERNAME\\Documents\\Obsidian\\YOUR_VAULT" + + # ---- Server Settings ---- + + server: + # Command to launch the MCP server + # Default: "npx" (downloads obsidian-mcp on first run) + # Alternative: "node" (if installed globally) + command: "npx" + + # Arguments passed to the command + # The vault_path is appended automatically as the last argument + # Default: ["-y", "obsidian-mcp"] + # -y = auto-confirm npm package installation + args: ["-y", "obsidian-mcp"] + + # Server startup timeout in seconds + # Increase if npx is slow on first download + startup_timeout: 30 + + # ---- Permission Controls ---- + # Control which operations the bot is allowed to perform. + # Disable categories to restrict the bot's access to your vault. + + permissions: + # Read operations (safe, no changes to vault) + # Tools: read-note, search-vault, list-available-vaults, manage-tags + allow_read: true + + # Write operations (creates new files or modifies existing ones) + # Tools: create-note, edit-note + allow_write: true + + # Delete operations (permanently removes files) + # Tools: delete-note + # DISABLED by default for safety - enable only if you trust the bot + allow_delete: false + + # Move/rename operations (changes file paths) + # Tools: move-note + allow_move: true + + # Tag operations (modifies YAML frontmatter) + # Tools: add-tags, remove-tags, rename-tag + allow_tags: true + + # Directory operations (creates new folders) + # Tools: create-directory + allow_directories: true + + # ---- Safety Settings ---- + + safety: + # Require user confirmation before any write operation + # When true, the bot will ask "Are you sure?" before creating/editing notes + confirm_writes: false + + # Create a backup copy before deleting a note + # Backup is saved to .trash/ in the vault root + backup_before_delete: true + + # Maximum note content size in characters + # Prevents accidental creation of very large notes + max_note_size: 50000 + + # Directories the bot should never access or modify + # Paths are relative to the vault root + restricted_paths: + - ".obsidian" # Obsidian app configuration + - ".trash" # Obsidian trash folder + - ".git" # Git repository data (if vault is version-controlled) + +# ---- Environment Variable Overrides ---- +# These environment variables override the YAML settings above. +# Add them to your .env file if you prefer not to use YAML config. +# +# OBSIDIAN_VAULT_PATH=C:\Users\username\Documents\Obsidian\MyVault +# OBSIDIAN_MCP_ENABLED=true +# OBSIDIAN_MCP_COMMAND=npx +# OBSIDIAN_MCP_ALLOW_DELETE=false +# OBSIDIAN_MCP_CONFIRM_WRITES=false diff --git a/config/scheduled_tasks.yaml b/config/scheduled_tasks.yaml index 26a0cdc..b035b71 100644 --- a/config/scheduled_tasks.yaml +++ b/config/scheduled_tasks.yaml @@ -5,13 +5,23 @@ tasks: # Morning briefing - sent to Slack/Telegram - name: morning-weather prompt: | - Check the user profile (Jordan.md) for the location (Centennial, CO). Use the get_weather tool with OpenWeatherMap API to fetch the current weather. Format the report as: + Check the user profile (Jordan.md) for the location (Centennial, CO). Use the get_weather tool with OpenWeatherMap API to fetch the current weather. + + Also use web_fetch to get today's high/low from a weather service: + https://wttr.in/Centennial,CO?format=j1 + + Parse the JSON response to extract: + - maxtempF (today's high) + - mintempF (today's low) + + Format the report as: 🌤️ **Weather Report for Centennial, CO** - - Current: [current]°F - - High: [high]°F - - Low: [low]°F + - Current: [current]°F (feels like [feels_like]°F) + - Today's High: [high]°F + - Today's Low: [low]°F - Conditions: [conditions] + - Wind: [wind speed] mph - Recommendation: [brief clothing/activity suggestion] Keep it brief and friendly! @@ -20,6 +30,35 @@ tasks: send_to_platform: "telegram" send_to_channel: "8088983654" # Your Telegram user ID + # Daily Zettelkasten Review + - name: zettelkasten-daily-review + prompt: | + Time for your daily zettelkasten review! Help Jordan process fleeting notes: + + 1. Use search_by_tags to find all notes tagged with "fleeting" + 2. Show Jordan the list of fleeting notes captured today/recently + 3. For each note, ask: "Would you like to: + a) Process this into a permanent note + b) Keep as fleeting for now + c) Delete (not useful)" + + Format: + 📝 **Daily Zettelkasten Review** + + You have [X] fleeting notes to review: + + 1. [Title] - [first line of content] + 2. [Title] - [first line of content] + ... + + Reply with the number to process, or 'skip' to review later. + + Keep it conversational and low-pressure! + schedule: "daily 20:00" + enabled: true + send_to_platform: "telegram" + send_to_channel: "8088983654" + # Daily API cost report - name: daily-cost-report prompt: | diff --git a/gitea_tools/__init__.py b/gitea_tools/__init__.py new file mode 100644 index 0000000..902142f --- /dev/null +++ b/gitea_tools/__init__.py @@ -0,0 +1,5 @@ +"""Gitea Tools - Private Gitea repository access for ajarbot.""" + +from .client import GiteaClient + +__all__ = ["GiteaClient"] diff --git a/gitea_tools/client.py b/gitea_tools/client.py new file mode 100644 index 0000000..2217229 --- /dev/null +++ b/gitea_tools/client.py @@ -0,0 +1,597 @@ +"""Gitea API Client - Access private Gitea repositories. + +Uses Gitea's REST API (compatible with GitHub API v3) to read files, +list directories, search code, and get directory trees from private repos. + +Authentication via Personal Access Token configured in config/gitea_config.yaml. +""" + +import base64 +import logging +from pathlib import Path +from typing import Any, Dict, List, Optional + +import httpx +import yaml + +logger = logging.getLogger(__name__) + +# Config file path +_CONFIG_PATH = Path("config/gitea_config.yaml") + +# Request timeout (seconds) +_REQUEST_TIMEOUT = 10.0 + +# Maximum file size to return (1MB) +_MAX_FILE_SIZE = 1_000_000 + +# Maximum output characters (prevents token explosion) +_MAX_OUTPUT_CHARS = 5000 + + +class GiteaClient: + """Client for Gitea REST API with Personal Access Token authentication.""" + + def __init__( + self, + base_url: Optional[str] = None, + token: Optional[str] = None, + default_owner: Optional[str] = None, + default_repo: Optional[str] = None, + ) -> None: + """Initialize Gitea client. + + Args: + base_url: Gitea instance URL (e.g., "https://vulcan.apophisnetworking.net"). + token: Personal Access Token for authentication. + default_owner: Default repository owner (e.g., "jramos"). + default_repo: Default repository name (e.g., "homelab"). + + If arguments are not provided, reads from config/gitea_config.yaml. + """ + config = self._load_config() + + self.base_url = (base_url or config.get("base_url", "")).rstrip("/") + self.token = token or config.get("token", "") + self.default_owner = default_owner or config.get("default_owner", "") + self.default_repo = default_repo or config.get("default_repo", "") + + if not self.base_url: + raise ValueError( + "Gitea base_url not configured. " + "Set it in config/gitea_config.yaml or pass base_url argument." + ) + if not self.token: + raise ValueError( + "Gitea token not configured. " + "Create a Personal Access Token at " + f"{self.base_url}/user/settings/applications " + "and add it to config/gitea_config.yaml" + ) + + self.api_url = f"{self.base_url}/api/v1" + + logger.info( + "[Gitea] Client initialized: %s (default: %s/%s)", + self.base_url, + self.default_owner, + self.default_repo, + ) + + @staticmethod + def _load_config() -> Dict[str, Any]: + """Load configuration from YAML file.""" + if not _CONFIG_PATH.exists(): + logger.warning( + "[Gitea] Config file not found: %s. " + "Copy config/gitea_config.example.yaml to config/gitea_config.yaml", + _CONFIG_PATH, + ) + return {} + + try: + content = _CONFIG_PATH.read_text(encoding="utf-8") + config = yaml.safe_load(content) or {} + return config + except Exception as e: + logger.error("[Gitea] Failed to load config: %s", e) + return {} + + def _parse_repo( + self, + repo: Optional[str] = None, + owner: Optional[str] = None, + ) -> tuple: + """Parse owner/repo from various input formats. + + Args: + repo: Repository in "owner/repo" format, or just "repo" name. + owner: Explicit owner (overrides repo string parsing). + + Returns: + Tuple of (owner, repo) strings. + """ + if repo and "/" in repo: + parts = repo.split("/", 1) + parsed_owner = parts[0] + parsed_repo = parts[1] + else: + parsed_owner = owner or self.default_owner + parsed_repo = repo or self.default_repo + + if owner: + parsed_owner = owner + + if not parsed_owner or not parsed_repo: + raise ValueError( + f"Repository not specified. Provide repo as 'owner/repo' " + f"or configure default_owner/default_repo in gitea_config.yaml. " + f"Got owner='{parsed_owner}', repo='{parsed_repo}'" + ) + + return parsed_owner, parsed_repo + + def _headers(self) -> Dict[str, str]: + """Build request headers with authentication.""" + return { + "Authorization": f"token {self.token}", + "Accept": "application/json", + "User-Agent": "Garvis/1.0 (Ajarbot Gitea Integration)", + } + + async def _request( + self, + method: str, + endpoint: str, + params: Optional[Dict] = None, + ) -> Dict[str, Any]: + """Make an authenticated API request. + + Args: + method: HTTP method (GET, POST, etc.). + endpoint: API endpoint path (e.g., "/repos/jramos/homelab/contents/README.md"). + params: Optional query parameters. + + Returns: + Dict with "success" key and either "data" or "error". + """ + url = f"{self.api_url}{endpoint}" + + try: + async with httpx.AsyncClient( + timeout=_REQUEST_TIMEOUT, + follow_redirects=True, + verify=True, + headers=self._headers(), + ) as client: + response = await client.request(method, url, params=params) + + if response.status_code == 401: + return { + "success": False, + "error": ( + "Authentication failed (HTTP 401). " + "Check your Personal Access Token in config/gitea_config.yaml. " + f"Generate a new token at: {self.base_url}/user/settings/applications" + ), + } + elif response.status_code == 404: + return { + "success": False, + "error": f"Not found (HTTP 404): {endpoint}", + } + elif response.status_code >= 400: + return { + "success": False, + "error": f"HTTP {response.status_code}: {response.text[:200]}", + } + + data = response.json() + return {"success": True, "data": data} + + except httpx.TimeoutException: + return { + "success": False, + "error": f"Request to {self.base_url} timed out after {_REQUEST_TIMEOUT}s", + } + except httpx.ConnectError as e: + return { + "success": False, + "error": f"Connection failed to {self.base_url}: {e}", + } + except Exception as e: + return { + "success": False, + "error": f"Request failed: {str(e)}", + } + + async def get_file_content( + self, + file_path: str, + owner: Optional[str] = None, + repo: Optional[str] = None, + branch: Optional[str] = None, + ) -> Dict[str, Any]: + """Get raw file content from a repository. + + Uses Gitea Contents API: GET /repos/{owner}/{repo}/contents/{filepath} + + Args: + file_path: Path to file in repo (e.g., "scripts/proxmox_collector.py"). + owner: Repository owner (default: from config). + repo: Repository name or "owner/repo" (default: from config). + branch: Branch name (default: repo default branch). + + Returns: + Dict with "success", and either "content"/"metadata" or "error". + """ + parsed_owner, parsed_repo = self._parse_repo(repo, owner) + + # Normalize file path (remove leading slash) + file_path = file_path.lstrip("/") + + endpoint = f"/repos/{parsed_owner}/{parsed_repo}/contents/{file_path}" + params = {} + if branch: + params["ref"] = branch + + result = await self._request("GET", endpoint, params=params) + + if not result["success"]: + return result + + data = result["data"] + + # Handle case where path is a directory (returns a list) + if isinstance(data, list): + return { + "success": False, + "error": ( + f"'{file_path}' is a directory, not a file. " + f"Use gitea_list_files to browse directories." + ), + } + + # Check file size + file_size = data.get("size", 0) + if file_size > _MAX_FILE_SIZE: + return { + "success": True, + "content": ( + f"[File too large: {file_size:,} bytes ({file_size / 1024 / 1024:.1f} MB). " + f"Maximum is {_MAX_FILE_SIZE:,} bytes. " + f"Use the download URL to fetch it directly.]" + ), + "metadata": { + "name": data.get("name", ""), + "path": data.get("path", ""), + "size": file_size, + "download_url": data.get("download_url", ""), + "sha": data.get("sha", ""), + }, + } + + # Decode base64 content + encoded_content = data.get("content", "") + try: + content = base64.b64decode(encoded_content).decode("utf-8") + except (UnicodeDecodeError, Exception): + return { + "success": True, + "content": "[Binary file - cannot display as text]", + "metadata": { + "name": data.get("name", ""), + "path": data.get("path", ""), + "size": file_size, + "encoding": data.get("encoding", ""), + "download_url": data.get("download_url", ""), + }, + } + + # Truncate if too long + truncated = False + if len(content) > _MAX_OUTPUT_CHARS: + content = content[:_MAX_OUTPUT_CHARS] + "\n\n... (file truncated)" + truncated = True + + return { + "success": True, + "content": content, + "metadata": { + "name": data.get("name", ""), + "path": data.get("path", ""), + "size": file_size, + "sha": data.get("sha", ""), + "last_commit_sha": data.get("last_commit_sha", ""), + "download_url": data.get("download_url", ""), + "truncated": truncated, + }, + } + + async def list_files( + self, + path: str = "", + owner: Optional[str] = None, + repo: Optional[str] = None, + branch: Optional[str] = None, + ) -> Dict[str, Any]: + """List files and directories at a path in the repository. + + Uses Gitea Contents API: GET /repos/{owner}/{repo}/contents/{path} + + Args: + path: Directory path in repo (e.g., "scripts/"). Empty for root. + owner: Repository owner. + repo: Repository name or "owner/repo". + branch: Branch name. + + Returns: + Dict with "success" and either "files" list or "error". + """ + parsed_owner, parsed_repo = self._parse_repo(repo, owner) + + # Normalize path + path = path.strip("/") + + endpoint = f"/repos/{parsed_owner}/{parsed_repo}/contents/{path}" if path else f"/repos/{parsed_owner}/{parsed_repo}/contents" + params = {} + if branch: + params["ref"] = branch + + result = await self._request("GET", endpoint, params=params) + + if not result["success"]: + return result + + data = result["data"] + + # If it's a single file (not a directory), inform the user + if isinstance(data, dict): + return { + "success": False, + "error": ( + f"'{path}' is a file, not a directory. " + f"Use gitea_read_file to read file contents." + ), + } + + # Build file listing + files = [] + for item in data: + entry = { + "name": item.get("name", ""), + "type": item.get("type", ""), # "file" or "dir" + "path": item.get("path", ""), + "size": item.get("size", 0), + } + files.append(entry) + + # Sort: directories first, then files, alphabetically + files.sort(key=lambda f: (0 if f["type"] == "dir" else 1, f["name"].lower())) + + return { + "success": True, + "files": files, + "path": path or "/", + "repo": f"{parsed_owner}/{parsed_repo}", + "count": len(files), + } + + async def search_code( + self, + query: str, + owner: Optional[str] = None, + repo: Optional[str] = None, + ) -> Dict[str, Any]: + """Search for code in a repository. + + Uses Gitea Code Search API: GET /repos/{owner}/{repo}/topics (fallback) + or the general search: GET /repos/search + + Note: Gitea's code search depends on indexer configuration. + Falls back to repo-level search if code search is unavailable. + + Args: + query: Search query string. + owner: Repository owner. + repo: Repository name or "owner/repo". + + Returns: + Dict with "success" and either "results" or "error". + """ + parsed_owner, parsed_repo = self._parse_repo(repo, owner) + + # Try Gitea's code search endpoint first + # GET /repos/{owner}/{repo}/contents - search by traversing + # Gitea doesn't have a direct per-repo code search API like GitHub + # Use the global code search with repo filter + endpoint = "/repos/search" + params = { + "q": query, + "owner": parsed_owner, + "limit": 10, + } + + # First try: global code search (if Gitea has it enabled) + code_endpoint = f"/repos/{parsed_owner}/{parsed_repo}/git/grep" + code_params = {"query": query} + + # Gitea doesn't have a git grep API, use the topic/label search + # or fall back to listing + content search + # Best approach: use the Gitea search API + search_endpoint = "/repos/search" + search_params = { + "q": query, + "limit": 10, + } + + # For code search, Gitea's best option is the global search endpoint + # with topic filter. But for actual file content search, we need to + # traverse the tree and search file contents. + # Use a pragmatic approach: get the repo tree and search filenames + # and provide useful results. + + # Strategy: Get flat tree, filter by query in filename and path + tree_result = await self.get_tree( + owner=parsed_owner, + repo=parsed_repo, + recursive=True, + ) + + if not tree_result["success"]: + return tree_result + + entries = tree_result.get("entries", []) + query_lower = query.lower() + + # Search filenames and paths + matches = [] + for entry in entries: + path = entry.get("path", "") + if query_lower in path.lower(): + matches.append({ + "path": path, + "type": entry.get("type", ""), + "size": entry.get("size", 0), + "match_type": "filename", + }) + + # Limit results + matches = matches[:20] + + if not matches: + return { + "success": True, + "results": [], + "query": query, + "message": ( + f"No files matching '{query}' found in " + f"{parsed_owner}/{parsed_repo}. " + f"Note: This searches file/directory names only. " + f"For content search, read specific files with gitea_read_file." + ), + } + + return { + "success": True, + "results": matches, + "query": query, + "repo": f"{parsed_owner}/{parsed_repo}", + "count": len(matches), + } + + async def get_tree( + self, + owner: Optional[str] = None, + repo: Optional[str] = None, + branch: Optional[str] = None, + recursive: bool = False, + ) -> Dict[str, Any]: + """Get the directory tree of a repository. + + Uses Gitea Git Trees API: GET /repos/{owner}/{repo}/git/trees/{sha} + + Args: + owner: Repository owner. + repo: Repository name or "owner/repo". + branch: Branch name (default: repo default branch). + recursive: If True, get full recursive tree. + + Returns: + Dict with "success" and either "entries" list or "error". + """ + parsed_owner, parsed_repo = self._parse_repo(repo, owner) + + # First, get the branch SHA (or default branch) + ref = branch or "main" + branch_endpoint = f"/repos/{parsed_owner}/{parsed_repo}/branches/{ref}" + branch_result = await self._request("GET", branch_endpoint) + + if not branch_result["success"]: + # Try "master" as fallback + if not branch and "404" in branch_result.get("error", ""): + ref = "master" + branch_endpoint = f"/repos/{parsed_owner}/{parsed_repo}/branches/{ref}" + branch_result = await self._request("GET", branch_endpoint) + + if not branch_result["success"]: + return branch_result + + branch_data = branch_result["data"] + tree_sha = branch_data.get("commit", {}).get("id", "") + + if not tree_sha: + return { + "success": False, + "error": f"Could not get tree SHA for branch '{ref}'", + } + + # Get the tree + tree_endpoint = f"/repos/{parsed_owner}/{parsed_repo}/git/trees/{tree_sha}" + params = {} + if recursive: + params["recursive"] = "true" + + tree_result = await self._request("GET", tree_endpoint, params=params) + + if not tree_result["success"]: + return tree_result + + tree_data = tree_result["data"] + raw_entries = tree_data.get("tree", []) + + # Format entries + entries = [] + for entry in raw_entries: + entry_type = entry.get("type", "") + # Map git object types to readable types + if entry_type == "blob": + readable_type = "file" + elif entry_type == "tree": + readable_type = "dir" + else: + readable_type = entry_type + + entries.append({ + "path": entry.get("path", ""), + "type": readable_type, + "size": entry.get("size", 0), + "sha": entry.get("sha", ""), + }) + + # Sort: directories first, then files + entries.sort(key=lambda e: (0 if e["type"] == "dir" else 1, e["path"].lower())) + + return { + "success": True, + "entries": entries, + "branch": ref, + "repo": f"{parsed_owner}/{parsed_repo}", + "total": len(entries), + "truncated": tree_data.get("truncated", False), + } + + +# Singleton client instance (lazy-loaded) +_gitea_client: Optional[GiteaClient] = None + + +def get_gitea_client() -> Optional[GiteaClient]: + """Get or create the singleton Gitea client. + + Returns None if configuration is missing or invalid. + """ + global _gitea_client + + if _gitea_client is not None: + return _gitea_client + + try: + _gitea_client = GiteaClient() + return _gitea_client + except ValueError as e: + logger.warning("[Gitea] Client not available: %s", e) + return None + except Exception as e: + logger.error("[Gitea] Failed to initialize client: %s", e) + return None diff --git a/llm_interface.py b/llm_interface.py index 6904d0a..786f1db 100644 --- a/llm_interface.py +++ b/llm_interface.py @@ -1,42 +1,49 @@ """LLM Interface - Claude API, GLM, and other models. -Supports three modes for Claude: -1. Agent SDK (v0.1.36+) - DEFAULT - Uses query() API with Pro subscription +Supports two modes for Claude: +1. Agent SDK (v0.1.36+) - DEFAULT - Uses query() API with Max subscription - Set USE_AGENT_SDK=true (default) - Model: claude-sonnet-4-5-20250929 (default for all operations) - - Optional: USE_OPUS_FOR_TOOLS=true (enables Opus for extremely intensive tasks only) - - MCP Tools: File/system tools (read_file, write_file, edit_file, list_directory, run_command) - - Traditional Tools: Google tools & weather (fall back to Direct API, requires ANTHROPIC_API_KEY) - - Flat-rate subscription cost (no per-token charges for MCP tools) + - All tools are MCP-based (no API key needed) + - Tools registered via mcp_tools.py MCP server + - Flat-rate subscription cost 2. Direct API (pay-per-token) - Set USE_DIRECT_API=true - - Model: claude-sonnet-4-5-20250929 (cost-effective, never uses Opus) + - Model: claude-sonnet-4-5-20250929 - Requires ANTHROPIC_API_KEY in .env - - Full tool support built-in (all tools via traditional API) - -3. Legacy: Local Claude Code server - Set USE_CLAUDE_CODE_SERVER=true (deprecated) - - For backward compatibility only + - Uses traditional tool definitions from tools.py """ +import asyncio +import atexit +import logging import os -from typing import Any, Dict, List, Optional +import subprocess +import threading +from typing import Any, Dict, List, Optional, Set import requests from anthropic import Anthropic -from anthropic.types import Message, ContentBlock, TextBlock, ToolUseBlock, Usage - from usage_tracker import UsageTracker +logger = logging.getLogger(__name__) +# Ensure our debug messages are visible even if root logger is at WARNING. +# Only add a handler if none exist (prevent duplicate output). +if not logger.handlers: + _handler = logging.StreamHandler() + _handler.setFormatter(logging.Formatter( + "%(asctime)s [%(name)s] %(levelname)s: %(message)s", + datefmt="%H:%M:%S", + )) + logger.addHandler(_handler) + logger.setLevel(logging.DEBUG) + # Try to import Agent SDK (optional dependency) try: from claude_agent_sdk import ( - query, - UserMessage, - AssistantMessage, - SystemMessage, ClaudeAgentOptions, + ResultMessage, ) - import anyio AGENT_SDK_AVAILABLE = True except ImportError: AGENT_SDK_AVAILABLE = False @@ -47,29 +54,61 @@ _API_KEY_ENV_VARS = { "glm": "GLM_API_KEY", } -# Mode selection (priority order: USE_DIRECT_API > USE_CLAUDE_CODE_SERVER > default to Agent SDK) +# Mode selection (priority: USE_DIRECT_API > default to Agent SDK) _USE_DIRECT_API = os.getenv("USE_DIRECT_API", "false").lower() == "true" -_CLAUDE_CODE_SERVER_URL = os.getenv("CLAUDE_CODE_SERVER_URL", "http://localhost:8000") -_USE_CLAUDE_CODE_SERVER = os.getenv("USE_CLAUDE_CODE_SERVER", "false").lower() == "true" -# Agent SDK is the default if available and no other mode is explicitly enabled _USE_AGENT_SDK = os.getenv("USE_AGENT_SDK", "true").lower() == "true" # Default models by provider _DEFAULT_MODELS = { - "claude": "claude-sonnet-4-5-20250929", # For Direct API (pay-per-token) - Sonnet is cost-effective - "claude_agent_sdk": "claude-sonnet-4-5-20250929", # For Agent SDK (flat-rate) - Sonnet for normal operations - "claude_agent_sdk_opus": "claude-opus-4-6", # For Agent SDK extremely intensive tasks only (flat-rate) + "claude": "claude-sonnet-4-5-20250929", + "claude_agent_sdk": "claude-sonnet-4-5-20250929", "glm": "glm-4-plus", } -# When to use Opus (only on Agent SDK flat-rate mode) -_USE_OPUS_FOR_TOOLS = os.getenv("USE_OPUS_FOR_TOOLS", "false").lower() == "true" - _GLM_BASE_URL = "https://open.bigmodel.cn/api/paas/v4/chat/completions" +# Track PIDs of claude.exe subprocesses we spawn (to avoid killing user's Claude Code session!) +_TRACKED_CLAUDE_PIDS: Set[int] = set() +_TRACKED_PIDS_LOCK = threading.Lock() + + +def _register_claude_subprocess(pid: int): + """Register a claude.exe subprocess PID for cleanup on exit.""" + with _TRACKED_PIDS_LOCK: + _TRACKED_CLAUDE_PIDS.add(pid) + logger.debug("[LLM] Registered claude.exe subprocess PID: %d", pid) + + +def _cleanup_tracked_claude_processes(): + """Kill only the claude.exe processes we spawned (not the user's Claude Code session!)""" + with _TRACKED_PIDS_LOCK: + if not _TRACKED_CLAUDE_PIDS: + return + + logger.info("[LLM] Cleaning up %d tracked claude.exe subprocess(es)", len(_TRACKED_CLAUDE_PIDS)) + for pid in _TRACKED_CLAUDE_PIDS: + try: + if os.name == 'nt': # Windows + subprocess.run( + ['taskkill', '/F', '/PID', str(pid), '/T'], + capture_output=True, + timeout=2 + ) + else: # Linux/Mac + subprocess.run(['kill', '-9', str(pid)], capture_output=True, timeout=2) + logger.debug("[LLM] Killed claude.exe subprocess PID: %d", pid) + except Exception as e: + logger.debug("[LLM] Failed to kill PID %d: %s", pid, e) + + _TRACKED_CLAUDE_PIDS.clear() + + +# Register cleanup on exit (only kills our tracked subprocesses, not all claude.exe!) +atexit.register(_cleanup_tracked_claude_processes) + class LLMInterface: - """Simple LLM interface supporting Claude and GLM.""" + """LLM interface supporting Claude (Agent SDK or Direct API) and GLM.""" def __init__( self, @@ -82,26 +121,27 @@ class LLMInterface: _API_KEY_ENV_VARS.get(provider, ""), ) self.client: Optional[Anthropic] = None - # Model will be set after determining mode - # Determine mode (priority: direct API > legacy server > agent SDK) + # Reference to the main asyncio event loop, set by the runtime. + # Used by Agent SDK mode to schedule async work from worker threads + # via asyncio.run_coroutine_threadsafe(). + self._event_loop: Optional[asyncio.AbstractEventLoop] = None + + # Determine mode (priority: direct API > agent SDK) if provider == "claude": if _USE_DIRECT_API: self.mode = "direct_api" - elif _USE_CLAUDE_CODE_SERVER: - self.mode = "legacy_server" elif _USE_AGENT_SDK and AGENT_SDK_AVAILABLE: self.mode = "agent_sdk" else: - # Fallback to direct API if Agent SDK not available self.mode = "direct_api" if _USE_AGENT_SDK and not AGENT_SDK_AVAILABLE: print("[LLM] Warning: Agent SDK not available, falling back to Direct API") print("[LLM] Install with: pip install claude-agent-sdk") else: - self.mode = "direct_api" # Non-Claude providers use direct API + self.mode = "direct_api" - # Usage tracking (disabled when using Agent SDK or legacy server) + # Usage tracking (only for Direct API pay-per-token mode) self.tracker = UsageTracker() if (track_usage and self.mode == "direct_api") else None # Set model based on mode @@ -109,28 +149,125 @@ class LLMInterface: if self.mode == "agent_sdk": self.model = _DEFAULT_MODELS.get("claude_agent_sdk", "claude-sonnet-4-5-20250929") else: - self.model = _DEFAULT_MODELS.get(provider, "claude-haiku-4-5-20251001") + self.model = _DEFAULT_MODELS.get(provider, "claude-sonnet-4-5-20250929") else: self.model = _DEFAULT_MODELS.get(provider, "") # Initialize based on mode if provider == "claude": if self.mode == "agent_sdk": - print(f"[LLM] Using Claude Agent SDK (flat-rate subscription) with model: {self.model}") - # No initialization needed - query() is a standalone function + print(f"[LLM] Using Agent SDK (Max subscription) with model: {self.model}") elif self.mode == "direct_api": print(f"[LLM] Using Direct API (pay-per-token) with model: {self.model}") self.client = Anthropic(api_key=self.api_key) - elif self.mode == "legacy_server": - print(f"[LLM] Using Claude Code server at {_CLAUDE_CODE_SERVER_URL} (Pro subscription) with model: {self.model}") - # Verify server is running - try: - response = requests.get(f"{_CLAUDE_CODE_SERVER_URL}/", timeout=2) - response.raise_for_status() - print(f"[LLM] Claude Code server is running: {response.json()}") - except Exception as e: - print(f"[LLM] Warning: Could not connect to Claude Code server: {e}") - print(f"[LLM] Note: Claude Code server mode is deprecated. Using Agent SDK instead.") + + def set_event_loop(self, loop: asyncio.AbstractEventLoop) -> None: + """Store a reference to the main asyncio event loop. + + This allows Agent SDK async calls to be scheduled back onto the + main event loop from worker threads (created by asyncio.to_thread). + Must be called from the async context that owns the loop. + """ + self._event_loop = loop + logger.info( + "[LLM] Event loop stored: %s (running=%s)", + type(loop).__name__, + loop.is_running(), + ) + + @staticmethod + def _clean_claude_env() -> dict: + """Remove Claude Code session markers from the environment. + + The Agent SDK's SubprocessCLITransport copies os.environ into the + child process. If the bot is launched from within a Claude Code + session (or any environment that sets CLAUDECODE), the child + ``claude`` CLI detects the nesting and refuses to start with: + + "Claude Code cannot be launched inside another Claude Code session." + + This method temporarily removes the offending variables and returns + them so the caller can restore them afterwards. + """ + saved = {} + # Keys that signal an active Claude Code parent session. + # CLAUDE_CODE_ENTRYPOINT and CLAUDE_AGENT_SDK_VERSION are set by + # the SDK itself on the child process, so removing them from the + # parent is safe -- the SDK will set them again. + markers = [ + "CLAUDECODE", + "CLAUDE_CODE_ENTRYPOINT", + "CLAUDE_AGENT_SDK_VERSION", + "CLAUDE_CODE_ENABLE_SDK_FILE_CHECKPOINTING", + ] + for key in markers: + if key in os.environ: + saved[key] = os.environ.pop(key) + if saved: + logger.debug("[LLM] Cleaned Claude session env vars: %s", list(saved.keys())) + return saved + + @staticmethod + def _restore_claude_env(saved: dict) -> None: + """Restore previously removed Claude session env vars.""" + os.environ.update(saved) + + def _run_async_from_thread(self, coro) -> Any: + """Run an async coroutine from a synchronous worker thread. + + Uses asyncio.run_coroutine_threadsafe() to schedule the coroutine + on the main event loop (if available), which is the correct way to + bridge sync -> async when called from an asyncio.to_thread() worker + or from any background thread (e.g., the scheduler). + + Falls back to asyncio.run() if no event loop reference is available + (e.g., direct script usage without the adapter runtime). + + Args: + coro: An already-created coroutine object (not a coroutine function). + """ + current_thread = threading.current_thread().name + has_loop = self._event_loop is not None + loop_running = has_loop and self._event_loop.is_running() + + if has_loop and loop_running: + logger.info( + "[LLM] _run_async_from_thread: using run_coroutine_threadsafe " + "(thread=%s, loop=%s)", + current_thread, + type(self._event_loop).__name__, + ) + # Schedule on the main event loop and block this thread until done. + # This works because: + # 1. asyncio.to_thread() runs us in a thread pool while the main + # loop continues processing other tasks. + # 2. Scheduler threads are plain daemon threads, also not blocking + # the main loop. + # The coroutine executes on the main loop without deadlocking + # because the main loop is free to run while we block here. + future = asyncio.run_coroutine_threadsafe(coro, self._event_loop) + try: + # Block with 10-minute timeout to prevent hangs + # Complex tasks (repo analysis, multi-step operations) can take 5-8 minutes + logger.info("[LLM] Waiting for Agent SDK response (timeout: 600s)...") + result = future.result(timeout=600) + logger.info("[LLM] Agent SDK response received successfully") + return result + except TimeoutError: + logger.error("[LLM] ⚠️ Agent SDK call TIMED OUT after 600 seconds!") + future.cancel() # Cancel the coroutine + raise TimeoutError("Agent SDK call exceeded 10 minute timeout - task may be too complex") + else: + logger.info( + "[LLM] _run_async_from_thread: using asyncio.run() fallback " + "(thread=%s, has_loop=%s, loop_running=%s)", + current_thread, + has_loop, + loop_running, + ) + # Fallback: no main loop available (standalone / test usage). + # Create a new event loop in this thread via asyncio.run(). + return asyncio.run(coro) def chat( self, @@ -140,44 +277,24 @@ class LLMInterface: ) -> str: """Send chat request and get response. + In Agent SDK mode, this uses query() which handles MCP tools automatically. + In Direct API mode, this is a simple messages.create() call without tools. + Raises: Exception: If the API call fails or returns an unexpected response. """ if self.provider == "claude": - # Agent SDK mode (Pro subscription) if self.mode == "agent_sdk": try: - # Use anyio.run to create event loop for async SDK - response = anyio.run( - self._agent_sdk_chat, - messages, - system, - max_tokens + logger.info("[LLM] chat: dispatching via Agent SDK") + response = self._run_async_from_thread( + self._agent_sdk_chat(messages, system, max_tokens) ) return response except Exception as e: + logger.error("[LLM] Agent SDK error in chat(): %s", e, exc_info=True) raise Exception(f"Agent SDK error: {e}") - # Legacy Claude Code server (Pro subscription) - elif self.mode == "legacy_server": - try: - payload = { - "messages": [{"role": m["role"], "content": m["content"]} for m in messages], - "system": system, - "max_tokens": max_tokens - } - response = requests.post( - f"{_CLAUDE_CODE_SERVER_URL}/v1/chat", - json=payload, - timeout=120 - ) - response.raise_for_status() - data = response.json() - return data.get("content", "") - except Exception as e: - raise Exception(f"Claude Code server error: {e}") - - # Direct API (pay-per-token) elif self.mode == "direct_api": response = self.client.messages.create( model=self.model, @@ -186,7 +303,6 @@ class LLMInterface: messages=messages, ) - # Track usage if self.tracker and hasattr(response, "usage"): self.tracker.track( model=self.model, @@ -222,177 +338,263 @@ class LLMInterface: raise ValueError(f"Unsupported provider: {self.provider}") + def _build_agent_sdk_options(self) -> Optional['ClaudeAgentOptions']: + """Build Agent SDK options with MCP servers and allowed tools. + + Returns configured ClaudeAgentOptions, or None if mcp_tools is unavailable. + """ + try: + from mcp_tools import file_system_server + + mcp_servers = {"file_system": file_system_server} + + # All tools registered in the MCP server + allowed_tools = [ + # File and system tools + "read_file", + "write_file", + "edit_file", + "list_directory", + "run_command", + # Web tool + "web_fetch", + # Zettelkasten tools + "fleeting_note", + "daily_note", + "literature_note", + "permanent_note", + "search_vault", + "search_by_tags", + # Google tools (Gmail, Calendar, Contacts) + "get_weather", + "send_email", + "read_emails", + "get_email", + "read_calendar", + "create_calendar_event", + "search_calendar", + "create_contact", + "list_contacts", + "get_contact", + # Gitea tools (private repo access) + "gitea_read_file", + "gitea_list_files", + "gitea_search_code", + "gitea_get_tree", + ] + + # Conditionally add Obsidian MCP server + try: + from obsidian_mcp import ( + is_obsidian_enabled, + check_obsidian_health, + get_obsidian_server_config, + OBSIDIAN_TOOLS, + ) + + if is_obsidian_enabled() and check_obsidian_health(): + obsidian_config = get_obsidian_server_config() + mcp_servers["obsidian"] = obsidian_config + allowed_tools.extend(OBSIDIAN_TOOLS) + print("[LLM] Obsidian MCP server registered (8 tools)") + elif is_obsidian_enabled(): + print("[LLM] Obsidian MCP enabled but health check failed") + except ImportError: + pass + except Exception as e: + print(f"[LLM] Obsidian MCP unavailable: {e}") + + def _stderr_callback(line: str) -> None: + """Log Claude CLI stderr for debugging transport failures.""" + logger.debug("[CLI stderr] %s", line) + + return ClaudeAgentOptions( + mcp_servers=mcp_servers, + allowed_tools=allowed_tools, + permission_mode="bypassPermissions", + max_turns=30, # Prevent infinite tool loops (matches MAX_TOOL_ITERATIONS) + stderr=_stderr_callback, + ) + except ImportError: + print("[LLM] Warning: mcp_tools not available, no MCP tools will be registered") + return None + async def _agent_sdk_chat( self, messages: List[Dict], system: Optional[str], max_tokens: int ) -> str: - """Internal async method for Agent SDK chat (called via anyio bridge).""" - # Convert messages to SDK format - sdk_messages = [] - for msg in messages: - if msg["role"] == "user": - sdk_messages.append(UserMessage(content=msg["content"])) - elif msg["role"] == "assistant": - sdk_messages.append(AssistantMessage(content=msg["content"])) + """Agent SDK chat via custom transport flow. - # Add system message if provided - if system: - sdk_messages.insert(0, SystemMessage(content=system)) + Uses the SDK's transport and query layers directly instead of the + high-level ``query()`` helper. This works around a bug in + ``claude_agent_sdk._internal.client.process_query`` where + ``end_input()`` is called immediately after sending the user message + for string prompts. That premature stdin close kills the + bidirectional control channel that SDK MCP servers need to handle + ``tools/list`` and ``tools/call`` requests from the CLI subprocess, + resulting in ``CLIConnectionError: ProcessTransport is not ready for + writing``. - # Configure MCP server for file/system tools - try: - from mcp_tools import file_system_server + Our fix: defer ``end_input()`` until after the first ``ResultMessage`` + is received, matching the logic already present in + ``Query.stream_input()`` for async-iterable prompts. + """ + import json as _json - options = ClaudeAgentOptions( - mcp_servers={"file_system": file_system_server}, - # Allow all MCP tools (file/system + web + zettelkasten) - allowed_tools=[ - "read_file", - "write_file", - "edit_file", - "list_directory", - "run_command", - "web_fetch", - "fleeting_note", - "daily_note", - "literature_note", - "permanent_note", - "search_vault", - "search_by_tags", - ], - ) - except ImportError: - # Fallback if mcp_tools not available - options = None - - # Call the new query() API - # Note: Agent SDK handles max_tokens internally, don't pass it explicitly - response = await query( - messages=sdk_messages, - options=options, - # model parameter is handled by the SDK based on settings + # Lazy imports from SDK internals. + from claude_agent_sdk._internal.transport.subprocess_cli import ( + SubprocessCLITransport, ) + from claude_agent_sdk._internal.query import Query + from claude_agent_sdk._internal.message_parser import parse_message - # Extract text from response - if hasattr(response, "content"): - # Handle list of content blocks - if isinstance(response.content, list): - text_parts = [] - for block in response.content: - if hasattr(block, "text"): - text_parts.append(block.text) - return "".join(text_parts) - # Handle single text content - elif isinstance(response.content, str): - return response.content + # Build the prompt from the system prompt and conversation history. + prompt = self._build_sdk_prompt(messages, system) + options = self._build_agent_sdk_options() - return str(response) + # Clean Claude session env vars so the child CLI process doesn't + # detect a "nested session" and refuse to start. + saved_env = self._clean_claude_env() - async def _agent_sdk_chat_with_tools( + try: + # --- 1. Create and connect the subprocess transport. --- + transport = SubprocessCLITransport(prompt=prompt, options=options) + await transport.connect() + + # Track the subprocess PID for cleanup on exit + if hasattr(transport, '_process') and transport._process: + _register_claude_subprocess(transport._process.pid) + + # --- 2. Extract in-process SDK MCP server instances. --- + sdk_mcp_servers: Dict = {} + if options.mcp_servers and isinstance(options.mcp_servers, dict): + for name, config in options.mcp_servers.items(): + if isinstance(config, dict) and config.get("type") == "sdk": + sdk_mcp_servers[name] = config["instance"] + + # --- 3. Create the Query object (control-protocol handler). --- + query_obj = Query( + transport=transport, + is_streaming_mode=True, + sdk_mcp_servers=sdk_mcp_servers, + ) + + try: + # Start the background reader task. + await query_obj.start() + + # Perform the initialize handshake with the CLI. + await query_obj.initialize() + + # Send the user message over stdin. + user_msg = { + "type": "user", + "session_id": "", + "message": {"role": "user", "content": prompt}, + "parent_tool_use_id": None, + } + await transport.write(_json.dumps(user_msg) + "\n") + + # **KEY FIX**: Do NOT call end_input() yet. The CLI will + # send MCP control requests (tools/list, tools/call) over + # the bidirectional channel. Closing stdin now would + # prevent us from writing responses back. We wait for the + # first ResultMessage instead. + + # --- 4. Consume messages until we get a ResultMessage. --- + result_text = "" + message_count = 0 + async for data in query_obj.receive_messages(): + message = parse_message(data) + message_count += 1 + + # Log all message types for debugging hangs + message_type = type(message).__name__ + logger.debug(f"[LLM] Received message #{message_count}: {message_type}") + + if isinstance(message, ResultMessage): + result_text = message.result or "" + logger.info( + "[LLM] Agent SDK result received after %d messages: cost=$%.4f, turns=%s", + message_count, + getattr(message, "total_cost_usd", 0), + getattr(message, "num_turns", "?"), + ) + break + + # Log non-result messages to detect loops + if message_count % 10 == 0: + logger.warning(f"[LLM] Still waiting for ResultMessage after {message_count} messages...") + + # Now that we have the result, close stdin gracefully. + try: + await transport.end_input() + except Exception: + pass # Process may have already exited; that's fine. + + return result_text + + finally: + # Always clean up the query/transport. + try: + await query_obj.close() + except Exception: + # Suppress errors during cleanup (e.g. if process + # already exited and there are pending control + # request tasks that can't write back). + pass + finally: + # Always restore env vars, even on error. + self._restore_claude_env(saved_env) + + def _build_sdk_prompt( self, messages: List[Dict], - tools: List[Dict[str, Any]], system: Optional[str], - max_tokens: int - ) -> Message: - """Internal async method for Agent SDK chat with tools (called via anyio bridge). + ) -> str: + """Build a prompt string for the Agent SDK query() from conversation history. - NOTE: The new Claude Agent SDK (v0.1.36+) uses MCP servers for tools. - For backward compatibility with the existing tool system, we fall back - to the Direct API for tool calls. This means tool calls will consume API tokens - even when Agent SDK mode is enabled. - - Uses Sonnet by default. Opus can be enabled via USE_OPUS_FOR_TOOLS=true for - extremely intensive tasks (only recommended for Agent SDK flat-rate mode). + The SDK expects a single prompt string. We combine the system prompt + and conversation history into a coherent prompt. """ - # Fallback to Direct API for tool calls (SDK tools use MCP servers) - from anthropic import Anthropic + parts = [] - if not self.api_key: - raise ValueError( - "ANTHROPIC_API_KEY required for tool calls in Agent SDK mode. " - "Set the API key in .env or migrate tools to MCP servers." - ) + if system: + parts.append(f"\n{system}\n\n") - temp_client = Anthropic(api_key=self.api_key) + # Include recent conversation history for context + for msg in messages: + content = msg.get("content", "") + role = msg["role"] - # Use Opus only if explicitly enabled (for intensive tasks on flat-rate) - # Otherwise default to Sonnet (cost-effective for normal tool operations) - if _USE_OPUS_FOR_TOOLS and self.mode == "agent_sdk": - model = _DEFAULT_MODELS.get("claude_agent_sdk_opus", "claude-opus-4-6") - else: - model = self.model # Use Sonnet (default) + if isinstance(content, str): + if role == "user": + parts.append(f"User: {content}") + elif role == "assistant": + parts.append(f"Assistant: {content}") + elif isinstance(content, list): + # Structured content (tool_use/tool_result blocks from Direct API history) + text_parts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") == "tool_result": + text_parts.append(f"[Tool result]: {block.get('content', '')}") + elif block.get("type") == "tool_use": + text_parts.append(f"[Used tool: {block.get('name', 'unknown')}]") + elif hasattr(block, "type"): + if block.type == "text": + text_parts.append(block.text) + if text_parts: + if role == "user": + parts.append(f"User: {' '.join(text_parts)}") + elif role == "assistant": + parts.append(f"Assistant: {' '.join(text_parts)}") - response = temp_client.messages.create( - model=model, - max_tokens=max_tokens, - system=system or "", - messages=messages, - tools=tools, - ) - - return response - - def _convert_sdk_response_to_message(self, sdk_response: Dict[str, Any]) -> Message: - """Convert Agent SDK response to anthropic.types.Message format. - - This ensures compatibility with agent.py's existing tool loop. - """ - # Extract content blocks - content_blocks = [] - raw_content = sdk_response.get("content", []) - - if isinstance(raw_content, str): - # Simple text response - content_blocks = [TextBlock(type="text", text=raw_content)] - elif isinstance(raw_content, list): - # List of content blocks - for block in raw_content: - if isinstance(block, dict): - if block.get("type") == "text": - content_blocks.append(TextBlock( - type="text", - text=block.get("text", "") - )) - elif block.get("type") == "tool_use": - content_blocks.append(ToolUseBlock( - type="tool_use", - id=block.get("id", ""), - name=block.get("name", ""), - input=block.get("input", {}) - )) - - # Extract usage information - usage_data = sdk_response.get("usage", {}) - usage = Usage( - input_tokens=usage_data.get("input_tokens", 0), - output_tokens=usage_data.get("output_tokens", 0) - ) - - # Create Message object - # Note: We create a minimal Message-compatible object - # The Message class from anthropic.types is read-only, so we create a mock - # Capture self.model before defining inner class - model_name = sdk_response.get("model", self.model) - - class MessageLike: - def __init__(self, content, stop_reason, usage, model): - self.content = content - self.stop_reason = stop_reason - self.usage = usage - self.id = sdk_response.get("id", "sdk_message") - self.model = model - self.role = "assistant" - self.type = "message" - - return MessageLike( - content=content_blocks, - stop_reason=sdk_response.get("stop_reason", "end_turn"), - usage=usage, - model=model_name - ) + return "\n\n".join(parts) def chat_with_tools( self, @@ -401,70 +603,43 @@ class LLMInterface: system: Optional[str] = None, max_tokens: int = 16384, use_cache: bool = False, - ) -> Message: - """Send chat request with tool support. Returns full Message object. + ) -> Any: + """Send chat request with tool support. + + In Agent SDK mode: Uses query() with MCP tools. The SDK handles tool + execution automatically. Returns a string (final response after all + tool calls are resolved). + + In Direct API mode: Returns an anthropic Message object with potential + tool_use blocks that agent.py processes in a manual loop. Args: - use_cache: Enable prompt caching for Sonnet models (saves 90% on repeated context) + tools: Tool definitions (used by Direct API; ignored in Agent SDK mode + since tools are registered via MCP servers). + use_cache: Enable prompt caching for Sonnet (Direct API only). """ if self.provider != "claude": raise ValueError("Tool use only supported for Claude provider") - # Agent SDK mode (Pro subscription) if self.mode == "agent_sdk": + # Agent SDK handles tool calls automatically via MCP servers. + # We use the same query() path as chat(), since MCP tools are + # already registered. The SDK will invoke tools, collect results, + # and return the final text response. try: - # Use anyio.run to create event loop for async SDK - response = anyio.run( - self._agent_sdk_chat_with_tools, - messages, - tools, - system, - max_tokens + logger.info("[LLM] chat_with_tools: dispatching via Agent SDK") + response = self._run_async_from_thread( + self._agent_sdk_chat(messages, system, max_tokens) ) return response except Exception as e: + logger.error("[LLM] Agent SDK error: %s", e, exc_info=True) raise Exception(f"Agent SDK error: {e}") - # Legacy Claude Code server (Pro subscription) - elif self.mode == "legacy_server": - try: - payload = { - "messages": messages, - "tools": tools, - "system": system, - "max_tokens": max_tokens - } - response = requests.post( - f"{_CLAUDE_CODE_SERVER_URL}/v1/chat/tools", - json=payload, - timeout=120 - ) - response.raise_for_status() - # Convert response to Message-like object - data = response.json() - - # Create a mock Message object with the response - class MockMessage: - def __init__(self, data): - self.content = data.get("content", []) - self.stop_reason = data.get("stop_reason", "end_turn") - self.usage = type('obj', (object,), { - 'input_tokens': data.get("usage", {}).get("input_tokens", 0), - 'output_tokens': data.get("usage", {}).get("output_tokens", 0) - }) - - return MockMessage(data) - except Exception as e: - raise Exception(f"Claude Code server error: {e}") - - # Direct API (pay-per-token) elif self.mode == "direct_api": - # Enable caching only for Sonnet models (not worth it for Haiku) enable_caching = use_cache and "sonnet" in self.model.lower() - # Structure system prompt for optimal caching if enable_caching and system: - # Convert string to list format with cache control system_blocks = [ { "type": "text", @@ -483,7 +658,6 @@ class LLMInterface: tools=tools, ) - # Track usage if self.tracker and hasattr(response, "usage"): self.tracker.track( model=self.model, diff --git a/mcp_tools.py b/mcp_tools.py index 68f4353..b163a2c 100644 --- a/mcp_tools.py +++ b/mcp_tools.py @@ -1033,22 +1033,789 @@ async def search_vault_tool(args: Dict[str, Any]) -> Dict[str, Any]: } -# Create the MCP server with all tools (file/system + web + zettelkasten) +# ============================================ +# Google and Weather Tools (MCP Migration) +# ============================================ +# Lazy-loaded Google clients +_gmail_client: Optional[Any] = None +_calendar_client: Optional[Any] = None +_people_client: Optional[Any] = None + + +def _initialize_google_clients(): + """Lazy-load Google API clients when needed.""" + global _gmail_client, _calendar_client, _people_client + + if _gmail_client is not None: + return _gmail_client, _calendar_client, _people_client + + try: + from google_tools.gmail_client import GmailClient + from google_tools.calendar_client import CalendarClient + from google_tools.people_client import PeopleClient + from google_tools.oauth_manager import GoogleOAuthManager + + oauth_manager = GoogleOAuthManager() + credentials = oauth_manager.get_credentials() + + if not credentials: + return None, None, None + + _gmail_client = GmailClient(oauth_manager) + _calendar_client = CalendarClient(oauth_manager) + _people_client = PeopleClient(oauth_manager) + + return _gmail_client, _calendar_client, _people_client + except Exception as e: + print(f"[MCP Google] Failed to initialize: {e}") + return None, None, None + + +@tool( + name="get_weather", + description="Get current weather for a location using OpenWeatherMap API. Returns temperature, conditions, and description.", + input_schema={"location": str}, +) +async def get_weather(args: Dict[str, Any]) -> Dict[str, Any]: + """Get current weather for a location using OpenWeatherMap API.""" + location = args.get("location", "Phoenix, US") + import os + import requests + + api_key = os.getenv("OPENWEATHERMAP_API_KEY") + if not api_key: + return { + "content": [{ + "type": "text", + "text": "Error: OPENWEATHERMAP_API_KEY not found in environment variables" + }], + "isError": True + } + + try: + base_url = "http://api.openweathermap.org/data/2.5/weather" + params = { + "q": location, + "appid": api_key, + "units": "imperial" + } + + response = requests.get(base_url, params=params, timeout=10) + response.raise_for_status() + data = response.json() + + temp = data["main"]["temp"] + feels_like = data["main"]["feels_like"] + humidity = data["main"]["humidity"] + conditions = data["weather"][0]["main"] + description = data["weather"][0]["description"] + city_name = data["name"] + + summary = ( + f"Weather in {city_name}:\n" + f"Temperature: {temp}°F (feels like {feels_like}°F)\n" + f"Conditions: {conditions} - {description}\n" + f"Humidity: {humidity}%" + ) + + return { + "content": [{"type": "text", "text": summary}] + } + + except Exception as e: + return { + "content": [{ + "type": "text", + "text": f"Error getting weather: {str(e)}" + }], + "isError": True + } + + +@tool( + name="send_email", + description="Send an email via Gmail API. Requires prior OAuth setup (--setup-google).", + input_schema={"to": str, "subject": str, "body": str, "cc": str, "reply_to_message_id": str}, +) +async def send_email(args: Dict[str, Any]) -> Dict[str, Any]: + """Send an email via Gmail API.""" + to = args["to"] + subject = args["subject"] + body = args["body"] + cc = args.get("cc") + reply_to_message_id = args.get("reply_to_message_id") + + gmail_client, _, _ = _initialize_google_clients() + + if not gmail_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = gmail_client.send_email( + to=to, subject=subject, body=body, cc=cc, + reply_to_message_id=reply_to_message_id, + ) + + if result["success"]: + msg_id = result.get("message_id", "unknown") + text = f"Email sent successfully to {to}\nMessage ID: {msg_id}\nSubject: {subject}" + else: + text = f"Error sending email: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +@tool( + name="read_emails", + description="Search and read emails from Gmail using Gmail query syntax (e.g., 'from:user@example.com after:2026/02/10').", + input_schema={"query": str, "max_results": int, "include_body": bool}, +) +async def read_emails(args: Dict[str, Any]) -> Dict[str, Any]: + """Search and read emails from Gmail.""" + query = args.get("query", "") + max_results = args.get("max_results", 10) + include_body = args.get("include_body", False) + + gmail_client, _, _ = _initialize_google_clients() + + if not gmail_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = gmail_client.search_emails(query=query, max_results=max_results, include_body=include_body) + + if result["success"]: + summary = result.get("summary", "") + if len(summary) > _MAX_TOOL_OUTPUT: + summary = summary[:_MAX_TOOL_OUTPUT] + "\n... (results truncated)" + else: + summary = f"Error reading emails: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": summary}], "isError": not result["success"]} + + +@tool( + name="get_email", + description="Get full content of a specific email by its Gmail message ID.", + input_schema={"message_id": str, "format_type": str}, +) +async def get_email(args: Dict[str, Any]) -> Dict[str, Any]: + """Get full content of a specific email.""" + message_id = args["message_id"] + format_type = args.get("format_type", "text") + + gmail_client, _, _ = _initialize_google_clients() + + if not gmail_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = gmail_client.get_email(message_id=message_id, format_type=format_type) + + if result["success"]: + email_data = result.get("email", {}) + text = ( + f"From: {email_data.get('from', 'Unknown')}\n" + f"To: {email_data.get('to', 'Unknown')}\n" + f"Subject: {email_data.get('subject', 'No subject')}\n" + f"Date: {email_data.get('date', 'Unknown')}\n\n" + f"{email_data.get('body', 'No content')}" + ) + if len(text) > _MAX_TOOL_OUTPUT: + text = text[:_MAX_TOOL_OUTPUT] + "\n... (content truncated)" + else: + text = f"Error getting email: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +@tool( + name="read_calendar", + description="Read upcoming events from Google Calendar. Shows events from today onwards.", + input_schema={"days_ahead": int, "calendar_id": str, "max_results": int}, +) +async def read_calendar(args: Dict[str, Any]) -> Dict[str, Any]: + """Read upcoming calendar events.""" + days_ahead = args.get("days_ahead", 7) + calendar_id = args.get("calendar_id", "primary") + max_results = args.get("max_results", 20) + + _, calendar_client, _ = _initialize_google_clients() + + if not calendar_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = calendar_client.list_events( + days_ahead=days_ahead, calendar_id=calendar_id, max_results=max_results, + ) + + if result["success"]: + summary = result.get("summary", "No events found") + if len(summary) > _MAX_TOOL_OUTPUT: + summary = summary[:_MAX_TOOL_OUTPUT] + "\n... (results truncated)" + text = f"Upcoming events (next {days_ahead} days):\n\n{summary}" + else: + text = f"Error reading calendar: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +@tool( + name="create_calendar_event", + description="Create a new event in Google Calendar. Use ISO 8601 format for times.", + input_schema={ + "summary": str, "start_time": str, "end_time": str, + "description": str, "location": str, "calendar_id": str, + }, +) +async def create_calendar_event(args: Dict[str, Any]) -> Dict[str, Any]: + """Create a new calendar event.""" + summary = args["summary"] + start_time = args["start_time"] + end_time = args["end_time"] + description = args.get("description", "") + location = args.get("location", "") + calendar_id = args.get("calendar_id", "primary") + + _, calendar_client, _ = _initialize_google_clients() + + if not calendar_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = calendar_client.create_event( + summary=summary, start_time=start_time, end_time=end_time, + description=description, location=location, calendar_id=calendar_id, + ) + + if result["success"]: + event_id = result.get("event_id", "unknown") + html_link = result.get("html_link", "") + start = result.get("start", start_time) + text = ( + f"Calendar event created successfully!\n" + f"Title: {summary}\nStart: {start}\n" + f"Event ID: {event_id}\nLink: {html_link}" + ) + else: + text = f"Error creating calendar event: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +@tool( + name="search_calendar", + description="Search calendar events by text query. Searches event titles and descriptions.", + input_schema={"query": str, "calendar_id": str}, +) +async def search_calendar(args: Dict[str, Any]) -> Dict[str, Any]: + """Search calendar events by text query.""" + query = args["query"] + calendar_id = args.get("calendar_id", "primary") + + _, calendar_client, _ = _initialize_google_clients() + + if not calendar_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = calendar_client.search_events(query=query, calendar_id=calendar_id) + + if result["success"]: + summary = result.get("summary", "No events found") + if len(summary) > _MAX_TOOL_OUTPUT: + summary = summary[:_MAX_TOOL_OUTPUT] + "\n... (results truncated)" + text = f"Calendar search results for '{query}':\n\n{summary}" + else: + text = f"Error searching calendar: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +# ============================================ +# Contacts Tools (MCP) +# ============================================ + + +@tool( + name="create_contact", + description="Create a new Google contact. Requires prior OAuth setup (--setup-google).", + input_schema={"given_name": str, "family_name": str, "email": str, "phone": str, "notes": str}, +) +async def create_contact(args: Dict[str, Any]) -> Dict[str, Any]: + """Create a new Google contact.""" + given_name = args["given_name"] + family_name = args.get("family_name", "") + email = args.get("email", "") + phone = args.get("phone") + notes = args.get("notes") + + _, _, people_client = _initialize_google_clients() + + if not people_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = people_client.create_contact( + given_name=given_name, family_name=family_name, + email=email, phone=phone, notes=notes, + ) + + if result["success"]: + name = result.get("name", given_name) + resource = result.get("resource_name", "") + text = f"Contact created: {name}\nResource: {resource}" + else: + text = f"Error creating contact: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +@tool( + name="list_contacts", + description="List or search Google contacts. Without a query, lists all contacts sorted by last name.", + input_schema={"max_results": int, "query": str}, +) +async def list_contacts(args: Dict[str, Any]) -> Dict[str, Any]: + """List or search Google contacts.""" + max_results = args.get("max_results", 100) + query = args.get("query") + + _, _, people_client = _initialize_google_clients() + + if not people_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = people_client.list_contacts(max_results=max_results, query=query) + + if result["success"]: + summary = result.get("summary", "No contacts found.") + if len(summary) > _MAX_TOOL_OUTPUT: + summary = summary[:_MAX_TOOL_OUTPUT] + "\n... (results truncated)" + text = f"Contacts ({result.get('count', 0)} found):\n\n{summary}" + else: + text = f"Error listing contacts: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +@tool( + name="get_contact", + description="Get full details of a specific Google contact by resource name.", + input_schema={"resource_name": str}, +) +async def get_contact(args: Dict[str, Any]) -> Dict[str, Any]: + """Get full details of a specific Google contact.""" + resource_name = args["resource_name"] + + _, _, people_client = _initialize_google_clients() + + if not people_client: + return { + "content": [{"type": "text", "text": "Error: Google not authorized. Run: python bot_runner.py --setup-google"}], + "isError": True + } + + result = people_client.get_contact(resource_name=resource_name) + + if result["success"]: + c = result.get("contact", {}) + output = [] + name = c.get("display_name") or f"{c.get('given_name', '')} {c.get('family_name', '')}".strip() + output.append(f"Name: {name or '(no name)'}") + if c.get("email"): + output.append(f"Email: {c['email']}") + if c.get("phone"): + output.append(f"Phone: {c['phone']}") + if c.get("notes"): + output.append(f"Notes: {c['notes']}") + output.append(f"Resource: {c.get('resource_name', resource_name)}") + text = "\n".join(output) + else: + text = f"Error getting contact: {result.get('error', 'Unknown error')}" + + return {"content": [{"type": "text", "text": text}], "isError": not result["success"]} + + +# ============================================ +# Gitea Tools (MCP) - Private repo access +# ============================================ + +# Lazy-loaded Gitea client +_gitea_client: Optional[Any] = None + + +def _get_gitea_client(): + """Lazy-load Gitea client when first needed.""" + global _gitea_client + + if _gitea_client is not None: + return _gitea_client + + try: + from gitea_tools.client import get_gitea_client + _gitea_client = get_gitea_client() + return _gitea_client + except Exception as e: + print(f"[MCP Gitea] Failed to initialize: {e}") + return None + + +@tool( + name="gitea_read_file", + description="Read a file from a Gitea repository. Use this to access files from Jordan's homelab repo or any configured Gitea repo. Returns the file content as text.", + input_schema={ + "file_path": str, + "repo": str, + "branch": str, + }, +) +async def gitea_read_file_tool(args: Dict[str, Any]) -> Dict[str, Any]: + """Read a file from a Gitea repository. + + Zero-cost MCP tool for accessing private Gitea repos. + """ + file_path = args.get("file_path", "") + repo = args.get("repo") + branch = args.get("branch") + + if not file_path: + return { + "content": [{"type": "text", "text": "Error: file_path is required"}], + "isError": True, + } + + client = _get_gitea_client() + if not client: + return { + "content": [{ + "type": "text", + "text": ( + "Error: Gitea not configured. " + "Copy config/gitea_config.example.yaml to config/gitea_config.yaml " + "and add your Personal Access Token." + ), + }], + "isError": True, + } + + # Parse owner/repo if provided + owner = None + if repo and "/" in repo: + parts = repo.split("/", 1) + owner = parts[0] + repo = parts[1] + + result = await client.get_file_content( + file_path=file_path, + owner=owner, + repo=repo, + branch=branch, + ) + + if result["success"]: + content = result["content"] + metadata = result.get("metadata", {}) + path_info = metadata.get("path", file_path) + size = metadata.get("size", 0) + + header = f"File: {path_info} ({size:,} bytes)" + if metadata.get("truncated"): + header += " [TRUNCATED]" + + return { + "content": [{"type": "text", "text": f"{header}\n\n{content}"}], + } + else: + return { + "content": [{"type": "text", "text": f"Error: {result['error']}"}], + "isError": True, + } + + +@tool( + name="gitea_list_files", + description="List files and folders in a directory in a Gitea repository. Use this to explore the structure of Jordan's homelab repo or any configured Gitea repo.", + input_schema={ + "path": str, + "repo": str, + "branch": str, + }, +) +async def gitea_list_files_tool(args: Dict[str, Any]) -> Dict[str, Any]: + """List files and directories in a Gitea repo path. + + Zero-cost MCP tool for browsing private Gitea repos. + """ + path = args.get("path", "") + repo = args.get("repo") + branch = args.get("branch") + + client = _get_gitea_client() + if not client: + return { + "content": [{ + "type": "text", + "text": ( + "Error: Gitea not configured. " + "Copy config/gitea_config.example.yaml to config/gitea_config.yaml " + "and add your Personal Access Token." + ), + }], + "isError": True, + } + + # Parse owner/repo if provided + owner = None + if repo and "/" in repo: + parts = repo.split("/", 1) + owner = parts[0] + repo = parts[1] + + result = await client.list_files( + path=path, + owner=owner, + repo=repo, + branch=branch, + ) + + if result["success"]: + files = result["files"] + repo_name = result.get("repo", "") + display_path = result.get("path", "/") + count = result.get("count", 0) + + # Format output + lines = [f"Directory: {repo_name}/{display_path} ({count} items)\n"] + for f in files: + if f["type"] == "dir": + lines.append(f" DIR {f['name']}/") + else: + size_str = f"({f['size']:,} bytes)" if f["size"] else "" + lines.append(f" FILE {f['name']} {size_str}") + + return { + "content": [{"type": "text", "text": "\n".join(lines)}], + } + else: + return { + "content": [{"type": "text", "text": f"Error: {result['error']}"}], + "isError": True, + } + + +@tool( + name="gitea_search_code", + description="Search for files by name/path in a Gitea repository. Searches file and directory names. For content search, use gitea_read_file on specific files.", + input_schema={ + "query": str, + "repo": str, + }, +) +async def gitea_search_code_tool(args: Dict[str, Any]) -> Dict[str, Any]: + """Search for code/files in a Gitea repository. + + Zero-cost MCP tool. Searches file/directory names in the repo tree. + """ + query = args.get("query", "") + repo = args.get("repo") + + if not query: + return { + "content": [{"type": "text", "text": "Error: query is required"}], + "isError": True, + } + + client = _get_gitea_client() + if not client: + return { + "content": [{ + "type": "text", + "text": ( + "Error: Gitea not configured. " + "Copy config/gitea_config.example.yaml to config/gitea_config.yaml " + "and add your Personal Access Token." + ), + }], + "isError": True, + } + + # Parse owner/repo if provided + owner = None + if repo and "/" in repo: + parts = repo.split("/", 1) + owner = parts[0] + repo = parts[1] + + result = await client.search_code( + query=query, + owner=owner, + repo=repo, + ) + + if result["success"]: + results = result.get("results", []) + count = result.get("count", 0) + repo_name = result.get("repo", "") + + if not results: + message = result.get("message", f"No results for '{query}'") + return { + "content": [{"type": "text", "text": message}], + } + + lines = [f"Search results for '{query}' in {repo_name} ({count} matches):\n"] + for r in results: + type_icon = "DIR " if r["type"] == "dir" else "FILE" + size_str = f"({r['size']:,} bytes)" if r.get("size") else "" + lines.append(f" {type_icon} {r['path']} {size_str}") + + return { + "content": [{"type": "text", "text": "\n".join(lines)}], + } + else: + return { + "content": [{"type": "text", "text": f"Error: {result['error']}"}], + "isError": True, + } + + +@tool( + name="gitea_get_tree", + description="Get the directory tree structure from a Gitea repository. Shows all files and folders. Use recursive=true for the full tree.", + input_schema={ + "repo": str, + "branch": str, + "recursive": bool, + }, +) +async def gitea_get_tree_tool(args: Dict[str, Any]) -> Dict[str, Any]: + """Get directory tree from a Gitea repository. + + Zero-cost MCP tool for viewing repo structure. + """ + repo = args.get("repo") + branch = args.get("branch") + recursive = args.get("recursive", False) + + client = _get_gitea_client() + if not client: + return { + "content": [{ + "type": "text", + "text": ( + "Error: Gitea not configured. " + "Copy config/gitea_config.example.yaml to config/gitea_config.yaml " + "and add your Personal Access Token." + ), + }], + "isError": True, + } + + # Parse owner/repo if provided + owner = None + if repo and "/" in repo: + parts = repo.split("/", 1) + owner = parts[0] + repo = parts[1] + + result = await client.get_tree( + owner=owner, + repo=repo, + branch=branch, + recursive=recursive, + ) + + if result["success"]: + entries = result.get("entries", []) + repo_name = result.get("repo", "") + branch_name = result.get("branch", "main") + total = result.get("total", 0) + truncated = result.get("truncated", False) + + lines = [f"Tree: {repo_name} (branch: {branch_name}, {total} entries)"] + if truncated: + lines[0] += " [TRUNCATED - tree too large]" + lines.append("") + + for entry in entries: + if entry["type"] == "dir": + lines.append(f" {entry['path']}/") + else: + size_str = f"({entry['size']:,} bytes)" if entry.get("size") else "" + lines.append(f" {entry['path']} {size_str}") + + # Truncate output if too long + text = "\n".join(lines) + if len(text) > _MAX_TOOL_OUTPUT: + text = text[:_MAX_TOOL_OUTPUT] + "\n\n... (tree truncated, use gitea_list_files for specific directories)" + + return { + "content": [{"type": "text", "text": text}], + } + else: + return { + "content": [{"type": "text", "text": f"Error: {result['error']}"}], + "isError": True, + } + + +# Create the MCP server with all tools file_system_server = create_sdk_mcp_server( name="file_system", - version="1.4.0", + version="2.0.0", tools=[ + # File and system tools read_file_tool, write_file_tool, edit_file_tool, list_directory_tool, run_command_tool, + # Web tool web_fetch_tool, + # Zettelkasten tools fleeting_note_tool, daily_note_tool, literature_note_tool, permanent_note_tool, search_vault_tool, search_by_tags_tool, + # Weather + get_weather, + # Gmail tools + send_email, + read_emails, + get_email, + # Calendar tools + read_calendar, + create_calendar_event, + search_calendar, + # Contacts tools + create_contact, + list_contacts, + get_contact, + # Gitea tools + gitea_read_file_tool, + gitea_list_files_tool, + gitea_search_code_tool, + gitea_get_tree_tool, ] ) diff --git a/memory_system.py b/memory_system.py index 523f4fe..4121ab9 100644 --- a/memory_system.py +++ b/memory_system.py @@ -540,6 +540,44 @@ class MemorySystem: return sorted_results[:max_results] + def compact_conversation(self, user_message: str, assistant_response: str, tools_used: list = None) -> str: + """Create a compact summary of a conversation for memory storage. + + Args: + user_message: The user's input + assistant_response: The assistant's full response + tools_used: Optional list of tool names used (e.g., ['read_file', 'edit_file']) + + Returns: + Compact summary string + """ + # Extract file paths mentioned + import re + file_paths = re.findall(r'[a-zA-Z]:[\\\/][\w\\\/\-\.]+\.\w+|[\w\/\-\.]+\.(?:py|md|yaml|yml|json|txt|js|ts)', assistant_response) + file_paths = list(set(file_paths))[:5] # Limit to 5 unique paths + + # Truncate long responses + if len(assistant_response) > 300: + # Try to get first complete sentence or paragraph + sentences = assistant_response.split('. ') + if sentences and len(sentences[0]) < 200: + summary = sentences[0] + '.' + else: + summary = assistant_response[:200] + '...' + else: + summary = assistant_response + + # Build compact entry + compact = f"**User**: {user_message}\n**Action**: {summary}" + + if tools_used: + compact += f"\n**Tools**: {', '.join(tools_used)}" + + if file_paths: + compact += f"\n**Files**: {', '.join(file_paths[:3])}" # Max 3 file paths + + return compact + def write_memory(self, content: str, daily: bool = True) -> None: """Write to memory file.""" if daily: diff --git a/memory_workspace/MEMORY.md b/memory_workspace/MEMORY.md deleted file mode 100644 index c3f364b..0000000 --- a/memory_workspace/MEMORY.md +++ /dev/null @@ -1,98 +0,0 @@ -# MEMORY - Ajarbot Project Context - -## Project -Multi-platform AI agent with memory, cost-optimized for personal/small team use. Supports Slack, Telegram. - -## Core Stack -- **Memory**: Hybrid search (0.7 vector + 0.3 BM25), SQLite FTS5 + Markdown files -- **Embeddings**: FastEmbed all-MiniLM-L6-v2 (384-dim, local, $0) -- **LLM**: Claude (Haiku default, Sonnet w/ caching optional), GLM fallback -- **Platforms**: Slack (Socket Mode), Telegram (polling) -- **Tools**: File ops, shell commands (5 tools total) -- **Monitoring**: Pulse & Brain (92% cheaper than Heartbeat - deprecated) - -## Key Files -- `agent.py` - Main agent (memory + LLM + tools) -- `memory_system.py` - SQLite FTS5 + markdown sync -- `llm_interface.py` - Claude/GLM API wrapper -- `tools.py` - read_file, write_file, edit_file, list_directory, run_command -- `bot_runner.py` - Multi-platform launcher -- `scheduled_tasks.py` - Cron-like task scheduler - -## Memory Files -- `SOUL.md` - Agent personality (auto-loaded) -- `MEMORY.md` - This file (project context) -- `users/{username}.md` - Per-user preferences -- `memory/YYYY-MM-DD.md` - Daily logs -- `memory_index.db` - SQLite FTS5 index -- `vectors.usearch` - Vector embeddings for semantic search - -## Cost Optimizations (2026-02-13) -**Target**: Minimize API costs while maintaining capability - -### Active -- Default: Haiku 4.5 ($0.25 input/$1.25 output per 1M tokens) = 12x cheaper -- Prompt caching: Auto on Sonnet (90% savings on repeated prompts) -- Context: 3 messages max (was 5) -- Memory: 2 results per query (was 3) -- Tool iterations: 5 max (was 10) -- SOUL.md: 45 lines (was 87) - -### Commands -- `/haiku` - Switch to fast/cheap -- `/sonnet` - Switch to smart/cached -- `/status` - Show current config - -### Results -- Haiku: ~$0.001/message -- Sonnet cached: ~$0.003/message (after first) -- $5 free credits = hundreds of interactions - -## Search System -**IMPLEMENTED (2026-02-13)**: Hybrid semantic + keyword search -- 0.7 vector similarity + 0.3 BM25 weighted scoring -- FastEmbed all-MiniLM-L6-v2 (384-dim, runs locally, $0 cost) -- usearch for vector index, SQLite FTS5 for keywords -- ~15ms average query time -- +1.5KB per memory chunk for embeddings -- 10x better semantic retrieval vs keyword-only -- Example: "reduce costs" finds "Cost Optimizations" (old search: no results) -- Auto-generates embeddings on memory write -- Automatic in agent.chat() - no user action needed - -## Recent Changes -**2026-02-13**: Hybrid search implemented -- Added FastEmbed + usearch for semantic vector search -- Upgraded from keyword-only to 0.7 vector + 0.3 BM25 hybrid -- 59 embeddings generated for existing memories -- Memory recall improved 10x for conceptual queries -- Changed agent.py line 71: search() -> search_hybrid() -- Zero cost (local embeddings, no API calls) - -**2026-02-13**: Documentation cleanup -- Removed 3 redundant docs (HEARTBEAT_HOOKS, QUICK_START_PULSE, MONITORING_COMPARISON) -- Consolidated monitoring into PULSE_BRAIN.md -- Updated README for accuracy -- Sanitized repo (no API keys, user IDs committed) - -**2026-02-13**: Tool system added -- Bot can read/write/edit files, run commands autonomously -- Integrated into SOUL.md instructions - -**2026-02-13**: Task scheduler integrated -- Morning weather task (6am daily to Telegram user 8088983654) -- Config: `config/scheduled_tasks.yaml` - -## Architecture Decisions -- SQLite not Postgres: Simpler, adequate for personal bot -- Haiku default: Cost optimization priority -- Local embeddings (FastEmbed): Zero API calls, runs on device -- Hybrid search (0.7 vector + 0.3 BM25): Best of both worlds -- Markdown + DB: Simple, fast, no external deps -- Tool use: Autonomous action without user copy/paste - -## Deployment -- Platform: Windows 11 primary -- Git: https://vulcan.apophisnetworking.net/jramos/ajarbot.git -- Config: `.env` for API keys, `config/adapters.local.yaml` for tokens (both gitignored) -- Venv: Python 3.11+ diff --git a/memory_workspace/SOUL.md b/memory_workspace/SOUL.md deleted file mode 100644 index 276c1c4..0000000 --- a/memory_workspace/SOUL.md +++ /dev/null @@ -1,48 +0,0 @@ -# SOUL - Garvis Identity & Instructions - -## Identity -- **Name**: Garvis -- **Email**: ramosgarvis@gmail.com (my account, used for Gmail API) -- **Owner**: Jordan (see users/jordan.md for full profile) -- **Role**: Family personal assistant -- scheduling, weather, email, calendar, contacts, file management -- Helpful, concise, proactive. Value clarity and action over explanation. - -## Critical Behaviors -1. **Always check the user's profile** (users/{username}.md) before answering location/preference questions -2. **DO things, don't explain** -- use tools to accomplish tasks, not describe how to do them -3. **Remember context** -- if Jordan tells you something, update the user file or MEMORY.md -4. **Use MST timezone** for all scheduling (Jordan is in Centennial, CO) - -## Available Tools (17) -### File & System -- read_file, write_file, edit_file, list_directory, run_command - -### Weather -- get_weather (OpenWeatherMap API -- default location: Centennial, CO) - -### Gmail (ramosgarvis@gmail.com) -- send_email, read_emails, get_email - -### Google Calendar -- read_calendar, create_calendar_event, search_calendar - -### Google Contacts -- create_contact, list_contacts, get_contact - -**Principle**: Use tools freely -- this runs on a flat-rate subscription. Be thorough. - -## Scheduler Management -When users ask to schedule tasks, edit `config/scheduled_tasks.yaml` directly. -Schedule formats: `hourly`, `daily HH:MM`, `weekly day HH:MM` - -## Memory System -- SOUL.md: This file (identity + instructions) -- MEMORY.md: Project context and important facts -- users/{username}.md: Per-user preferences and info -- memory/YYYY-MM-DD.md: Daily conversation logs - -## Communication Style -- Concise, action-oriented (Jordan has ADHD/scanner personality) -- Break tasks into small chunks -- Vary language to maintain interest -- Frame suggestions as exploration opportunities, not obligations diff --git a/obsidian_mcp.py b/obsidian_mcp.py new file mode 100644 index 0000000..16e57b9 --- /dev/null +++ b/obsidian_mcp.py @@ -0,0 +1,168 @@ +"""Obsidian MCP Server Integration. + +Manages the external obsidian-mcp-server process and provides +health checking, fallback routing, and configuration loading. +""" + +import os +import time +import threading +from pathlib import Path +from typing import Any, Dict, Optional + +import yaml +import httpx + +# Default config path +_CONFIG_FILE = Path("config/obsidian_mcp.yaml") + +# Cached state +_obsidian_healthy: bool = False +_last_health_check: float = 0.0 +_health_lock = threading.Lock() +_config_cache: Optional[Dict] = None + + +def load_obsidian_config() -> Dict[str, Any]: + """Load Obsidian MCP configuration with env var overrides.""" + global _config_cache + + if _config_cache is not None: + return _config_cache + + config = {} + if _CONFIG_FILE.exists(): + with open(_CONFIG_FILE, encoding="utf-8") as f: + config = yaml.safe_load(f) or {} + + obsidian = config.get("obsidian_mcp", {}) + + # Apply environment variable overrides + env_overrides = { + "OBSIDIAN_API_KEY": ("connection", "api_key"), + "OBSIDIAN_BASE_URL": ("connection", "base_url"), + "OBSIDIAN_MCP_ENABLED": None, # Special: top-level "enabled" + "OBSIDIAN_ROUTING_STRATEGY": ("routing", "strategy"), + "OBSIDIAN_VAULT_PATH": ("vault", "path"), + } + + for env_var, path in env_overrides.items(): + value = os.getenv(env_var) + if value is None: + continue + + if path is None: + # Top-level key + obsidian["enabled"] = value.lower() in ("true", "1", "yes") + else: + section, key = path + obsidian.setdefault(section, {})[key] = value + + _config_cache = obsidian + return obsidian + + +def is_obsidian_enabled() -> bool: + """Check if Obsidian MCP integration is enabled in config.""" + config = load_obsidian_config() + return config.get("enabled", False) + + +def check_obsidian_health(force: bool = False) -> bool: + """Check if Obsidian REST API is reachable. + + Uses cached result unless force=True or cache has expired. + Thread-safe. + """ + global _obsidian_healthy, _last_health_check + + config = load_obsidian_config() + check_interval = config.get("routing", {}).get("health_check_interval", 60) + timeout = config.get("routing", {}).get("api_timeout", 10) + + with _health_lock: + now = time.time() + if not force and (now - _last_health_check) < check_interval: + return _obsidian_healthy + + base_url = config.get("connection", {}).get( + "base_url", "http://127.0.0.1:27123" + ) + api_key = config.get("connection", {}).get("api_key", "") + + try: + # Obsidian Local REST API health endpoint + response = httpx.get( + f"{base_url}/", + headers={"Authorization": f"Bearer {api_key}"}, + timeout=timeout, + verify=config.get("connection", {}).get("verify_ssl", False), + ) + _obsidian_healthy = response.status_code == 200 + except Exception: + _obsidian_healthy = False + + _last_health_check = now + return _obsidian_healthy + + +def get_obsidian_server_config() -> Dict[str, Any]: + """Build the MCP server configuration for Agent SDK registration. + + Returns the config dict suitable for ClaudeAgentOptions.mcp_servers. + The obsidian-mcp-server runs as a stdio subprocess. + """ + config = load_obsidian_config() + connection = config.get("connection", {}) + vault = config.get("vault", {}) + cache = config.get("cache", {}) + logging = config.get("logging", {}) + + env = { + "OBSIDIAN_API_KEY": connection.get("api_key", ""), + "OBSIDIAN_BASE_URL": connection.get( + "base_url", "http://127.0.0.1:27123" + ), + "OBSIDIAN_VERIFY_SSL": str( + connection.get("verify_ssl", False) + ).lower(), + "OBSIDIAN_VAULT_PATH": str(vault.get("path", "")), + "OBSIDIAN_ENABLE_CACHE": str( + cache.get("enabled", True) + ).lower(), + "OBSIDIAN_CACHE_REFRESH_INTERVAL_MIN": str( + cache.get("refresh_interval_min", 10) + ), + "MCP_LOG_LEVEL": logging.get("level", "info"), + } + + return { + "command": "npx", + "args": ["obsidian-mcp-server"], + "env": env, + } + + +def get_routing_strategy() -> str: + """Get the configured tool routing strategy.""" + config = load_obsidian_config() + return config.get("routing", {}).get("strategy", "obsidian_preferred") + + +def should_fallback_to_custom() -> bool: + """Check if fallback to custom tools is enabled.""" + config = load_obsidian_config() + return config.get("routing", {}).get("fallback_to_custom", True) + + +# List of all Obsidian MCP tool names +OBSIDIAN_TOOLS = [ + "obsidian_read_note", + "obsidian_update_note", + "obsidian_search_replace", + "obsidian_global_search", + "obsidian_list_notes", + "obsidian_manage_frontmatter", + "obsidian_manage_tags", + "obsidian_delete_note", +] diff --git a/scripts/collect-homelab-config.sh b/scripts/collect-homelab-config.sh new file mode 100644 index 0000000..68ff0a0 --- /dev/null +++ b/scripts/collect-homelab-config.sh @@ -0,0 +1,1023 @@ +#!/usr/bin/env bash + +################################################################################ +# Homelab Infrastructure Collection Script +# Version: 1.0.0 +# Purpose: Collects Proxmox VE configurations, system information, and exports +# infrastructure state in an organized, documented format +# +# Usage: ./collect-homelab-config.sh [OPTIONS] +# +# This script performs READ-ONLY operations and makes no modifications to your +# Proxmox environment. It is designed to be run directly on the Proxmox host +# or remotely via SSH. +################################################################################ + +set -euo pipefail # Exit on error, undefined variables, and pipe failures + +# Script metadata +SCRIPT_VERSION="1.0.0" +SCRIPT_NAME="$(basename "$0")" +TIMESTAMP="$(date +%Y%m%d-%H%M%S)" + +# Color codes for output (disabled if not a TTY) +if [[ -t 1 ]]; then + RED='\033[0;31m' + GREEN='\033[0;32m' + YELLOW='\033[1;33m' + BLUE='\033[0;34m' + MAGENTA='\033[0;35m' + CYAN='\033[0;36m' + BOLD='\033[1m' + NC='\033[0m' # No Color +else + RED='' GREEN='' YELLOW='' BLUE='' MAGENTA='' CYAN='' BOLD='' NC='' +fi + +################################################################################ +# Configuration Variables +################################################################################ + +# Default collection level: basic, standard, full, paranoid +COLLECTION_LEVEL="${COLLECTION_LEVEL:-standard}" + +# Sanitization options +SANITIZE_IPS="${SANITIZE_IPS:-false}" +SANITIZE_PASSWORDS="${SANITIZE_PASSWORDS:-true}" +SANITIZE_TOKENS="${SANITIZE_TOKENS:-true}" + +# Output configuration +OUTPUT_BASE_DIR="${OUTPUT_DIR:-./homelab-export-${TIMESTAMP}}" +COMPRESS_OUTPUT="${COMPRESS_OUTPUT:-true}" + +# Logging +LOG_FILE="${OUTPUT_BASE_DIR}/collection.log" +VERBOSE="${VERBOSE:-false}" + +# Summary tracking +COLLECTED_ITEMS=() +SKIPPED_ITEMS=() +ERROR_ITEMS=() + +################################################################################ +# Utility Functions +################################################################################ + +log() { + local level="$1" + shift + local message="$*" + local timestamp="$(date '+%Y-%m-%d %H:%M:%S')" + + echo "[${timestamp}] [${level}] ${message}" >> "${LOG_FILE}" 2>/dev/null || true + + case "${level}" in + INFO) + echo -e "${BLUE}[INFO]${NC} ${message}" + ;; + SUCCESS) + echo -e "${GREEN}[✓]${NC} ${message}" + ;; + WARN) + echo -e "${YELLOW}[WARN]${NC} ${message}" + ;; + ERROR) + echo -e "${RED}[ERROR]${NC} ${message}" >&2 + ;; + DEBUG) + if [[ "${VERBOSE}" == "true" ]]; then + echo -e "${MAGENTA}[DEBUG]${NC} ${message}" + fi + ;; + esac +} + +banner() { + local text="$1" + local width=80 + echo "" + echo -e "${BOLD}${CYAN}$(printf '=%.0s' $(seq 1 ${width}))${NC}" + echo -e "${BOLD}${CYAN} ${text}${NC}" + echo -e "${BOLD}${CYAN}$(printf '=%.0s' $(seq 1 ${width}))${NC}" + echo "" +} + +check_command() { + local cmd="$1" + if command -v "${cmd}" &> /dev/null; then + log DEBUG "Command '${cmd}' is available" + return 0 + else + log DEBUG "Command '${cmd}' is NOT available" + return 1 + fi +} + +check_proxmox() { + if [[ ! -f /etc/pve/.version ]]; then + log ERROR "This does not appear to be a Proxmox VE host" + log ERROR "/etc/pve/.version not found" + return 1 + fi + return 0 +} + +safe_copy() { + local source="$1" + local dest="$2" + local description="${3:-file}" + + if [[ -f "${source}" ]]; then + mkdir -p "$(dirname "${dest}")" + cp "${source}" "${dest}" 2>/dev/null && { + log SUCCESS "Collected ${description}" + COLLECTED_ITEMS+=("${description}") + return 0 + } || { + log WARN "Failed to copy ${description} from ${source}" + ERROR_ITEMS+=("${description}") + return 1 + } + elif [[ -d "${source}" ]]; then + mkdir -p "${dest}" + cp -r "${source}"/* "${dest}/" 2>/dev/null && { + log SUCCESS "Collected ${description}" + COLLECTED_ITEMS+=("${description}") + return 0 + } || { + log WARN "Failed to copy directory ${description} from ${source}" + ERROR_ITEMS+=("${description}") + return 1 + } + else + log DEBUG "Source does not exist: ${source} (${description})" + SKIPPED_ITEMS+=("${description}") + return 1 + fi +} + +safe_command() { + local output_file="$1" + local description="$2" + shift 2 + local cmd=("$@") + + mkdir -p "$(dirname "${output_file}")" + + if "${cmd[@]}" > "${output_file}" 2>/dev/null; then + log SUCCESS "Collected ${description}" + COLLECTED_ITEMS+=("${description}") + return 0 + else + log WARN "Failed to execute: ${cmd[*]} (${description})" + ERROR_ITEMS+=("${description}") + rm -f "${output_file}" + return 1 + fi +} + +sanitize_file() { + local file="$1" + + [[ ! -f "${file}" ]] && return 0 + + # Sanitize passwords + if [[ "${SANITIZE_PASSWORDS}" == "true" ]]; then + sed -i 's/password=.*/password=/g' "${file}" 2>/dev/null || true + sed -i 's/passwd=.*/passwd=/g' "${file}" 2>/dev/null || true + sed -i 's/"password"[[:space:]]*:[[:space:]]*"[^"]*"/"password": ""/g' "${file}" 2>/dev/null || true + fi + + # Sanitize tokens and keys + if [[ "${SANITIZE_TOKENS}" == "true" ]]; then + sed -i 's/token=.*/token=/g' "${file}" 2>/dev/null || true + sed -i 's/api[_-]key=.*/api_key=/g' "${file}" 2>/dev/null || true + sed -i 's/secret=.*/secret=/g' "${file}" 2>/dev/null || true + fi + + # Sanitize IP addresses (if requested) + if [[ "${SANITIZE_IPS}" == "true" ]]; then + # Replace IPv4 with 10.x.x.x equivalents + sed -i 's/\b\([0-9]\{1,3\}\.\)\{3\}[0-9]\{1,3\}\b/10.X.X.X/g' "${file}" 2>/dev/null || true + fi +} + +################################################################################ +# Directory Structure Creation +################################################################################ + +create_directory_structure() { + banner "Creating Directory Structure" + + local dirs=( + "${OUTPUT_BASE_DIR}" + "${OUTPUT_BASE_DIR}/docs" + "${OUTPUT_BASE_DIR}/configs/proxmox" + "${OUTPUT_BASE_DIR}/configs/vms" + "${OUTPUT_BASE_DIR}/configs/lxc" + "${OUTPUT_BASE_DIR}/configs/storage" + "${OUTPUT_BASE_DIR}/configs/network" + "${OUTPUT_BASE_DIR}/configs/backup" + "${OUTPUT_BASE_DIR}/exports/system" + "${OUTPUT_BASE_DIR}/exports/cluster" + "${OUTPUT_BASE_DIR}/exports/guests" + "${OUTPUT_BASE_DIR}/scripts" + "${OUTPUT_BASE_DIR}/diagrams" + ) + + for dir in "${dirs[@]}"; do + mkdir -p "${dir}" + log DEBUG "Created directory: ${dir}" + done + + # Initialize log file + mkdir -p "$(dirname "${LOG_FILE}")" + touch "${LOG_FILE}" + + log SUCCESS "Directory structure created at: ${OUTPUT_BASE_DIR}" +} + +################################################################################ +# Collection Functions +################################################################################ + +collect_system_information() { + banner "Collecting System Information" + + local sys_dir="${OUTPUT_BASE_DIR}/exports/system" + + # Proxmox version + safe_command "${sys_dir}/pve-version.txt" "Proxmox VE version" pveversion -v || true + + # System information + safe_command "${sys_dir}/hostname.txt" "Hostname" hostname || true + safe_command "${sys_dir}/uname.txt" "Kernel information" uname -a || true + safe_command "${sys_dir}/uptime.txt" "System uptime" uptime || true + safe_command "${sys_dir}/date.txt" "System date/time" date || true + + # CPU information + safe_command "${sys_dir}/cpuinfo.txt" "CPU information" lscpu || true + safe_copy "/proc/cpuinfo" "${sys_dir}/proc-cpuinfo.txt" "Detailed CPU info" || true + + # Memory information + safe_command "${sys_dir}/meminfo.txt" "Memory information" free -h || true + safe_copy "/proc/meminfo" "${sys_dir}/proc-meminfo.txt" "Detailed memory info" || true + + # Disk information + safe_command "${sys_dir}/df.txt" "Filesystem usage" df -h || true + safe_command "${sys_dir}/lsblk.txt" "Block devices" lsblk || true + + if check_command "pvdisplay"; then + safe_command "${sys_dir}/pvdisplay.txt" "LVM physical volumes" pvdisplay || true + safe_command "${sys_dir}/vgdisplay.txt" "LVM volume groups" vgdisplay || true + safe_command "${sys_dir}/lvdisplay.txt" "LVM logical volumes" lvdisplay || true + fi + + # Network information + safe_command "${sys_dir}/ip-addr.txt" "IP addresses" ip addr show || true + safe_command "${sys_dir}/ip-route.txt" "Routing table" ip route show || true + safe_command "${sys_dir}/ss-listening.txt" "Listening sockets" ss -tulpn || true + + # Installed packages + if check_command "dpkg"; then + safe_command "${sys_dir}/dpkg-list.txt" "Installed packages" dpkg -l || true + fi +} + +collect_proxmox_configs() { + banner "Collecting Proxmox Configurations" + + local pve_dir="${OUTPUT_BASE_DIR}/configs/proxmox" + + # Main Proxmox configuration files + safe_copy "/etc/pve/datacenter.cfg" "${pve_dir}/datacenter.cfg" "Datacenter config" || true + safe_copy "/etc/pve/storage.cfg" "${pve_dir}/storage.cfg" "Storage config" || true + safe_copy "/etc/pve/user.cfg" "${pve_dir}/user.cfg" "User config" || true + safe_copy "/etc/pve/domains.cfg" "${pve_dir}/domains.cfg" "Authentication domains" || true + safe_copy "/etc/pve/authkey.pub" "${pve_dir}/authkey.pub" "Auth public key" || true + + # Firewall configurations + if [[ -f /etc/pve/firewall/cluster.fw ]]; then + safe_copy "/etc/pve/firewall/cluster.fw" "${pve_dir}/firewall-cluster.fw" "Cluster firewall rules" || true + fi + + # Cluster configuration (if in a cluster) + if [[ -f /etc/pve/corosync.conf ]]; then + safe_copy "/etc/pve/corosync.conf" "${pve_dir}/corosync.conf" "Corosync config" || true + fi + + # HA configuration + if [[ -d /etc/pve/ha ]]; then + safe_copy "/etc/pve/ha" "${pve_dir}/ha" "HA configuration" || true + fi + + # Sanitize sensitive information + for file in "${pve_dir}"/*; do + [[ -f "${file}" ]] && sanitize_file "${file}" || true + done +} + +collect_vm_configs() { + banner "Collecting VM Configurations" + + local vm_dir="${OUTPUT_BASE_DIR}/configs/vms" + + # Get list of VMs + if [[ -d /etc/pve/nodes ]]; then + for node_dir in /etc/pve/nodes/*; do + local node_name="$(basename "${node_dir}")" + + if [[ -d "${node_dir}/qemu-server" ]]; then + for vm_config in "${node_dir}/qemu-server"/*.conf; do + [[ -f "${vm_config}" ]] || continue + + local vmid="$(basename "${vm_config}" .conf)" + local vm_name="$(grep -E '^name:' "${vm_config}" 2>/dev/null | cut -d' ' -f2 || echo "unknown")" + + safe_copy "${vm_config}" "${vm_dir}/${vmid}-${vm_name}.conf" "VM ${vmid} (${vm_name}) config" || true + + # Firewall rules for this VM + if [[ -f "${node_dir}/qemu-server/${vmid}.fw" ]]; then + safe_copy "${node_dir}/qemu-server/${vmid}.fw" "${vm_dir}/${vmid}-${vm_name}.fw" "VM ${vmid} firewall rules" || true + fi + done + fi + done + fi + + # Sanitize VM configs + for file in "${vm_dir}"/*; do + [[ -f "${file}" ]] && sanitize_file "${file}" || true + done +} + +collect_lxc_configs() { + banner "Collecting LXC Container Configurations" + + local lxc_dir="${OUTPUT_BASE_DIR}/configs/lxc" + + # Get list of containers + if [[ -d /etc/pve/nodes ]]; then + for node_dir in /etc/pve/nodes/*; do + local node_name="$(basename "${node_dir}")" + + if [[ -d "${node_dir}/lxc" ]]; then + for lxc_config in "${node_dir}/lxc"/*.conf; do + [[ -f "${lxc_config}" ]] || continue + + local ctid="$(basename "${lxc_config}" .conf)" + local ct_name="$(grep -E '^hostname:' "${lxc_config}" 2>/dev/null | cut -d' ' -f2 || echo "unknown")" + + safe_copy "${lxc_config}" "${lxc_dir}/${ctid}-${ct_name}.conf" "Container ${ctid} (${ct_name}) config" || true + + # Firewall rules for this container + if [[ -f "${node_dir}/lxc/${ctid}.fw" ]]; then + safe_copy "${node_dir}/lxc/${ctid}.fw" "${lxc_dir}/${ctid}-${ct_name}.fw" "Container ${ctid} firewall rules" || true + fi + done + fi + done + fi + + # Sanitize LXC configs + for file in "${lxc_dir}"/*; do + [[ -f "${file}" ]] && sanitize_file "${file}" || true + done +} + +collect_network_configs() { + banner "Collecting Network Configurations" + + local net_dir="${OUTPUT_BASE_DIR}/configs/network" + + # Network interface configurations + safe_copy "/etc/network/interfaces" "${net_dir}/interfaces" "Network interfaces config" || true + + if [[ -d /etc/network/interfaces.d ]]; then + safe_copy "/etc/network/interfaces.d" "${net_dir}/interfaces.d" "Additional interface configs" || true + fi + + # SDN configuration (Software Defined Networking) + if [[ -d /etc/pve/sdn ]]; then + safe_copy "/etc/pve/sdn" "${net_dir}/sdn" "SDN configuration" || true + fi + + # Hosts file + safe_copy "/etc/hosts" "${net_dir}/hosts" "Hosts file" || true + safe_copy "/etc/resolv.conf" "${net_dir}/resolv.conf" "DNS resolver config" || true + + # Sanitize network configs + for file in "${net_dir}"/*; do + [[ -f "${file}" ]] && sanitize_file "${file}" || true + done +} + +collect_storage_configs() { + banner "Collecting Storage Information" + + local storage_dir="${OUTPUT_BASE_DIR}/configs/storage" + + # Storage configuration is already in proxmox config, but let's get status + if check_command "pvesm"; then + safe_command "${storage_dir}/pvesm-status.txt" "Storage status" pvesm status || true + fi + + # ZFS pools (if any) + if check_command "zpool"; then + safe_command "${storage_dir}/zpool-status.txt" "ZFS pool status" zpool status || true + safe_command "${storage_dir}/zpool-list.txt" "ZFS pool list" zpool list || true + fi + + if check_command "zfs"; then + safe_command "${storage_dir}/zfs-list.txt" "ZFS datasets" zfs list || true + fi + + # NFS exports + if [[ -f /etc/exports ]]; then + safe_copy "/etc/exports" "${storage_dir}/nfs-exports" "NFS exports" || true + fi + + # Samba configuration + if [[ -f /etc/samba/smb.conf ]]; then + safe_copy "/etc/samba/smb.conf" "${storage_dir}/smb.conf" "Samba config" || true + fi + + # iSCSI configuration + if [[ -f /etc/iscsi/iscsid.conf ]]; then + safe_copy "/etc/iscsi/iscsid.conf" "${storage_dir}/iscsid.conf" "iSCSI initiator config" || true + fi +} + +collect_backup_configs() { + banner "Collecting Backup Configurations" + + local backup_dir="${OUTPUT_BASE_DIR}/configs/backup" + + # Vzdump configuration + if [[ -f /etc/vzdump.conf ]]; then + safe_copy "/etc/vzdump.conf" "${backup_dir}/vzdump.conf" "Vzdump config" || true + fi + + # Backup jobs + if [[ -d /etc/pve/jobs ]]; then + safe_copy "/etc/pve/jobs" "${backup_dir}/jobs" "Scheduled backup jobs" || true + fi + + # PBS configuration (if connected to Proxmox Backup Server) + if [[ -f /etc/pve/priv/storage.cfg ]]; then + grep -A5 "type: pbs" /etc/pve/priv/storage.cfg > "${backup_dir}/pbs-storage.txt" 2>/dev/null || true + fi +} + +collect_cluster_information() { + banner "Collecting Cluster Information" + + local cluster_dir="${OUTPUT_BASE_DIR}/exports/cluster" + + # Cluster status + if check_command "pvecm"; then + safe_command "${cluster_dir}/cluster-status.txt" "Cluster status" pvecm status || true + safe_command "${cluster_dir}/cluster-nodes.txt" "Cluster nodes" pvecm nodes || true + fi + + # Resource information + if check_command "pvesh"; then + safe_command "${cluster_dir}/cluster-resources.json" "Cluster resources" pvesh get /cluster/resources --output-format json + safe_command "${cluster_dir}/cluster-tasks.json" "Recent tasks" pvesh get /cluster/tasks --output-format json + fi +} + +collect_guest_information() { + banner "Collecting Guest Information" + + local guests_dir="${OUTPUT_BASE_DIR}/exports/guests" + + # List all VMs + if check_command "qm"; then + safe_command "${guests_dir}/vm-list.txt" "VM list" qm list + fi + + # List all containers + if check_command "pct"; then + safe_command "${guests_dir}/container-list.txt" "Container list" pct list + fi + + # Detailed guest information in JSON format + if check_command "pvesh"; then + safe_command "${guests_dir}/all-guests.json" "All guests (JSON)" pvesh get /cluster/resources --type vm --output-format json + fi +} + +collect_service_configs() { + banner "Collecting Service Configurations (Advanced)" + + # Only collect if level is 'full' or 'paranoid' + if [[ "${COLLECTION_LEVEL}" != "full" ]] && [[ "${COLLECTION_LEVEL}" != "paranoid" ]]; then + log INFO "Skipping service configs (collection level: ${COLLECTION_LEVEL})" + return 0 + fi + + local services_dir="${OUTPUT_BASE_DIR}/configs/services" + mkdir -p "${services_dir}" + + # Systemd service status + safe_command "${services_dir}/systemd-services.txt" "Systemd services" systemctl list-units --type=service --all + + # Collect specific Proxmox service configs + local pve_services=( + "pve-cluster" + "pvedaemon" + "pveproxy" + "pvestatd" + "pve-firewall" + "pvescheduler" + ) + + for service in "${pve_services[@]}"; do + if systemctl list-unit-files | grep -q "^${service}"; then + safe_command "${services_dir}/${service}-status.txt" "${service} status" systemctl status "${service}" || true + fi + done +} + +################################################################################ +# Documentation Generation +################################################################################ + +generate_readme() { + banner "Generating Documentation" + + local readme="${OUTPUT_BASE_DIR}/README.md" + + cat > "${readme}" <<'EOF' +# Homelab Infrastructure Export + +This directory contains a complete snapshot of your Proxmox-based homelab infrastructure, collected automatically via the homelab collection script. + +## Collection Information + +- **Collection Date**: $(date '+%Y-%m-%d %H:%M:%S') +- **Proxmox Node**: $(hostname) +- **Collection Level**: ${COLLECTION_LEVEL} +- **Sanitization Applied**: IPs=${SANITIZE_IPS}, Passwords=${SANITIZE_PASSWORDS}, Tokens=${SANITIZE_TOKENS} + +## Directory Structure + +``` +homelab-export-/ +├── README.md # This file +├── SUMMARY.md # Collection summary report +├── collection.log # Detailed collection log +├── configs/ # Configuration files +│ ├── proxmox/ # Proxmox VE configurations +│ ├── vms/ # Virtual machine configs +│ ├── lxc/ # LXC container configs +│ ├── storage/ # Storage configurations +│ ├── network/ # Network configurations +│ ├── backup/ # Backup job configurations +│ └── services/ # System service configs (if collected) +├── exports/ # System state exports +│ ├── system/ # System information +│ ├── cluster/ # Cluster status and resources +│ └── guests/ # Guest VM/CT information +├── docs/ # Documentation (for manual additions) +├── scripts/ # Automation scripts (for manual additions) +└── diagrams/ # Network diagrams (for manual additions) +``` + +## Configuration Files + +### Proxmox Core Configurations +- `datacenter.cfg` - Datacenter-wide settings +- `storage.cfg` - Storage pool definitions +- `user.cfg` - User and permission configurations +- `firewall-cluster.fw` - Cluster-level firewall rules + +### Virtual Machines +Each VM configuration is named: `-.conf` +Firewall rules (if present): `-.fw` + +### LXC Containers +Each container configuration is named: `-.conf` +Firewall rules (if present): `-.fw` + +## System Exports + +### System Information +- Proxmox version, hostname, kernel info +- CPU, memory, and disk information +- Network configuration and routing +- Installed packages + +### Cluster Information +- Cluster status and membership +- Resource allocation +- Recent tasks + +### Guest Information +- List of all VMs and containers +- Resource usage and status +- JSON exports for programmatic access + +## Security Notes + +This export may contain sensitive information depending on sanitization settings: + +- **Passwords**: ${SANITIZE_PASSWORDS} +- **API Tokens**: ${SANITIZE_TOKENS} +- **IP Addresses**: ${SANITIZE_IPS} + +**Recommendation**: Store this export securely. Do not commit to public repositories without careful review. + +## Using This Export + +### As Documentation +These files serve as a snapshot of your infrastructure at a point in time. Use them for: +- Documentation and disaster recovery +- Change tracking (diff with previous exports) +- Migration planning + +### Infrastructure as Code +Use the collected configurations to: +- Create Terraform/OpenTofu templates +- Build Ansible playbooks +- Document network architecture + +### Restoration Reference +In a disaster recovery scenario: +1. Reinstall Proxmox VE +2. Reference storage configuration from `configs/proxmox/storage.cfg` +3. Reference network setup from `configs/network/interfaces` +4. Recreate VMs/containers using configs in `configs/vms/` and `configs/lxc/` +5. Restore VM disk images from backups + +## Next Steps + +1. **Review the SUMMARY.md** for collection statistics +2. **Check collection.log** for any warnings or errors +3. **Manually add documentation** to the `docs/` folder +4. **Create network diagrams** and place in `diagrams/` +5. **Version control** this export in a private Git repository +6. **Set up regular collections** to track infrastructure changes + +## Collection Script + +This export was created by the Homelab Infrastructure Collection Script. +For questions or issues, consult the script documentation. + +--- +*Generated by homelab-export-script v${SCRIPT_VERSION}* +EOF + + # Perform variable substitution + eval "cat > \"${readme}\" < "${summary}" <> "${summary}" + done + + cat >> "${summary}" <> "${summary}" + done + else + echo "*None*" >> "${summary}" + fi + + cat >> "${summary}" <> "${summary}" + done + else + echo "*None*" >> "${summary}" + fi + + cat >> "${summary}" </dev/null || echo "Unable to retrieve version") +\`\`\` + +### Virtual Machines +\`\`\` +$(qm list 2>/dev/null || echo "Unable to retrieve VM list") +\`\`\` + +### Containers +\`\`\` +$(pct list 2>/dev/null || echo "Unable to retrieve container list") +\`\`\` + +### Storage +\`\`\` +$(pvesm status 2>/dev/null || echo "Unable to retrieve storage status") +\`\`\` + +### Disk Usage +\`\`\` +$(df -h 2>/dev/null || echo "Unable to retrieve disk usage") +\`\`\` + +## Next Actions + +1. Review any errors or skipped items above +2. Consult collection.log for detailed information +3. Manually verify sensitive information was sanitized +4. Add this export to your documentation repository +5. Create diagrams and additional documentation in respective folders + +--- +*Report generated $(date '+%Y-%m-%d %H:%M:%S')* +EOF + + log SUCCESS "Generated SUMMARY.md" +} + +################################################################################ +# Main Collection Orchestration +################################################################################ + +run_collection() { + banner "Starting Homelab Infrastructure Collection" + + log INFO "Collection Level: ${COLLECTION_LEVEL}" + log INFO "Output Directory: ${OUTPUT_BASE_DIR}" + log INFO "Sanitization: IPs=${SANITIZE_IPS} | Passwords=${SANITIZE_PASSWORDS} | Tokens=${SANITIZE_TOKENS}" + + # Check if we're on a Proxmox host + if ! check_proxmox; then + log ERROR "This script must be run on a Proxmox VE host" + exit 1 + fi + + # Create directory structure + create_directory_structure + + # System information (always collected) + collect_system_information + + # Proxmox configurations (always collected) + collect_proxmox_configs + + # VM and container configs (always collected) + collect_vm_configs + collect_lxc_configs + + # Network configurations (always collected) + collect_network_configs + + # Storage configurations (always collected) + collect_storage_configs + + # Backup configurations (standard and above) + if [[ "${COLLECTION_LEVEL}" != "basic" ]]; then + collect_backup_configs + collect_cluster_information + collect_guest_information + fi + + # Service configurations (full and paranoid) + collect_service_configs + + # Generate documentation + generate_readme + generate_summary + + banner "Collection Complete" + + log SUCCESS "Total items collected: ${#COLLECTED_ITEMS[@]}" + log INFO "Total items skipped: ${#SKIPPED_ITEMS[@]}" + + if [[ ${#ERROR_ITEMS[@]} -gt 0 ]]; then + log WARN "Total errors: ${#ERROR_ITEMS[@]}" + log WARN "Review ${LOG_FILE} for details" + fi + + # Compress output if requested + if [[ "${COMPRESS_OUTPUT}" == "true" ]]; then + banner "Compressing Export" + local archive="${OUTPUT_BASE_DIR}.tar.gz" + tar -czf "${archive}" -C "$(dirname "${OUTPUT_BASE_DIR}")" "$(basename "${OUTPUT_BASE_DIR}")" 2>/dev/null && { + log SUCCESS "Created archive: ${archive}" + log INFO "Archive size: $(du -h "${archive}" | cut -f1)" + } || { + log WARN "Failed to create archive" + } + fi + + echo "" + echo -e "${BOLD}${GREEN}Export Location:${NC} ${OUTPUT_BASE_DIR}" + echo -e "${BOLD}${GREEN}Summary Report:${NC} ${OUTPUT_BASE_DIR}/SUMMARY.md" + echo -e "${BOLD}${GREEN}Collection Log:${NC} ${LOG_FILE}" + echo "" +} + +################################################################################ +# Script Usage and Help +################################################################################ + +usage() { + cat <) + + -s, --sanitize WHAT Sanitize sensitive data. Options: + all - Sanitize everything (IPs, passwords, tokens) + ips - Sanitize IP addresses only + none - No sanitization + Default: passwords and tokens only + + -c, --compress Compress output to .tar.gz (default: true) + --no-compress Skip compression + + -v, --verbose Verbose output + -h, --help Show this help message + +${BOLD}COLLECTION LEVELS:${NC} + basic - System info, Proxmox configs, VM/CT configs + standard - Basic + storage, network, backup configs, cluster info + full - Standard + service configs, detailed system state + paranoid - Full + everything possible (experimental) + +${BOLD}EXAMPLES:${NC} + # Standard collection with default settings + ${SCRIPT_NAME} + + # Full collection with complete sanitization + ${SCRIPT_NAME} --level full --sanitize all + + # Basic collection without compression + ${SCRIPT_NAME} --level basic --no-compress + + # Custom output location with verbose logging + ${SCRIPT_NAME} -o /backup/homelab-export -v + +${BOLD}NOTES:${NC} + - Must be run on the Proxmox VE host (or via SSH) + - Requires root privileges for full access to configurations + - Output can be transferred to your documentation repository + - Review SUMMARY.md and collection.log after completion + +${BOLD}SECURITY:${NC} + By default, passwords and tokens are sanitized. Use --sanitize all + to also redact IP addresses. Review exported files before committing + to version control or sharing. + +For more information, consult the README.md generated with each export. +EOF +} + +################################################################################ +# Argument Parsing +################################################################################ + +parse_arguments() { + while [[ $# -gt 0 ]]; do + case "$1" in + -l|--level) + COLLECTION_LEVEL="$2" + shift 2 + ;; + -o|--output) + OUTPUT_BASE_DIR="$2" + LOG_FILE="${OUTPUT_BASE_DIR}/collection.log" + shift 2 + ;; + -s|--sanitize) + case "$2" in + all) + SANITIZE_IPS=true + SANITIZE_PASSWORDS=true + SANITIZE_TOKENS=true + ;; + ips) + SANITIZE_IPS=true + SANITIZE_PASSWORDS=false + SANITIZE_TOKENS=false + ;; + none) + SANITIZE_IPS=false + SANITIZE_PASSWORDS=false + SANITIZE_TOKENS=false + ;; + *) + echo "Invalid sanitization option: $2" + usage + exit 1 + ;; + esac + shift 2 + ;; + -c|--compress) + COMPRESS_OUTPUT=true + shift + ;; + --no-compress) + COMPRESS_OUTPUT=false + shift + ;; + -v|--verbose) + VERBOSE=true + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + echo "Unknown option: $1" + usage + exit 1 + ;; + esac + done + + # Validate collection level + case "${COLLECTION_LEVEL}" in + basic|standard|full|paranoid) + ;; + *) + echo "Invalid collection level: ${COLLECTION_LEVEL}" + usage + exit 1 + ;; + esac +} + +################################################################################ +# Main Execution +################################################################################ + +main() { + parse_arguments "$@" + run_collection +} + +# Check if script is being sourced or executed +if [[ "${BASH_SOURCE[0]}" == "${0}" ]]; then + main "$@" +fi diff --git a/scripts/collect-remote.sh b/scripts/collect-remote.sh new file mode 100644 index 0000000..4cfa079 --- /dev/null +++ b/scripts/collect-remote.sh @@ -0,0 +1,416 @@ +#!/usr/bin/env bash + +################################################################################ +# Remote Homelab Collection Wrapper +# Purpose: Executes the collection script on a remote Proxmox host via SSH +# and retrieves the results back to your local machine (WSL/Linux) +# +# Usage: ./collect-remote.sh [PROXMOX_HOST] [OPTIONS] +################################################################################ + +set -euo pipefail + +# Color codes +RED='\033[0;31m' +GREEN='\033[0;32m' +YELLOW='\033[1;33m' +BLUE='\033[0;34m' +CYAN='\033[0;36m' +BOLD='\033[1m' +NC='\033[0m' + +# Script configuration +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +COLLECTION_SCRIPT="${SCRIPT_DIR}/collect-homelab-config.sh" +REMOTE_SCRIPT_PATH="/tmp/collect-homelab-config.sh" +LOCAL_OUTPUT_DIR="${SCRIPT_DIR}" + +# SSH configuration +SSH_USER="${SSH_USER:-root}" +SSH_PORT="${SSH_PORT:-22}" +SSH_OPTS="-o ConnectTimeout=10 -o StrictHostKeyChecking=no" + +################################################################################ +# Functions +################################################################################ + +log() { + local level="$1" + shift + local message="$*" + + case "${level}" in + INFO) + echo -e "${BLUE}[INFO]${NC} ${message}" + ;; + SUCCESS) + echo -e "${GREEN}[✓]${NC} ${message}" + ;; + WARN) + echo -e "${YELLOW}[WARN]${NC} ${message}" + ;; + ERROR) + echo -e "${RED}[ERROR]${NC} ${message}" >&2 + ;; + esac +} + +banner() { + echo "" + echo -e "${BOLD}${CYAN}======================================================================${NC}" + echo -e "${BOLD}${CYAN} $1${NC}" + echo -e "${BOLD}${CYAN}======================================================================${NC}" + echo "" +} + +usage() { + cat < /dev/null; then + log ERROR "SSH client not found. Please install openssh-client" + exit 1 + fi + + # Check if scp is available + if ! command -v scp &> /dev/null; then + log ERROR "SCP not found. Please install openssh-client" + exit 1 + fi +} + +test_ssh_connection() { + local host="$1" + + log INFO "Testing SSH connection to ${SSH_USER}@${host}:${SSH_PORT}..." + + if ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" "exit 0" 2>/dev/null; then + log SUCCESS "SSH connection successful" + return 0 + else + log ERROR "Cannot connect to ${SSH_USER}@${host}:${SSH_PORT}" + log ERROR "Possible issues:" + log ERROR " - Host is unreachable" + log ERROR " - SSH service is not running" + log ERROR " - Incorrect credentials" + log ERROR " - Firewall blocking connection" + log ERROR "" + log ERROR "Try manually: ssh -p ${SSH_PORT} ${SSH_USER}@${host}" + return 1 + fi +} + +verify_proxmox_host() { + local host="$1" + + log INFO "Verifying Proxmox installation on remote host..." + + if ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" "test -f /etc/pve/.version" 2>/dev/null; then + local pve_version=$(ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" "cat /etc/pve/.version" 2>/dev/null) + log SUCCESS "Confirmed Proxmox VE installation (version: ${pve_version})" + return 0 + else + log WARN "Remote host does not appear to be a Proxmox VE server" + log WARN "Proceeding anyway, but collection may fail..." + return 0 + fi +} + +upload_script() { + local host="$1" + + banner "Uploading Collection Script" + + log INFO "Copying collection script to ${host}..." + + if scp ${SSH_OPTS} -P "${SSH_PORT}" "${COLLECTION_SCRIPT}" "${SSH_USER}@${host}:${REMOTE_SCRIPT_PATH}"; then + log SUCCESS "Script uploaded successfully" + + # Make executable + ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" "chmod +x ${REMOTE_SCRIPT_PATH}" + log SUCCESS "Script permissions set" + return 0 + else + log ERROR "Failed to upload script" + return 1 + fi +} + +execute_remote_collection() { + local host="$1" + shift + local collection_args=("$@") + + banner "Executing Collection on Remote Host" + + log INFO "Running collection script on ${host}..." + log INFO "Arguments: ${collection_args[*]}" + + # Build the remote command + local remote_cmd="${REMOTE_SCRIPT_PATH} ${collection_args[*]}" + + # Execute remotely and stream output + if ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" "${remote_cmd}"; then + log SUCCESS "Collection completed successfully on remote host" + return 0 + else + log ERROR "Collection failed on remote host" + return 1 + fi +} + +download_results() { + local host="$1" + local output_dir="$2" + + banner "Downloading Results" + + log INFO "Finding remote export archive..." + + # Find the most recent export archive + local remote_archive=$(ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" \ + "ls -t /root/homelab-export-*.tar.gz 2>/dev/null | head -1" 2>/dev/null) + + if [[ -z "${remote_archive}" ]]; then + log ERROR "No export archive found on remote host" + log ERROR "Collection may have failed or compression was disabled" + return 1 + fi + + log INFO "Found archive: ${remote_archive}" + + # Create output directory + mkdir -p "${output_dir}" + + # Download the archive + local local_archive="${output_dir}/$(basename "${remote_archive}")" + + log INFO "Downloading to: ${local_archive}" + + if scp ${SSH_OPTS} -P "${SSH_PORT}" "${SSH_USER}@${host}:${remote_archive}" "${local_archive}"; then + log SUCCESS "Archive downloaded successfully" + + # Extract the archive + log INFO "Extracting archive..." + if tar -xzf "${local_archive}" -C "${output_dir}"; then + log SUCCESS "Archive extracted to: ${output_dir}/$(basename "${local_archive}" .tar.gz)" + + # Show summary + local extracted_dir="${output_dir}/$(basename "${local_archive}" .tar.gz)" + if [[ -f "${extracted_dir}/SUMMARY.md" ]]; then + echo "" + log INFO "Collection Summary:" + echo "" + head -30 "${extracted_dir}/SUMMARY.md" + echo "" + log INFO "Full summary: ${extracted_dir}/SUMMARY.md" + fi + + return 0 + else + log ERROR "Failed to extract archive" + return 1 + fi + else + log ERROR "Failed to download archive" + return 1 + fi +} + +cleanup_remote() { + local host="$1" + local keep_remote="$2" + + if [[ "${keep_remote}" == "true" ]]; then + log INFO "Keeping export on remote host (--keep-remote specified)" + return 0 + fi + + banner "Cleaning Up Remote Host" + + log INFO "Removing export files from remote host..." + + # Remove the script + ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" "rm -f ${REMOTE_SCRIPT_PATH}" 2>/dev/null || true + + # Remove export directories and archives + ssh ${SSH_OPTS} -p "${SSH_PORT}" "${SSH_USER}@${host}" \ + "rm -rf /root/homelab-export-* 2>/dev/null" 2>/dev/null || true + + log SUCCESS "Remote cleanup completed" +} + +################################################################################ +# Main Execution +################################################################################ + +main() { + # Parse arguments + if [[ $# -eq 0 ]]; then + usage + exit 1 + fi + + local proxmox_host="" + local collection_level="standard" + local sanitize_option="" + local keep_remote="false" + local verbose="false" + + # First argument is the host + proxmox_host="$1" + shift + + # Parse remaining options + local collection_args=() + + while [[ $# -gt 0 ]]; do + case "$1" in + -u|--user) + SSH_USER="$2" + shift 2 + ;; + -p|--port) + SSH_PORT="$2" + shift 2 + ;; + -l|--level) + collection_level="$2" + collection_args+=("--level" "$2") + shift 2 + ;; + -s|--sanitize) + sanitize_option="$2" + collection_args+=("--sanitize" "$2") + shift 2 + ;; + -o|--output) + LOCAL_OUTPUT_DIR="$2" + shift 2 + ;; + -k|--keep-remote) + keep_remote="true" + shift + ;; + -v|--verbose) + verbose="true" + collection_args+=("--verbose") + shift + ;; + -h|--help) + usage + exit 0 + ;; + *) + log ERROR "Unknown option: $1" + usage + exit 1 + ;; + esac + done + + # Validate host + if [[ -z "${proxmox_host}" ]]; then + log ERROR "Proxmox host not specified" + usage + exit 1 + fi + + # Display configuration + banner "Remote Homelab Collection" + echo -e "${BOLD}Target Host:${NC} ${proxmox_host}" + echo -e "${BOLD}SSH User:${NC} ${SSH_USER}" + echo -e "${BOLD}SSH Port:${NC} ${SSH_PORT}" + echo -e "${BOLD}Collection Level:${NC} ${collection_level}" + echo -e "${BOLD}Output Directory:${NC} ${LOCAL_OUTPUT_DIR}" + echo -e "${BOLD}Keep Remote:${NC} ${keep_remote}" + echo "" + + # Execute workflow + check_prerequisites + test_ssh_connection "${proxmox_host}" || exit 1 + verify_proxmox_host "${proxmox_host}" + upload_script "${proxmox_host}" || exit 1 + execute_remote_collection "${proxmox_host}" "${collection_args[@]}" || exit 1 + download_results "${proxmox_host}" "${LOCAL_OUTPUT_DIR}" || exit 1 + cleanup_remote "${proxmox_host}" "${keep_remote}" + + banner "Collection Complete" + + log SUCCESS "Homelab infrastructure export completed successfully" + log INFO "Results are available in: ${LOCAL_OUTPUT_DIR}" + echo "" +} + +# Run main function +main "$@" diff --git a/scripts/collection_output.txt b/scripts/collection_output.txt new file mode 100644 index 0000000..59c2163 --- /dev/null +++ b/scripts/collection_output.txt @@ -0,0 +1,152 @@ +=== COLLECTION OUTPUT === + +================================================================================ + Starting Homelab Infrastructure Collection +================================================================================ + +[INFO] Collection Level: full +[INFO] Output Directory: /tmp/homelab-export +[INFO] Sanitization: IPs=false | Passwords=false | Tokens=false + +================================================================================ + Creating Directory Structure +================================================================================ + +[✓] Directory structure created at: /tmp/homelab-export + +================================================================================ + Collecting System Information +================================================================================ + +[✓] Collected Proxmox VE version +[✓] Collected Hostname +[✓] Collected Kernel information +[✓] Collected System uptime +[✓] Collected System date/time +[✓] Collected CPU information +[✓] Collected Detailed CPU info +[✓] Collected Memory information +[✓] Collected Detailed memory info +[✓] Collected Filesystem usage +[✓] Collected Block devices +[✓] Collected LVM physical volumes +[✓] Collected LVM volume groups +[✓] Collected LVM logical volumes +[✓] Collected IP addresses +[✓] Collected Routing table +[✓] Collected Listening sockets +[✓] Collected Installed packages + +================================================================================ + Collecting Proxmox Configurations +================================================================================ + +[✓] Collected Datacenter config +[✓] Collected Storage config +[✓] Collected User config +[✓] Collected Auth public key +[WARN] Failed to copy directory HA configuration from /etc/pve/ha + +================================================================================ + Collecting VM Configurations +================================================================================ + +[✓] Collected VM 100 (docker-hub) config +[✓] Collected VM 101 (monitoring-docker) config +[✓] Collected VM 104 (ubuntu-dev) config +[✓] Collected VM 105 (pfSense-Firewall) config +[✓] Collected VM 106 (Ansible-Control) config +[✓] Collected VM 107 (ubuntu-docker) config +[✓] Collected VM 108 (CML) config +[✓] Collected VM 114 (haos) config +[✓] Collected VM 119 (moltbot) config + +================================================================================ + Collecting LXC Container Configurations +================================================================================ + +[✓] Collected Container 102 (nginx) config +[✓] Collected Container 103 (netbox) config +[✓] Collected Container 112 (twingate-connector) config +[✓] Collected Container 113 (n8n +n8n +n8n) config +[✓] Collected Container 117 (test-cve-database) config + +================================================================================ + Collecting Network Configurations +================================================================================ + +[✓] Collected Network interfaces config +[WARN] Failed to copy directory Additional interface configs from /etc/network/interfaces.d +[✓] Collected SDN configuration +[✓] Collected Hosts file +[✓] Collected DNS resolver config + +================================================================================ + Collecting Storage Information +================================================================================ + +[✓] Collected Storage status +[✓] Collected ZFS pool status +[✓] Collected ZFS pool list +[✓] Collected ZFS datasets +[✓] Collected Samba config +[✓] Collected iSCSI initiator config + +================================================================================ + Collecting Backup Configurations +================================================================================ + +[✓] Collected Vzdump config + +================================================================================ + Collecting Cluster Information +================================================================================ + +[WARN] Failed to execute: pvecm status (Cluster status) +[WARN] Failed to execute: pvecm nodes (Cluster nodes) +[✓] Collected Cluster resources +[✓] Collected Recent tasks + +================================================================================ + Collecting Guest Information +================================================================================ + +[✓] Collected VM list +[✓] Collected Container list +[✓] Collected All guests (JSON) + +================================================================================ + Collecting Service Configurations (Advanced) +================================================================================ + +[✓] Collected Systemd services + +================================================================================ + Generating Documentation +================================================================================ + +[✓] Generated README.md + +================================================================================ + Generating Summary Report +================================================================================ + +[✓] Generated SUMMARY.md + +================================================================================ + Collection Complete +================================================================================ + +[✓] Total items collected: 53 +[INFO] Total items skipped: 1 +[WARN] Total errors: 4 +[WARN] Review /tmp/homelab-export/collection.log for details + +Export Location: /tmp/homelab-export +Summary Report: /tmp/homelab-export/SUMMARY.md +Collection Log: /tmp/homelab-export/collection.log + + +Exit code: 0 diff --git a/scripts/proxmox_ssh.py b/scripts/proxmox_ssh.py new file mode 100644 index 0000000..d00f8cc --- /dev/null +++ b/scripts/proxmox_ssh.py @@ -0,0 +1,53 @@ +#!/usr/bin/env python3 +"""Proxmox SSH Helper - serviceslab (192.168.2.100) +Uses paramiko for native Python SSH (no sshpass needed). +Usage: python proxmox_ssh.py "command to run" +""" + +import sys +import paramiko + +PROXMOX_HOST = "192.168.2.100" +PROXMOX_USER = "root" +PROXMOX_PASS = "Nbkx4mdmay1)" +PROXMOX_PORT = 22 +TIMEOUT = 15 + + +def run_command(command: str) -> tuple: + """Execute a command on the Proxmox server via SSH. + Returns (stdout, stderr, exit_code). + """ + client = paramiko.SSHClient() + client.set_missing_host_key_policy(paramiko.AutoAddPolicy()) + try: + client.connect( + hostname=PROXMOX_HOST, + port=PROXMOX_PORT, + username=PROXMOX_USER, + password=PROXMOX_PASS, + timeout=TIMEOUT, + look_for_keys=False, + allow_agent=False, + ) + stdin, stdout, stderr = client.exec_command(command, timeout=TIMEOUT) + exit_code = stdout.channel.recv_exit_status() + out = stdout.read().decode("utf-8", errors="replace") + err = stderr.read().decode("utf-8", errors="replace") + return out, err, exit_code + finally: + client.close() + + +if __name__ == "__main__": + if len(sys.argv) < 2: + print("Usage: python proxmox_ssh.py \"command\"") + sys.exit(1) + + cmd = sys.argv[1] + out, err, code = run_command(cmd) + if out: + print(out, end="") + if err: + print(err, end="", file=sys.stderr) + sys.exit(code) diff --git a/scripts/proxmox_ssh.sh b/scripts/proxmox_ssh.sh new file mode 100644 index 0000000..b592f0c --- /dev/null +++ b/scripts/proxmox_ssh.sh @@ -0,0 +1,9 @@ +#!/bin/bash +# Proxmox SSH Helper - serviceslab (192.168.2.100) +# Usage: proxmox_ssh.sh "command to run" + +PROXMOX_HOST="192.168.2.100" +PROXMOX_USER="root" +PROXMOX_PASS="Nbkx4mdmay1)" + +sshpass -p "$PROXMOX_PASS" ssh -o StrictHostKeyChecking=no -o ConnectTimeout=10 "${PROXMOX_USER}@${PROXMOX_HOST}" "$1" diff --git a/test_agent_sdk.py b/test_agent_sdk.py deleted file mode 100644 index 3b689e1..0000000 --- a/test_agent_sdk.py +++ /dev/null @@ -1,311 +0,0 @@ -"""Test script for Agent SDK implementation. - -This script tests the Agent SDK integration without running the full bot. -""" - -import os -import sys - -# Ensure we're testing the Agent SDK mode -os.environ["USE_AGENT_SDK"] = "true" -os.environ["USE_DIRECT_API"] = "false" -os.environ["USE_CLAUDE_CODE_SERVER"] = "false" - -def test_llm_interface_initialization(): - """Test 1: LLMInterface initialization with Agent SDK.""" - print("\n=== Test 1: LLMInterface Initialization ===") - try: - from llm_interface import LLMInterface - - llm = LLMInterface(provider="claude") - - print(f"✓ LLMInterface created successfully") - print(f" - Provider: {llm.provider}") - print(f" - Mode: {llm.mode}") - print(f" - Model: {llm.model}") - print(f" - Agent SDK available: {llm.agent_sdk is not None}") - - if llm.mode != "agent_sdk": - print(f"✗ WARNING: Expected mode 'agent_sdk', got '{llm.mode}'") - if llm.mode == "direct_api": - print(" - This likely means claude-agent-sdk is not installed") - print(" - Run: pip install claude-agent-sdk") - - return True - except Exception as e: - print(f"✗ Test failed: {e}") - import traceback - traceback.print_exc() - return False - - -def test_simple_chat(): - """Test 2: Simple chat without tools.""" - print("\n=== Test 2: Simple Chat (No Tools) ===") - try: - from llm_interface import LLMInterface - - llm = LLMInterface(provider="claude") - - if llm.mode != "agent_sdk": - print(f"⊘ Skipping test (mode is '{llm.mode}', not 'agent_sdk')") - return False - - print("Sending simple chat message...") - messages = [ - {"role": "user", "content": "Say 'Hello from Agent SDK!' in exactly those words."} - ] - - response = llm.chat(messages, system="You are a helpful assistant.", max_tokens=100) - - print(f"✓ Chat completed successfully") - print(f" - Response: {response[:100]}...") - print(f" - Response type: {type(response)}") - - return True - except Exception as e: - print(f"✗ Test failed: {e}") - import traceback - traceback.print_exc() - return False - - -def test_chat_with_tools(): - """Test 3: Chat with tools (message format compatibility).""" - print("\n=== Test 3: Chat with Tools ===") - try: - from llm_interface import LLMInterface - from tools import TOOL_DEFINITIONS - - llm = LLMInterface(provider="claude") - - if llm.mode != "agent_sdk": - print(f"⊘ Skipping test (mode is '{llm.mode}', not 'agent_sdk')") - return False - - print("Sending chat message with tool definitions...") - messages = [ - {"role": "user", "content": "What is 2+2? Just respond with the number, don't use any tools."} - ] - - response = llm.chat_with_tools( - messages, - tools=TOOL_DEFINITIONS, - system="You are a helpful assistant.", - max_tokens=100 - ) - - print(f"✓ Chat with tools completed successfully") - print(f" - Response type: {type(response)}") - print(f" - Has .content: {hasattr(response, 'content')}") - print(f" - Has .stop_reason: {hasattr(response, 'stop_reason')}") - print(f" - Has .usage: {hasattr(response, 'usage')}") - print(f" - Stop reason: {response.stop_reason}") - - if hasattr(response, 'content') and response.content: - print(f" - Content blocks: {len(response.content)}") - for i, block in enumerate(response.content): - print(f" - Block {i}: {type(block).__name__}") - if hasattr(block, 'type'): - print(f" - Type: {block.type}") - if hasattr(block, 'text'): - print(f" - Text: {block.text[:50]}...") - - return True - except Exception as e: - print(f"✗ Test failed: {e}") - import traceback - traceback.print_exc() - return False - - -def test_response_format_compatibility(): - """Test 4: Verify response format matches what agent.py expects.""" - print("\n=== Test 4: Response Format Compatibility ===") - try: - from llm_interface import LLMInterface - from anthropic.types import TextBlock, ToolUseBlock - - llm = LLMInterface(provider="claude") - - if llm.mode != "agent_sdk": - print(f"⊘ Skipping test (mode is '{llm.mode}', not 'agent_sdk')") - return False - - # Simulate SDK response - mock_sdk_response = { - "content": [ - {"type": "text", "text": "Test response"} - ], - "stop_reason": "end_turn", - "usage": { - "input_tokens": 10, - "output_tokens": 5 - }, - "id": "test_message_id", - "model": "claude-haiku-4-5-20251001" - } - - print("Converting mock SDK response to Message format...") - message = llm._convert_sdk_response_to_message(mock_sdk_response) - - print(f"✓ Conversion successful") - print(f" - Message type: {type(message).__name__}") - print(f" - Has content: {hasattr(message, 'content')}") - print(f" - Has stop_reason: {hasattr(message, 'stop_reason')}") - print(f" - Has usage: {hasattr(message, 'usage')}") - print(f" - Content[0] type: {type(message.content[0]).__name__}") - print(f" - Content[0].type: {message.content[0].type}") - print(f" - Content[0].text: {message.content[0].text}") - print(f" - Stop reason: {message.stop_reason}") - print(f" - Usage.input_tokens: {message.usage.input_tokens}") - print(f" - Usage.output_tokens: {message.usage.output_tokens}") - - # Verify all required attributes exist - required_attrs = ['content', 'stop_reason', 'usage', 'id', 'model', 'role', 'type'] - missing_attrs = [attr for attr in required_attrs if not hasattr(message, attr)] - - if missing_attrs: - print(f"✗ Missing attributes: {missing_attrs}") - return False - - print(f"✓ All required attributes present") - return True - - except Exception as e: - print(f"✗ Test failed: {e}") - import traceback - traceback.print_exc() - return False - - -def test_mode_selection(): - """Test 5: Verify mode selection logic.""" - print("\n=== Test 5: Mode Selection Logic ===") - - test_cases = [ - { - "name": "Default (Agent SDK)", - "env": {}, - "expected": "agent_sdk" - }, - { - "name": "Explicit Direct API", - "env": {"USE_DIRECT_API": "true"}, - "expected": "direct_api" - }, - { - "name": "Legacy Server", - "env": {"USE_CLAUDE_CODE_SERVER": "true"}, - "expected": "legacy_server" - }, - { - "name": "Priority: Direct API > Agent SDK", - "env": {"USE_DIRECT_API": "true", "USE_AGENT_SDK": "true"}, - "expected": "direct_api" - }, - { - "name": "Priority: Legacy > Agent SDK", - "env": {"USE_CLAUDE_CODE_SERVER": "true", "USE_AGENT_SDK": "true"}, - "expected": "legacy_server" - } - ] - - all_passed = True - - for test_case in test_cases: - print(f"\n Testing: {test_case['name']}") - - # Save current env - old_env = {} - for key in ["USE_DIRECT_API", "USE_CLAUDE_CODE_SERVER", "USE_AGENT_SDK"]: - old_env[key] = os.environ.get(key) - - # Set test env - for key in old_env.keys(): - if key in os.environ: - del os.environ[key] - for key, value in test_case["env"].items(): - os.environ[key] = value - - # Force reimport to pick up new env vars - if 'llm_interface' in sys.modules: - del sys.modules['llm_interface'] - - try: - from llm_interface import LLMInterface - llm = LLMInterface(provider="claude") - - if llm.mode == test_case["expected"]: - print(f" ✓ Correct mode: {llm.mode}") - else: - print(f" ✗ Wrong mode: expected '{test_case['expected']}', got '{llm.mode}'") - all_passed = False - - except Exception as e: - print(f" ✗ Error: {e}") - all_passed = False - - # Restore env - for key in old_env.keys(): - if key in os.environ: - del os.environ[key] - if old_env[key] is not None: - os.environ[key] = old_env[key] - - # Force reimport one more time to reset - if 'llm_interface' in sys.modules: - del sys.modules['llm_interface'] - - return all_passed - - -def main(): - """Run all tests.""" - print("=" * 70) - print("AGENT SDK IMPLEMENTATION TEST SUITE") - print("=" * 70) - - tests = [ - ("Initialization", test_llm_interface_initialization), - ("Simple Chat", test_simple_chat), - ("Chat with Tools", test_chat_with_tools), - ("Response Format", test_response_format_compatibility), - ("Mode Selection", test_mode_selection), - ] - - results = {} - - for name, test_func in tests: - try: - results[name] = test_func() - except Exception as e: - print(f"\n✗ Test '{name}' crashed: {e}") - import traceback - traceback.print_exc() - results[name] = False - - # Summary - print("\n" + "=" * 70) - print("TEST SUMMARY") - print("=" * 70) - - for name, passed in results.items(): - status = "✓ PASS" if passed else "✗ FAIL" - print(f"{status:8} {name}") - - passed_count = sum(1 for p in results.values() if p) - total_count = len(results) - - print(f"\nTotal: {passed_count}/{total_count} tests passed") - - if passed_count == total_count: - print("\n🎉 All tests passed!") - return 0 - else: - print(f"\n⚠ {total_count - passed_count} test(s) failed") - return 1 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/tools.py b/tools.py index ef4b6b8..bce2289 100644 --- a/tools.py +++ b/tools.py @@ -340,7 +340,12 @@ TOOL_DEFINITIONS = [ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any = None) -> str: - """Execute a tool and return the result as a string.""" + """Execute a tool and return the result as a string. + + This is used by the Direct API tool loop in agent.py. + In Agent SDK mode, tools are executed automatically via MCP servers + and this function is not called. + """ import time from logging_config import get_tool_logger @@ -348,71 +353,9 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any start_time = time.time() try: - # MCP tools (zettelkasten + web_fetch) - route to mcp_tools.py - MCP_TOOLS = { - "web_fetch", "fleeting_note", "daily_note", "literature_note", - "permanent_note", "search_vault", "search_by_tags" - } - - if tool_name in MCP_TOOLS: - # Route to MCP tool handlers - import anyio - from mcp_tools import ( - web_fetch_tool, fleeting_note_tool, daily_note_tool, - literature_note_tool, permanent_note_tool, - search_vault_tool, search_by_tags_tool - ) - - # Map tool names to their handlers - mcp_handlers = { - "web_fetch": web_fetch_tool, - "fleeting_note": fleeting_note_tool, - "daily_note": daily_note_tool, - "literature_note": literature_note_tool, - "permanent_note": permanent_note_tool, - "search_vault": search_vault_tool, - "search_by_tags": search_by_tags_tool, - } - - # Execute MCP tool asynchronously - handler = mcp_handlers[tool_name] - result = anyio.run(handler, tool_input) - - # Convert result to string if needed - if isinstance(result, dict): - if "error" in result: - error_msg = f"Error: {result['error']}" - duration_ms = (time.time() - start_time) * 1000 - logger.log_tool_call( - tool_name=tool_name, - inputs=tool_input, - success=False, - error=error_msg, - duration_ms=duration_ms - ) - return error_msg - elif "content" in result: - result_str = result["content"] - else: - result_str = str(result) - else: - result_str = str(result) - - # Log successful execution - duration_ms = (time.time() - start_time) * 1000 - logger.log_tool_call( - tool_name=tool_name, - inputs=tool_input, - success=True, - result=result_str, - duration_ms=duration_ms - ) - return result_str - - # File tools (traditional handlers - kept for backward compatibility) - # Execute traditional tool and capture result result_str = None + # --- File and system tools (sync handlers) --- if tool_name == "read_file": result_str = _read_file(tool_input["file_path"]) elif tool_name == "write_file": @@ -424,16 +367,31 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any tool_input["new_text"], ) elif tool_name == "list_directory": - path = tool_input.get("path", ".") - result_str = _list_directory(path) + result_str = _list_directory(tool_input.get("path", ".")) elif tool_name == "run_command": - command = tool_input["command"] - working_dir = tool_input.get("working_dir", ".") - result_str = _run_command(command, working_dir) + result_str = _run_command( + tool_input["command"], + tool_input.get("working_dir", "."), + ) + + # --- Weather tool (sync handler) --- elif tool_name == "get_weather": - location = tool_input.get("location", "Phoenix, US") - result_str = _get_weather(location) - # Gmail tools + result_str = _get_weather(tool_input.get("location", "Phoenix, US")) + + # --- Async MCP tools (web, zettelkasten, gitea) --- + elif tool_name in { + "web_fetch", "fleeting_note", "daily_note", "literature_note", + "permanent_note", "search_vault", "search_by_tags", + "gitea_read_file", "gitea_list_files", "gitea_search_code", "gitea_get_tree", + }: + # Note: These tools should only execute via Agent SDK MCP servers. + # If you're seeing this message, the tool routing needs adjustment. + return ( + f"[MCP Tool] '{tool_name}' should be dispatched by Agent SDK MCP server. " + f"Direct API fallback is disabled for this tool to ensure zero API cost." + ) + + # --- Google tools (sync handlers using traditional API clients) --- elif tool_name == "send_email": result_str = _send_email( to=tool_input["to"], @@ -453,7 +411,6 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any message_id=tool_input["message_id"], format_type=tool_input.get("format", "text"), ) - # Calendar tools elif tool_name == "read_calendar": result_str = _read_calendar( days_ahead=tool_input.get("days_ahead", 7), @@ -474,7 +431,6 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any query=tool_input["query"], calendar_id=tool_input.get("calendar_id", "primary"), ) - # Contacts tools elif tool_name == "create_contact": result_str = _create_contact( given_name=tool_input["given_name"], @@ -493,7 +449,16 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any resource_name=tool_input["resource_name"], ) - # Log successful traditional tool execution + # --- Obsidian MCP tools (external server with fallback) --- + elif tool_name in { + "obsidian_read_note", "obsidian_update_note", + "obsidian_search_replace", "obsidian_global_search", + "obsidian_list_notes", "obsidian_manage_frontmatter", + "obsidian_manage_tags", "obsidian_delete_note", + }: + result_str = _execute_obsidian_tool(tool_name, tool_input, logger, start_time) + + # --- Unknown tool --- if result_str is not None: duration_ms = (time.time() - start_time) * 1000 logger.log_tool_call( @@ -501,7 +466,7 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any inputs=tool_input, success=True, result=result_str, - duration_ms=duration_ms + duration_ms=duration_ms, ) return result_str else: @@ -512,9 +477,10 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any inputs=tool_input, success=False, error=error_msg, - duration_ms=duration_ms + duration_ms=duration_ms, ) return error_msg + except Exception as e: duration_ms = (time.time() - start_time) * 1000 error_msg = str(e) @@ -539,6 +505,61 @@ def execute_tool(tool_name: str, tool_input: Dict[str, Any], healing_system: Any return f"Error executing {tool_name}: {error_msg}" +def _extract_mcp_result(result: Any) -> str: + """Convert an MCP tool result dict to a plain string.""" + if isinstance(result, dict): + if "error" in result: + return f"Error: {result['error']}" + elif "content" in result: + content = result["content"] + if isinstance(content, list): + # Extract text from content blocks + parts = [] + for block in content: + if isinstance(block, dict) and block.get("type") == "text": + parts.append(block.get("text", "")) + return "\n".join(parts) if parts else str(content) + return str(content) + return str(result) + return str(result) + + +def _execute_obsidian_tool( + tool_name: str, + tool_input: Dict[str, Any], + logger: Any, + start_time: float, +) -> str: + """Execute an Obsidian MCP tool with fallback to custom tools.""" + try: + from obsidian_mcp import ( + check_obsidian_health, + should_fallback_to_custom, + ) + + if check_obsidian_health(): + return ( + f"[Obsidian MCP] Tool '{tool_name}' should be dispatched " + f"by the Agent SDK MCP server. If you're seeing this, " + f"the tool call routing may need adjustment." + ) + elif should_fallback_to_custom(): + fallback_result = _obsidian_fallback(tool_name, tool_input) + if fallback_result is not None: + return fallback_result + return ( + f"Error: Obsidian is not running and no fallback " + f"available for '{tool_name}'." + ) + else: + return ( + f"Error: Obsidian is not running and fallback is disabled. " + f"Please start Obsidian desktop app." + ) + except ImportError: + return f"Error: obsidian_mcp module not found for tool '{tool_name}'" + + # Maximum characters of tool output to return (prevents token explosion) _MAX_TOOL_OUTPUT = 5000 @@ -1001,3 +1022,86 @@ def _get_contact(resource_name: str) -> str: return "\n".join(output) else: return f"Error getting contact: {result.get('error', 'Unknown error')}" + + +def _obsidian_fallback(tool_name: str, tool_input: Dict[str, Any]) -> Optional[str]: + """Map Obsidian MCP tools to custom zettelkasten/file tool equivalents. + + Returns None if no fallback is possible for the given tool. + """ + from pathlib import Path + + if tool_name == "obsidian_read_note": + # Map to read_file with vault-relative path + vault_path = Path("memory_workspace/obsidian") + file_path = str(vault_path / tool_input.get("filePath", "")) + return _read_file(file_path) + + elif tool_name == "obsidian_global_search": + # Map to search_vault + import anyio + from mcp_tools import search_vault_tool + result = anyio.run(search_vault_tool, { + "query": tool_input.get("query", ""), + "limit": tool_input.get("pageSize", 10), + }) + if isinstance(result, dict) and "content" in result: + return str(result["content"]) + return str(result) + + elif tool_name == "obsidian_list_notes": + # Map to list_directory + vault_path = Path("memory_workspace/obsidian") + dir_path = str(vault_path / tool_input.get("dirPath", "")) + return _list_directory(dir_path) + + elif tool_name == "obsidian_update_note": + # Map to write_file or edit_file based on mode + vault_path = Path("memory_workspace/obsidian") + target = tool_input.get("targetIdentifier", "") + content = tool_input.get("content", "") + mode = tool_input.get("wholeFileMode", "overwrite") + file_path = str(vault_path / target) + + if mode == "overwrite": + return _write_file(file_path, content) + elif mode == "append": + existing = Path(file_path).read_text(encoding="utf-8") if Path(file_path).exists() else "" + return _write_file(file_path, existing + "\n" + content) + elif mode == "prepend": + existing = Path(file_path).read_text(encoding="utf-8") if Path(file_path).exists() else "" + return _write_file(file_path, content + "\n" + existing) + + elif tool_name == "obsidian_search_replace": + # Map to edit_file + vault_path = Path("memory_workspace/obsidian") + target = tool_input.get("targetIdentifier", "") + file_path = str(vault_path / target) + replacements = tool_input.get("replacements", []) + if replacements: + first = replacements[0] + return _edit_file( + file_path, + first.get("search", ""), + first.get("replace", ""), + ) + + elif tool_name == "obsidian_manage_tags": + # Map to search_by_tags (list operation only) + operation = tool_input.get("operation", "list") + if operation == "list": + tags = tool_input.get("tags", "") + if isinstance(tags, list): + tags = ",".join(tags) + import anyio + from mcp_tools import search_by_tags_tool + result = anyio.run(search_by_tags_tool, {"tags": tags}) + if isinstance(result, dict) and "content" in result: + return str(result["content"]) + return str(result) + + # No fallback possible for: + # - obsidian_manage_frontmatter (new capability, no custom equivalent) + # - obsidian_delete_note (safety: deliberate no-fallback for destructive ops) + # - obsidian_manage_tags add/remove (requires YAML frontmatter parsing) + return None