feat: RSO observation system, child safety, Discord adapter, Telegram watchdog, email attachments

Core agent improvements:
- RSO (Relevance Scoring & Observation) system: interaction_logger, memory_scorer, signal_detector
- Memory access logging (memory_access_log table) for relevance scoring; high-signal turn detection
- Rich conversation storage for notable turns; compact_conversation truncates long user messages
- Task-type classifier (query/action/analysis/creative) for observation tagging
- Nested sub-agent visibility: deep delegations now register against the main agent's manager

Child safety (Gabriel profile):
- child_safety.py: filtering, audit logging, prompt constants for restricted sessions
- .kiro/specs/child-safety-profile: requirements, design, tasks specs
- GABRIEL_BOT_PROPOSAL.md: initial proposal doc
- Reduced context window (10 msgs) and tutor-mode identity for restricted users

Telegram adapter:
- Polling watchdog: auto-restarts updater if polling drops unexpectedly
- get_me() with exponential-backoff retry on NetworkError at startup
- Correct stop() ordering: signal watchdog before cancelling tasks

Email / Gmail:
- send_email: supports file attachments (attachments list param)
- get_email: surfaces attachment metadata in response

Scheduled tasks / weather:
- Remove OpenWeatherMap API calls from morning-weather task; use wttr.in exclusively
- New scheduled tasks and scheduler state persistence

Discord:
- adapters/discord/__init__.py scaffold
- discord-plugin: MCP plugin for Claude Code Discord integration (server.ts, skills, config)

Infrastructure:
- n8n workflow exports (garvis_webhook, content_pipeline variants)
- memory_workspace: context, homelab-repo-updates, weekly observation summaries, error logs
- UCS C240 migration plan doc
- requirements.txt: new deps
- .claude/settings.json, fix_hooks.py: hook/permission tuning
This commit is contained in:
2026-04-23 07:54:01 -06:00
parent 1232490c3b
commit 916f86725d
70 changed files with 10945 additions and 187 deletions

454
agent.py
View File

@@ -3,8 +3,14 @@
import random
import threading
import time
from concurrent.futures import Future, TimeoutError as FutureTimeoutError
from datetime import datetime
from typing import Any, List, Optional, Callable
from logging_config import StructuredLogger as _StructuredLogger
# Use the project's structured logger so STATE[...] lines go to ajarbot.log, not /dev/null.
_agent_logger = _StructuredLogger("agent").logger
from hooks import HooksSystem
from llm_interface import LLMInterface
from memory_system import MemorySystem
@@ -14,11 +20,29 @@ from sub_agent_manager import SubAgentManager
# Maximum number of recent messages to include in LLM context
MAX_CONTEXT_MESSAGES = 20 # Optimized for Agent SDK flat-rate subscription
CHILD_MAX_CONTEXT_MESSAGES = 10 # Reduced window for restricted child sessions
# Maximum conversation history entries before pruning
MAX_CONVERSATION_HISTORY = 50 # Conservative limit to prevent Agent SDK JSON buffer overflow (1MB max)
# Maximum tool execution iterations (generous limit for complex operations like zettelkasten)
MAX_TOOL_ITERATIONS = 30 # Allows complex multi-step workflows with auto-linking, hybrid search, etc.
import uuid as _uuid
def _classify_task_type(message: str) -> str:
"""Heuristic task-type classifier for RSO observation entries.
Returns one of: "query" | "action" | "analysis" | "creative"
"""
text = message.lower()
if any(w in text for w in ("write", "create", "draft", "compose", "generate", "make", "build")):
return "creative"
if any(w in text for w in ("analyze", "analyse", "review", "summarize", "compare", "evaluate", "explain")):
return "analysis"
if any(w in text for w in ("run", "execute", "send", "update", "delete", "move", "deploy", "schedule", "set")):
return "action"
return "query"
class Agent:
"""AI Agent with memory, LLM, and hooks."""
@@ -48,6 +72,32 @@ class Agent:
self.sub_agents: dict = {} # Cache for spawned sub-agents
self.agent_id: Optional[str] = None # Set when this is a sub-agent
# RSO observation (main agent only — sub-agents never log)
self._interaction_logger = None
self._last_interaction_id: Optional[str] = None
self._last_interaction_ts: Optional[float] = None
if not is_sub_agent:
try:
from observation.interaction_logger import InteractionLogger
self._interaction_logger = InteractionLogger(self.memory.workspace_dir)
self._interaction_logger.cleanup_old_logs()
except Exception as _e:
print(f"[Agent] Observation logger unavailable: {_e}")
# Child safety config (main agent only — controls restricted-user prompt path)
self._child_safety_config = None
if not is_sub_agent:
try:
from child_safety import ChildSafetyConfig
from pathlib import Path as _Path
_cs_path = _Path("config/adapters.local.yaml")
if _cs_path.exists():
self._child_safety_config = ChildSafetyConfig.from_yaml(_cs_path)
if self._child_safety_config:
print(f"[Agent] Child safety active for: {self._child_safety_config.restricted_users}")
except Exception as _e:
print(f"[Agent] Child safety config not loaded: {_e}")
self.memory.sync()
if not is_sub_agent: # Only trigger hooks for main agent
self.hooks.trigger("agent", "startup", {"workspace_dir": workspace_dir})
@@ -98,9 +148,11 @@ class Agent:
# Cache if ID provided
if agent_id:
self.sub_agents[agent_id] = sub_agent
# Register with sub-agent manager for monitoring
if not self.is_sub_agent:
self.sub_agent_manager.register_sub_agent(agent_id, specialist_prompt[:100])
# Register with sub-agent manager for monitoring.
# Register nested sub-agents against the main agent's manager too, so deep
# delegations are visible rather than running as ghosts.
manager = self.sub_agent_manager
manager.register_sub_agent(agent_id, specialist_prompt[:100])
return sub_agent
@@ -120,47 +172,69 @@ class Agent:
# Add timestamp to user-provided ID to ensure uniqueness
agent_id = f"{agent_id}_{int(time.time()*1000)}"
# Enforce the watchdog's total_timeout at the caller. Add a small buffer so
# the watchdog gets first crack at marking the agent hung before we TimeoutError.
total_timeout = self.sub_agent_manager.total_timeout_seconds + 30
for attempt in range(max_retries + 1):
if attempt > 0:
print(f"[Agent] Retrying {agent_id} (attempt {attempt+1}/{max_retries+1})")
if agent_id and not self.is_sub_agent:
self.sub_agent_manager.cleanup_agent(f"{agent_id}_prev")
retry_id = f"{agent_id}_r{attempt}" if (agent_id and attempt > 0) else agent_id
_agent_logger.warning("[Agent] STATE[retry] id=%s attempt=%d/%d", agent_id, attempt + 1, max_retries + 1)
retry_id = f"{agent_id}_r{attempt}" if attempt > 0 else agent_id
sub_agent = self.spawn_sub_agent(specialist_prompt, agent_id=retry_id)
# Heartbeat for activity tracking
heartbeat_running = [True]
def heartbeat():
while heartbeat_running[0]:
if retry_id and not self.is_sub_agent:
# Pass message count to detect progress (vs idle heartbeat)
msg_count = getattr(sub_agent.llm, 'message_count', 0)
self.sub_agent_manager.update_activity(retry_id, message_count=msg_count)
msg_count = getattr(sub_agent.llm, '_last_message_count', 0)
self.sub_agent_manager.update_activity(retry_id, message_count=msg_count)
time.sleep(10)
heartbeat_thread = None
if retry_id and not self.is_sub_agent:
heartbeat_thread = threading.Thread(target=heartbeat, daemon=True)
heartbeat_thread.start()
heartbeat_thread = threading.Thread(target=heartbeat, daemon=True, name=f"hb-{retry_id}")
heartbeat_thread.start()
# Manual Future + daemon thread: lets the worker be orphaned cleanly on timeout
# without keeping the process alive at shutdown.
future: Future = Future()
def _worker(f=future, sa=sub_agent, t=task, u=username):
if not f.set_running_or_notify_cancel():
return
try:
f.set_result(sa.chat(t, username=u))
except BaseException as exc:
f.set_exception(exc)
worker_thread = threading.Thread(target=_worker, daemon=True, name=f"sub-{retry_id}")
try:
result = sub_agent.chat(task, username=username)
if retry_id and not self.is_sub_agent:
self.sub_agent_manager.mark_complete(retry_id, result=result)
worker_thread.start()
self.sub_agent_manager.attach_future(retry_id, future)
_agent_logger.info("[Agent] STATE[dispatch] id=%s timeout=%ds", retry_id, total_timeout)
try:
result = future.result(timeout=total_timeout)
except FutureTimeoutError:
future.cancel()
err = f"delegate() hit total timeout ({total_timeout}s) for {retry_id}"
_agent_logger.error("[Agent] STATE[timeout] id=%s", retry_id)
self.sub_agent_manager.mark_complete(retry_id, error=err)
if attempt >= max_retries:
raise TimeoutError(err)
continue
self.sub_agent_manager.mark_complete(retry_id, result=result)
return result
except Exception as e:
if retry_id and not self.is_sub_agent:
self.sub_agent_manager.mark_complete(retry_id, error=str(e))
# If this was the last attempt, raise the error
# future.result() re-raises worker exceptions here.
self.sub_agent_manager.mark_complete(retry_id, error=str(e))
_agent_logger.warning("[Agent] STATE[error] id=%s err=%s", retry_id, str(e)[:200])
if attempt >= max_retries:
raise
# Otherwise, retry will happen in next loop iteration
finally:
heartbeat_running[0] = False
if heartbeat_thread:
heartbeat_thread.join(timeout=1)
heartbeat_thread.join(timeout=1)
# Don't join worker_thread — it may be stuck in sub_agent.chat(). Daemon=True
# means the process can still exit cleanly; a timed-out worker is orphaned.
def _get_context_messages(self, max_messages: int) -> List[dict]:
"""Get recent messages without breaking tool_use/tool_result pairs.
@@ -210,6 +284,57 @@ class Agent:
return result
def _cap_old_message_content(self, messages: List[dict], keep_full_turns: int = 4, cap_chars: int = 600) -> List[dict]:
"""Cap the content of older messages to reduce prompt size.
The most recent `keep_full_turns` turns (2 messages per turn) are kept
in full. Older messages have their text content capped at `cap_chars`
characters — enough to preserve the gist without the full prose.
Returns a new list; does NOT mutate conversation_history.
"""
keep_full_messages = keep_full_turns * 2 # user + assistant per turn
if len(messages) <= keep_full_messages:
return messages
import copy
result = []
cutoff = len(messages) - keep_full_messages
for i, msg in enumerate(messages):
if i >= cutoff:
# Recent turns — include in full
result.append(msg)
continue
# Older turn — cap text content
msg_copy = copy.copy(msg)
content = msg.get("content", "")
if isinstance(content, str):
if len(content) > cap_chars:
msg_copy["content"] = content[:cap_chars] + "... [truncated]"
else:
msg_copy["content"] = content
elif isinstance(content, list):
new_blocks = []
for block in content:
if isinstance(block, dict) and block.get("type") == "text":
text = block.get("text", "")
if len(text) > cap_chars:
new_blocks.append({**block, "text": text[:cap_chars] + "... [truncated]"})
else:
new_blocks.append(block)
else:
new_blocks.append(block)
msg_copy["content"] = new_blocks
else:
msg_copy["content"] = content
result.append(msg_copy)
return result
def _prune_conversation_history(self) -> None:
"""Prune conversation history to prevent unbounded growth.
@@ -344,6 +469,30 @@ class Agent:
f"Commands: /sonnet, /haiku, /status"
)
# RSO: classify signal from this message relative to the prior interaction
if (
self._interaction_logger is not None
and self._last_interaction_id is not None
and not self.is_sub_agent
):
try:
from observation.signal_detector import classify_signal
_now = time.time()
_delta = (_now - self._last_interaction_ts) if self._last_interaction_ts else None
_signal_type = classify_signal(user_message, time_delta_seconds=_delta)
self._interaction_logger.update_signal(
self._last_interaction_id,
{
"follow_up_type": _signal_type,
"explicit_positive": _signal_type == "positive",
"explicit_negative": _signal_type in ("negative", "correction"),
"correction_followed": _signal_type == "correction",
"time_delta_seconds": round(_delta, 1) if _delta is not None else None,
},
)
except Exception as _e:
print(f"[Agent] Signal detection failed: {_e}")
with self._chat_lock:
try:
return self._chat_inner(user_message, username, inbound_message)
@@ -351,7 +500,7 @@ class Agent:
# Clear the callback when done
self._progress_callback = None
def _build_system_prompt(self, user_message: str, username: str) -> str:
def _build_system_prompt(self, user_message: str, username: str, platform: str = "unknown") -> str:
"""Build the system prompt with SOUL, user profile, and memory context."""
if self.specialist_prompt:
return (
@@ -362,18 +511,80 @@ class Agent:
)
soul = self.memory.get_soul()
context = self.memory.get_context()
user_profile = self.memory.get_user(username)
relevant_memory = self.memory.search_hybrid(user_message, max_results=5)
memory_lines = [f"- {mem['snippet']}" for mem in relevant_memory]
return (
f"{soul}\n\nUser Profile:\n{user_profile}\n\n"
f"Relevant Memory:\n" + "\n".join(memory_lines) +
f"\n\nYou have access to tools for file operations, command execution, "
f"web fetching, note-taking, and Google services (Gmail, Calendar, Contacts). "
f"Use them freely to help the user."
parts = [soul]
if context:
parts.append(f"Operational Context:\n{context}")
parts.append(f"User Profile:\n{user_profile}")
parts.append("Relevant Memory:\n" + "\n".join(memory_lines))
# Inject platform-specific formatting and tone reminder
if platform == "slack":
parts.append(
"FORMATTING (Slack): Use *text* for bold, _text_ for italic. "
"No ## headers, no --- dividers. Tables in triple backtick code blocks. "
"Links as <url|text>.\n\n"
"TONE: You are JARVIS — dry British wit, deadpan delivery, calm competence. "
"Lead with the result. Wit lives in word choice, not length. "
"Occasional 'Sir'. Never chipper. Never flat."
)
parts.append(
"You have access to tools for file operations, command execution, "
"web fetching, note-taking, and Google services (Gmail, Calendar, Contacts). "
"Use them freely to help the user."
)
parts.append(
"DELEGATION: Call `delegate_task` when a request involves any of: "
"(a) reading/analyzing more than ~3 independent files or sources, "
"(b) multiple independent research threads that can run in parallel, "
"(c) a scoped sub-task you expect to take more than ~5 tool calls. "
"Inline tool loops past ~10 steps degrade quality and context. "
"Sequentially dependent work (e.g. server provisioning where each step gates the next) "
"stays inline — delegation only helps when subtasks are independent."
)
return "\n\n".join(parts)
def _build_child_system_prompt(self, username: str) -> str:
"""Build a stripped system prompt for restricted child users.
Skips SOUL.md, context.md, hybrid memory search, and delegation block
to save ~1,500 tokens per turn. Injects gabriel_context.md if present.
"""
from child_safety import (
CHILD_GUARDRAIL_BLOCK,
CHILD_TUTOR_IDENTITY,
FIRST_RUN_BLOCK,
SESSION_UPDATE_INSTRUCTION,
)
user_profile = self.memory.get_user(username)
context_path = self.memory.workspace_dir / "users" / "gabriel_context.md"
gabriel_context = (
context_path.read_text(encoding="utf-8") if context_path.exists() else None
)
is_first_run = gabriel_context is None
parts = [CHILD_TUTOR_IDENTITY, f"User Profile:\n{user_profile}"]
if gabriel_context:
parts.append(f"Project Context & Skills:\n{gabriel_context}")
if is_first_run:
parts.append(FIRST_RUN_BLOCK)
parts.append(CHILD_GUARDRAIL_BLOCK)
parts.append(SESSION_UPDATE_INSTRUCTION)
parts.append(
"You have access to file tools. Use them to update gabriel_context.md "
"at the end of this conversation."
)
return "\n\n".join(parts)
def _prepare_message_content(
self,
user_message: str,
@@ -492,11 +703,27 @@ class Agent:
def _chat_inner(self, user_message: str, username: str, inbound_message: Optional['InboundMessage'] = None) -> str:
"""Inner chat logic, called while holding _chat_lock."""
_rso_start_time = time.time()
# Prepare content (may include images)
content, has_images = self._prepare_message_content(user_message, inbound_message)
# Build system prompt
system = self._build_system_prompt(user_message, username)
# Extract platform for formatting hints
platform = inbound_message.platform if inbound_message else "unknown"
# Determine if this is a restricted child session
is_child = (
not self.is_sub_agent
and self._child_safety_config is not None
and self._child_safety_config.is_restricted(username)
)
context_cap = CHILD_MAX_CONTEXT_MESSAGES if is_child else MAX_CONTEXT_MESSAGES
# Build system prompt (child gets a stripped prompt without SOUL/context/memory)
if is_child:
system = self._build_child_system_prompt(username)
else:
system = self._build_system_prompt(user_message, username, platform)
# Enhance prompt for images
if has_images:
@@ -515,9 +742,9 @@ class Agent:
# In Agent SDK mode, query() handles tool calls automatically via MCP.
# The tool loop is only needed for Direct API mode.
if self.llm.mode == "agent_sdk":
return self._chat_agent_sdk(user_message, system)
return self._chat_agent_sdk(user_message, system, username, _rso_start_time, context_cap=context_cap)
else:
return self._chat_direct_api(user_message, system)
return self._chat_direct_api(user_message, system, username, _rso_start_time, context_cap=context_cap)
def _send_progress_update(self, elapsed_seconds: int):
"""Send a progress update if callback is set."""
@@ -554,9 +781,11 @@ class Agent:
self._progress_timer.cancel()
self._progress_timer = None
def _chat_agent_sdk(self, user_message: str, system: str) -> str:
def _chat_agent_sdk(self, user_message: str, system: str, username: str = "default", _rso_start: float = 0.0, context_cap: int = MAX_CONTEXT_MESSAGES) -> str:
"""Chat using Agent SDK. Tools are handled automatically by MCP."""
context_messages = self._get_context_messages(MAX_CONTEXT_MESSAGES)
context_messages = self._cap_old_message_content(
self._get_context_messages(context_cap)
)
# DEBUG: Count images in conversation history
image_count = 0
@@ -627,24 +856,81 @@ class Agent:
# Prune old tool results to prevent buffer overflow during diagram generation
self._prune_old_tool_results(keep_recent=10)
# Write compact summary to memory
compact_summary = self.memory.compact_conversation(
user_message=user_message,
assistant_response=final_response,
tools_used=None # SDK handles tools internally; we don't track them here
)
self.memory.write_memory(compact_summary, daily=True)
# Write memory entry — one-liner for scheduled tasks, rich/compact for real turns
if username == "scheduler":
timestamp = datetime.now().strftime('%H:%M')
summary = f"**Scheduled**: {user_message[:60]}... → delivered at {timestamp}"
elif self.memory.is_high_signal(user_message):
summary = self.memory.rich_conversation(user_message, final_response)
else:
summary = self.memory.compact_conversation(
user_message=user_message,
assistant_response=final_response,
tools_used=None # SDK handles tools internally; we don't track them here
)
self.memory.write_memory(summary, daily=True)
# RSO Phase 1: log interaction entry (agent_sdk mode)
if self._interaction_logger is not None:
try:
_iid = str(_uuid.uuid4())
_duration_ms = int((time.time() - _rso_start) * 1000) if _rso_start else 0
_tool_names = list(getattr(self.llm, '_last_tool_names', []) or [])
_total_cost = getattr(self.llm, '_last_total_cost_usd', 0) or 0
_num_turns = getattr(self.llm, '_last_num_turns', 0) or 0
_tool_count = len(_tool_names)
_complexity = (
"simple" if _tool_count < 3 else
"moderate" if _tool_count <= 6 else
"complex"
)
_entry = {
"record_type": "interaction",
"interaction_id": _iid,
"timestamp": datetime.now().astimezone().isoformat(),
"session_id": id(self),
"request": {
"source": "agent_sdk",
"user": username,
"message_preview": user_message[:100],
"task_type": _classify_task_type(user_message),
"complexity": _complexity,
},
"response": {
"duration_ms": _duration_ms,
"tool_calls": [{"tool": t} for t in _tool_names],
"total_tool_calls": _tool_count,
"tokens_in": 0, # not available in agent_sdk mode
"tokens_out": 0, # not available in agent_sdk mode
"cost_usd": _total_cost,
"num_turns": _num_turns,
"error": None,
},
"user_signal": {
"explicit_positive": False,
"explicit_negative": False,
"correction_followed": False,
"follow_up_type": None,
},
}
self._interaction_logger.log_interaction(_entry)
self._last_interaction_id = _iid
self._last_interaction_ts = time.time()
except Exception as _e:
print(f"[Agent] RSO log failed: {_e}")
return final_response
def _chat_direct_api(self, user_message: str, system: str) -> str:
def _chat_direct_api(self, user_message: str, system: str, username: str = "default", _rso_start: float = 0.0, context_cap: int = MAX_CONTEXT_MESSAGES) -> str:
"""Chat using Direct API with manual tool execution loop."""
max_iterations = MAX_TOOL_ITERATIONS
use_caching = "sonnet" in self.llm.model.lower()
tools_used = []
for iteration in range(max_iterations):
context_messages = self._get_context_messages(MAX_CONTEXT_MESSAGES)
context_messages = self._cap_old_message_content(
self._get_context_messages(context_cap)
)
# DEBUG: Count images in conversation history (only log on first iteration)
if iteration == 0:
@@ -699,12 +985,66 @@ class Agent:
# Prune old tool results to prevent buffer overflow during diagram generation
self._prune_old_tool_results(keep_recent=10)
compact_summary = self.memory.compact_conversation(
user_message=user_message,
assistant_response=final_response,
tools_used=tools_used if tools_used else None
)
self.memory.write_memory(compact_summary, daily=True)
if username == "scheduler":
timestamp = datetime.now().strftime('%H:%M')
summary = f"**Scheduled**: {user_message[:60]}... → delivered at {timestamp}"
elif self.memory.is_high_signal(user_message):
summary = self.memory.rich_conversation(
user_message, final_response,
tools_used=tools_used if tools_used else None
)
else:
summary = self.memory.compact_conversation(
user_message=user_message,
assistant_response=final_response,
tools_used=tools_used if tools_used else None
)
self.memory.write_memory(summary, daily=True)
# RSO Phase 1: log interaction entry (direct_api mode)
if self._interaction_logger is not None:
try:
_iid = str(_uuid.uuid4())
_duration_ms = int((time.time() - _rso_start) * 1000) if _rso_start else 0
_tool_count = len(tools_used)
_complexity = (
"simple" if _tool_count < 3 else
"moderate" if _tool_count <= 6 else
"complex"
)
_entry = {
"record_type": "interaction",
"interaction_id": _iid,
"timestamp": datetime.now().astimezone().isoformat(),
"session_id": id(self),
"request": {
"source": "direct_api",
"user": username,
"message_preview": user_message[:100],
"task_type": _classify_task_type(user_message),
"complexity": _complexity,
},
"response": {
"duration_ms": _duration_ms,
"tool_calls": [{"tool": t} for t in tools_used],
"total_tool_calls": _tool_count,
"tokens_in": 0,
"tokens_out": 0,
"cost_usd": 0,
"error": None,
},
"user_signal": {
"explicit_positive": False,
"explicit_negative": False,
"correction_followed": False,
"follow_up_type": None,
},
}
self._interaction_logger.log_interaction(_entry)
self._last_interaction_id = _iid
self._last_interaction_ts = time.time()
except Exception as _e:
print(f"[Agent] RSO log failed: {_e}")
return final_response