Add comprehensive structured logging system
Features: - JSON-formatted logs for easy parsing and analysis - Rotating log files (prevents disk space issues) * ajarbot.log: All events, 10MB rotation, 5 backups * errors.log: Errors only, 5MB rotation, 3 backups * tools.log: Tool execution tracking, 10MB rotation, 3 backups Tool Execution Tracking: - Every tool call logged with inputs, outputs, duration - Success/failure status tracking - Performance metrics (execution time in milliseconds) - Error messages captured with full context Logging Integration: - tools.py: All tool executions automatically logged - Structured logger classes with context preservation - Console output (human-readable) + file logs (JSON) - Separate error log for quick issue identification Log Analysis: - JSON format enables programmatic analysis - Easy to search for patterns (max tokens, iterations, etc.) - Performance tracking (slow tools, failure rates) - Historical debugging with full context Documentation: - LOGGING.md: Complete usage guide - Log analysis examples with jq commands - Error pattern reference - Maintenance and integration instructions Benefits: - Quick error diagnosis with separate errors.log - Performance monitoring and optimization - Historical analysis for troubleshooting - Automatic log rotation (max 95MB total) Updated .gitignore to exclude logs/ directory Co-Authored-By: Claude Sonnet 4.5 <noreply@anthropic.com>
This commit is contained in:
203
logging_config.py
Normal file
203
logging_config.py
Normal file
@@ -0,0 +1,203 @@
|
||||
"""
|
||||
Structured logging configuration for Ajarbot.
|
||||
|
||||
Provides consistent logging across all components with:
|
||||
- Rotating file logs (prevents disk space issues)
|
||||
- Separate error log for quick issue identification
|
||||
- JSON-structured logs for easy parsing
|
||||
- Console output for development
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import logging.handlers
|
||||
from pathlib import Path
|
||||
from datetime import datetime
|
||||
from typing import Optional, Dict, Any
|
||||
|
||||
|
||||
# Log directory
|
||||
LOG_DIR = Path("logs")
|
||||
LOG_DIR.mkdir(exist_ok=True)
|
||||
|
||||
# Log file paths
|
||||
MAIN_LOG = LOG_DIR / "ajarbot.log"
|
||||
ERROR_LOG = LOG_DIR / "errors.log"
|
||||
TOOL_LOG = LOG_DIR / "tools.log"
|
||||
|
||||
|
||||
class JSONFormatter(logging.Formatter):
|
||||
"""Format log records as JSON for easy parsing."""
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
"""Format log record as JSON."""
|
||||
log_data = {
|
||||
"timestamp": datetime.utcnow().isoformat() + "Z",
|
||||
"level": record.levelname,
|
||||
"logger": record.name,
|
||||
"message": record.getMessage(),
|
||||
"module": record.module,
|
||||
"function": record.funcName,
|
||||
"line": record.lineno,
|
||||
}
|
||||
|
||||
# Add exception info if present
|
||||
if record.exc_info:
|
||||
log_data["exception"] = self.formatException(record.exc_info)
|
||||
|
||||
# Add extra fields if present
|
||||
if hasattr(record, "extra_data"):
|
||||
log_data["extra"] = record.extra_data
|
||||
|
||||
return json.dumps(log_data)
|
||||
|
||||
|
||||
class StructuredLogger:
|
||||
"""Wrapper for structured logging with context."""
|
||||
|
||||
def __init__(self, name: str):
|
||||
"""Initialize structured logger."""
|
||||
self.logger = logging.getLogger(name)
|
||||
self._setup_handlers()
|
||||
|
||||
def _setup_handlers(self):
|
||||
"""Set up log handlers if not already configured."""
|
||||
if self.logger.handlers:
|
||||
return # Already configured
|
||||
|
||||
self.logger.setLevel(logging.DEBUG)
|
||||
|
||||
# Console handler (human-readable)
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.INFO)
|
||||
console_formatter = logging.Formatter(
|
||||
"%(asctime)s - %(name)s - %(levelname)s - %(message)s",
|
||||
datefmt="%Y-%m-%d %H:%M:%S"
|
||||
)
|
||||
console_handler.setFormatter(console_formatter)
|
||||
|
||||
# Main log file handler (JSON, rotating)
|
||||
main_handler = logging.handlers.RotatingFileHandler(
|
||||
MAIN_LOG,
|
||||
maxBytes=10 * 1024 * 1024, # 10MB
|
||||
backupCount=5,
|
||||
encoding="utf-8"
|
||||
)
|
||||
main_handler.setLevel(logging.DEBUG)
|
||||
main_handler.setFormatter(JSONFormatter())
|
||||
|
||||
# Error log file handler (JSON, errors only)
|
||||
error_handler = logging.handlers.RotatingFileHandler(
|
||||
ERROR_LOG,
|
||||
maxBytes=5 * 1024 * 1024, # 5MB
|
||||
backupCount=3,
|
||||
encoding="utf-8"
|
||||
)
|
||||
error_handler.setLevel(logging.ERROR)
|
||||
error_handler.setFormatter(JSONFormatter())
|
||||
|
||||
# Add handlers
|
||||
self.logger.addHandler(console_handler)
|
||||
self.logger.addHandler(main_handler)
|
||||
self.logger.addHandler(error_handler)
|
||||
|
||||
def log(self, level: int, message: str, extra: Optional[Dict[str, Any]] = None):
|
||||
"""Log message with optional extra context."""
|
||||
if extra:
|
||||
self.logger.log(level, message, extra={"extra_data": extra})
|
||||
else:
|
||||
self.logger.log(level, message)
|
||||
|
||||
def debug(self, message: str, **kwargs):
|
||||
"""Log debug message."""
|
||||
self.log(logging.DEBUG, message, kwargs if kwargs else None)
|
||||
|
||||
def info(self, message: str, **kwargs):
|
||||
"""Log info message."""
|
||||
self.log(logging.INFO, message, kwargs if kwargs else None)
|
||||
|
||||
def warning(self, message: str, **kwargs):
|
||||
"""Log warning message."""
|
||||
self.log(logging.WARNING, message, kwargs if kwargs else None)
|
||||
|
||||
def error(self, message: str, exc_info: bool = False, **kwargs):
|
||||
"""Log error message."""
|
||||
self.logger.error(
|
||||
message,
|
||||
exc_info=exc_info,
|
||||
extra={"extra_data": kwargs} if kwargs else None
|
||||
)
|
||||
|
||||
def critical(self, message: str, exc_info: bool = False, **kwargs):
|
||||
"""Log critical message."""
|
||||
self.logger.critical(
|
||||
message,
|
||||
exc_info=exc_info,
|
||||
extra={"extra_data": kwargs} if kwargs else None
|
||||
)
|
||||
|
||||
|
||||
class ToolLogger(StructuredLogger):
|
||||
"""Specialized logger for tool execution tracking."""
|
||||
|
||||
def __init__(self):
|
||||
"""Initialize tool logger with separate log file."""
|
||||
super().__init__("tools")
|
||||
|
||||
# Add specialized tool log handler
|
||||
tool_handler = logging.handlers.RotatingFileHandler(
|
||||
TOOL_LOG,
|
||||
maxBytes=10 * 1024 * 1024, # 10MB
|
||||
backupCount=3,
|
||||
encoding="utf-8"
|
||||
)
|
||||
tool_handler.setLevel(logging.INFO)
|
||||
tool_handler.setFormatter(JSONFormatter())
|
||||
self.logger.addHandler(tool_handler)
|
||||
|
||||
def log_tool_call(
|
||||
self,
|
||||
tool_name: str,
|
||||
inputs: Dict[str, Any],
|
||||
success: bool,
|
||||
result: Optional[str] = None,
|
||||
error: Optional[str] = None,
|
||||
duration_ms: Optional[float] = None
|
||||
):
|
||||
"""Log tool execution with structured data."""
|
||||
log_data = {
|
||||
"tool_name": tool_name,
|
||||
"inputs": inputs,
|
||||
"success": success,
|
||||
"duration_ms": duration_ms,
|
||||
}
|
||||
|
||||
if success:
|
||||
log_data["result_length"] = len(result) if result else 0
|
||||
self.info(f"Tool executed: {tool_name}", **log_data)
|
||||
else:
|
||||
log_data["error"] = error
|
||||
self.error(f"Tool failed: {tool_name}", **log_data)
|
||||
|
||||
|
||||
# Global logger instances
|
||||
def get_logger(name: str) -> StructuredLogger:
|
||||
"""Get or create a structured logger."""
|
||||
return StructuredLogger(name)
|
||||
|
||||
|
||||
def get_tool_logger() -> ToolLogger:
|
||||
"""Get the tool execution logger."""
|
||||
return ToolLogger()
|
||||
|
||||
|
||||
# Configure root logger
|
||||
logging.basicConfig(
|
||||
level=logging.WARNING,
|
||||
format="%(asctime)s - %(name)s - %(levelname)s - %(message)s"
|
||||
)
|
||||
|
||||
# Suppress noisy third-party loggers
|
||||
logging.getLogger("anthropic").setLevel(logging.WARNING)
|
||||
logging.getLogger("httpx").setLevel(logging.WARNING)
|
||||
logging.getLogger("httpcore").setLevel(logging.WARNING)
|
||||
Reference in New Issue
Block a user