diff --git a/backend/routes/archerTickets.js b/backend/routes/archerTickets.js index 147ce47..3c28342 100644 --- a/backend/routes/archerTickets.js +++ b/backend/routes/archerTickets.js @@ -217,6 +217,25 @@ function createArcherTicketsRouter(db) { }); }); + // GET /status-trend — ticket counts grouped by creation date + status + // Used for time-based Archer pipeline chart on the Compliance page. + router.get('/status-trend', requireAuth(db), (req, res) => { + db.all( + `SELECT DATE(created_at) AS date, status, COUNT(*) AS count + FROM archer_tickets + GROUP BY DATE(created_at), status + ORDER BY date ASC`, + [], + (err, rows) => { + if (err) { + console.error('Error fetching Archer status trend:', err); + return res.status(500).json({ error: 'Internal server error.' }); + } + res.json({ statusTrend: rows }); + } + ); + }); + return router; } diff --git a/backend/routes/compliance.js b/backend/routes/compliance.js index a9262e5..640f0c9 100644 --- a/backend/routes/compliance.js +++ b/backend/routes/compliance.js @@ -584,6 +584,128 @@ function createComplianceRouter(db, upload, requireAuth, requireRole) { } }); + // ----------------------------------------------------------------------- + // GET /trends + // Per-upload active totals + per-team counts for time-series charts. + // Returns rows ordered ascending by report_date. + // ----------------------------------------------------------------------- + router.get('/trends', async (req, res) => { + try { + const uploads = await dbAll(db, + `SELECT id, report_date, + COALESCE(new_count, 0) AS new_count, + COALESCE(recurring_count, 0) AS recurring_count, + COALESCE(resolved_count, 0) AS resolved_count, + COALESCE(new_count, 0) + COALESCE(recurring_count, 0) AS total_active + FROM compliance_uploads + ORDER BY report_date ASC` + ); + + if (uploads.length === 0) return res.json({ trends: [] }); + + // Per-team active counts — items whose upload_id matches the upload + // (recurring items have upload_id bumped each cycle, so this is accurate) + const teamRows = await dbAll(db, + `SELECT ci.upload_id, ci.team, COUNT(ci.id) AS count + FROM compliance_items ci + WHERE ci.team IS NOT NULL + GROUP BY ci.upload_id, ci.team` + ); + + const teamMap = {}; + teamRows.forEach(r => { + if (!teamMap[r.upload_id]) teamMap[r.upload_id] = {}; + teamMap[r.upload_id][r.team] = r.count; + }); + + const trends = uploads.map(u => ({ + report_date: u.report_date, + new_count: u.new_count, + recurring_count: u.recurring_count, + resolved_count: u.resolved_count, + total_active: u.total_active, + STEAM: teamMap[u.id]?.STEAM || 0, + 'ACCESS-ENG': teamMap[u.id]?.['ACCESS-ENG'] || 0, + 'ACCESS-OPS': teamMap[u.id]?.['ACCESS-OPS'] || 0, + INTELDEV: teamMap[u.id]?.INTELDEV || 0, + })); + + res.json({ trends }); + } catch (err) { + console.error('[Compliance] GET /trends error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /mttr + // Mean time to resolution (calendar days) per team, for resolved items. + // ----------------------------------------------------------------------- + router.get('/mttr', async (req, res) => { + try { + const rows = await dbAll(db, + `SELECT + ci.team, + ROUND(AVG(JULIANDAY(ru.report_date) - JULIANDAY(fu.report_date)), 1) AS avg_days, + COUNT(*) AS resolved_count + FROM compliance_items ci + JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id + JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id + WHERE ci.resolved_upload_id IS NOT NULL + AND fu.report_date IS NOT NULL + AND ru.report_date IS NOT NULL + GROUP BY ci.team + ORDER BY avg_days DESC` + ); + res.json({ mttr: rows }); + } catch (err) { + console.error('[Compliance] GET /mttr error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /top-recurring + // Active findings grouped by team + metric_id, sorted by seen_count desc. + // Identifies chronic compliance gaps that keep reappearing. + // ----------------------------------------------------------------------- + router.get('/top-recurring', async (req, res) => { + try { + const rows = await dbAll(db, + `SELECT team, metric_id, metric_desc, seen_count, COUNT(*) AS host_count + FROM compliance_items + WHERE status = 'active' + GROUP BY team, metric_id + ORDER BY seen_count DESC, host_count DESC + LIMIT 20` + ); + res.json({ items: rows }); + } catch (err) { + console.error('[Compliance] GET /top-recurring error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /category-trend + // Active item counts per category per upload, for stacked area chart. + // ----------------------------------------------------------------------- + router.get('/category-trend', async (req, res) => { + try { + const rows = await dbAll(db, + `SELECT cu.report_date, COALESCE(ci.category, 'Unknown') AS category, COUNT(ci.id) AS count + FROM compliance_uploads cu + JOIN compliance_items ci ON ci.upload_id = cu.id + GROUP BY cu.id, category + ORDER BY cu.report_date ASC` + ); + res.json({ categoryTrend: rows }); + } catch (err) { + console.error('[Compliance] GET /category-trend error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + return router; } diff --git a/docs/time-based-reporting-recommendations.md b/docs/time-based-reporting-recommendations.md new file mode 100644 index 0000000..5069add --- /dev/null +++ b/docs/time-based-reporting-recommendations.md @@ -0,0 +1,333 @@ +# Time-Based Reporting Recommendations +**Date:** 2026-04-02 +**Author:** Engineering (Claude Code) +**Status:** Draft — for director review + +--- + +## Executive Summary + +This document analyzes the current CVE Dashboard data model and recommends a set of time-based visualizations that can be added to the Reporting page. Recommendations are grouped by feasibility: **Tier 1** can be built with data already in the database, **Tier 2** requires a lightweight new tracking table, and **Tier 3** requires structural additions. + +--- + +## Current Data Inventory + +### What Already Has Time-Series History + +| Source | Table | Date Fields | History? | +|--------|-------|-------------|----------| +| Compliance uploads | `compliance_uploads` | `report_date`, `uploaded_at` | **Yes** — one row per report cycle | +| Compliance items | `compliance_items` | `created_at`, `first_seen_upload_id`, `resolved_upload_id` | **Yes** — tracks lifecycle | +| Archer tickets | `archer_tickets` | `created_at`, `updated_at` | **Yes** — full history | +| Todo queue | `ivanti_todo_queue` | `created_at`, `updated_at` | **Yes** — by action | +| Finding notes | `ivanti_finding_notes` | `updated_at` | **Yes** — note activity | + +### What Is Point-in-Time Only (no history yet) + +| Source | Table | Problem | +|--------|-------|---------| +| Ivanti findings | `ivanti_findings_cache` | Single-row cache — overwritten on every sync | +| Ivanti counts | `ivanti_counts_cache` | Single-row cache — no snapshots stored | +| FP workflow states | Computed from `findings_json` | Ephemeral — not persisted historically | + +--- + +## Tier 1 Recommendations — Build Now (No Schema Changes) + +All of these use data that is already in the database. + +--- + +### 1.1 Compliance Trend Line — Total Active Findings Over Time + +**Description:** A line chart showing the total number of active (non-compliant) items per compliance upload date. This directly answers "are we improving over time?" + +**Data Source:** +```sql +SELECT + cu.report_date, + COUNT(ci.id) AS active_count +FROM compliance_uploads cu +JOIN compliance_items ci ON ci.upload_id = cu.id AND ci.status = 'active' +GROUP BY cu.id +ORDER BY cu.report_date ASC; +``` + +**Chart Type:** Line chart with data points per upload +**Axes:** X = Report Date, Y = Number of Active Findings +**Value-add:** Overlay a trend line (linear regression) to show trajectory + +--- + +### 1.2 New / Recurring / Resolved Bar Chart — Per Report Cycle + +**Description:** A grouped or stacked bar chart showing the delta breakdown for each compliance upload: how many findings were newly introduced, how many recurred from a prior cycle, and how many were resolved. + +**Data Source:** Already computed and stored in `compliance_uploads`: +```sql +SELECT report_date, new_count, recurring_count, resolved_count +FROM compliance_uploads +ORDER BY report_date ASC; +``` + +**Chart Type:** Stacked bar chart (one bar per upload date) +**Legend:** New (red/amber), Recurring (yellow), Resolved (green) +**Value-add:** Shows whether each reporting cycle is improving (more resolved than new) or degrading + +--- + +### 1.3 Team Compliance Health Over Time — Multi-Line Chart + +**Description:** A multi-line chart showing the active finding count per team per upload date. Answers "which team is trending better or worse?" + +**Data Source:** +```sql +SELECT + cu.report_date, + ci.team, + COUNT(ci.id) AS active_count +FROM compliance_uploads cu +JOIN compliance_items ci ON ci.upload_id = cu.id AND ci.status = 'active' +GROUP BY cu.id, ci.team +ORDER BY cu.report_date ASC; +``` + +**Chart Type:** Multi-line chart (one line per team) +**Teams:** STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV +**Value-add:** Immediately visible which team is outlier or improving fastest + +--- + +### 1.4 Mean Time to Resolution (MTTR) — Per Team + +**Description:** A bar chart showing average number of upload cycles between when a finding first appeared and when it was resolved, broken out by team. + +**Data Source:** +```sql +SELECT + ci.team, + AVG(ci.resolved_upload_id - ci.first_seen_upload_id) AS avg_cycles_to_resolve, + COUNT(*) AS resolved_count +FROM compliance_items ci +WHERE ci.resolved_upload_id IS NOT NULL +GROUP BY ci.team; +``` + +**Chart Type:** Horizontal bar chart +**Axes:** Y = Team, X = Average Cycles to Resolution +**Value-add:** Normalize to calendar days by joining with upload dates for true MTTR in days + +--- + +### 1.5 Recurring Findings Heatmap — Seen Count Distribution + +**Description:** A heatmap or bubble chart showing findings grouped by how many times they have recurred (`seen_count`). Identifies chronic, long-standing compliance gaps. + +**Data Source:** +```sql +SELECT + team, + metric_id, + metric_desc, + seen_count, + COUNT(*) AS host_count +FROM compliance_items +WHERE status = 'active' +GROUP BY team, metric_id +ORDER BY seen_count DESC; +``` + +**Chart Type:** Horizontal bar chart sorted by `seen_count`, grouped by team +**Value-add:** Highlights the "chronic" findings that repeatedly appear — high priority for remediation + +--- + +### 1.6 Archer Exception Ticket Status Over Time + +**Description:** A line chart or cumulative area chart showing Archer ticket status transitions over time using `created_at` and `updated_at`. + +**Data Source:** +```sql +SELECT + DATE(created_at) AS date, + status, + COUNT(*) AS count +FROM archer_tickets +GROUP BY DATE(created_at), status +ORDER BY date ASC; +``` + +**Chart Type:** Stacked area chart +**Statuses:** Draft, Open, Under Review, Accepted +**Value-add:** Tracks exception request pipeline velocity — are exceptions getting processed or stacking up? + +--- + +### 1.7 Compliance Category Breakdown Over Time + +**Description:** A stacked area chart showing what categories of compliance failures are driving the total over time (if the `category` field in `compliance_items` is populated). + +**Data Source:** +```sql +SELECT + cu.report_date, + ci.category, + COUNT(ci.id) AS count +FROM compliance_uploads cu +JOIN compliance_items ci ON ci.upload_id = cu.id AND ci.status = 'active' +WHERE ci.category IS NOT NULL +GROUP BY cu.id, ci.category +ORDER BY cu.report_date ASC; +``` + +**Chart Type:** Stacked area chart +**Value-add:** Shows whether one category dominates or if failures are spread across areas + +--- + +## Tier 2 Recommendations — Lightweight Schema Addition Required + +These require adding one new table to persist snapshots of data that is currently overwritten on each sync. + +--- + +### 2.1 Ivanti Findings Count Over Time — Open vs Closed Trend + +**Description:** The single most-requested metric: "are we making progress on vulnerabilities?" A line chart showing open and closed Ivanti finding counts over time. + +**Problem:** The current `ivanti_counts_cache` is a single-row table overwritten on each sync. No history is kept. + +**Solution:** Add a `ivanti_counts_history` table and append a row on every successful sync: +```sql +CREATE TABLE ivanti_counts_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + open_count INTEGER NOT NULL, + closed_count INTEGER NOT NULL, + recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +``` + +**Backend change:** In the sync route (`POST /api/ivanti/findings/sync`), after updating the cache, also `INSERT INTO ivanti_counts_history`. + +**New API endpoint:** `GET /api/ivanti/findings/counts/history` +```sql +SELECT open_count, closed_count, recorded_at +FROM ivanti_counts_history +ORDER BY recorded_at ASC; +``` + +**Chart Type:** Dual-line chart +**Lines:** Open findings (red), Closed findings (green) +**Value-add:** Most direct measure of vulnerability remediation velocity + +--- + +### 2.2 FP Workflow State Snapshots Over Time + +**Description:** A stacked area or line chart showing how FP workflow states (Actionable, Requested, Approved, Rejected, Expired) trend over sync cycles. + +**Solution:** Add a `ivanti_fp_workflow_history` table: +```sql +CREATE TABLE ivanti_fp_workflow_history ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + state TEXT NOT NULL, + finding_count INTEGER NOT NULL DEFAULT 0, + id_count INTEGER NOT NULL DEFAULT 0, + recorded_at DATETIME DEFAULT CURRENT_TIMESTAMP +); +``` + +**Chart Type:** Stacked area chart +**Value-add:** Shows whether FP requests are being worked through or stacking up in "Requested" state + +--- + +### 2.3 Todo Queue Velocity — Items Added vs Completed Per Week + +**Description:** A bar chart showing weekly queue throughput (items added vs items marked complete). + +**Data Source:** Already available in `ivanti_todo_queue.created_at` and `updated_at` + `status = 'complete'`: +```sql +SELECT + STRFTIME('%Y-W%W', created_at) AS week, + COUNT(*) AS items_added, + SUM(CASE WHEN status = 'complete' THEN 1 ELSE 0 END) AS items_completed +FROM ivanti_todo_queue +GROUP BY week +ORDER BY week ASC; +``` + +**Chart Type:** Grouped bar chart (weekly) +**Value-add:** Measures operational pace of the team's workflow action throughput + +--- + +## Tier 3 Recommendations — Structural Additions (Future Consideration) + +These require more significant changes but would provide powerful long-term reporting. + +--- + +### 3.1 Finding Age / Dwell Time Distribution + +**Description:** A histogram showing how long open findings have been open (age in days). The `lastFoundOn` field exists in the Ivanti findings JSON but is not persisted to a structured table. + +**Requirement:** Parse and store `lastFoundOn` from findings JSON into a structured column during sync. + +**Value-add:** Highlights findings that have been open for 90+ days — high-priority remediation targets. + +--- + +### 3.2 SLA Breach Trends + +**Description:** Track how many findings breach SLA (Due Date exceeded) over time. Currently SLA status is computed in the frontend on-the-fly. + +**Requirement:** Add SLA breach tracking during sync — stamp findings that cross SLA date. + +**Value-add:** Compliance and audit reporting for SLA adherence metrics. + +--- + +## Recommended Implementation Order + +| Priority | Chart | Effort | Impact | +|----------|-------|--------|--------| +| 1 | 1.2 — New/Recurring/Resolved bar chart | Low (data ready) | High | +| 2 | 1.1 — Compliance trend line | Low (data ready) | High | +| 3 | 1.3 — Team health multi-line | Low (data ready) | High | +| 4 | 2.1 — Ivanti open/closed history | Medium (new table) | Very High | +| 5 | 1.4 — MTTR per team | Low (data ready) | Medium | +| 6 | 1.6 — Archer ticket pipeline | Low (data ready) | Medium | +| 7 | 2.3 — Queue velocity | Low (data ready) | Medium | +| 8 | 1.5 — Recurring findings heatmap | Low (data ready) | Medium | +| 9 | 2.2 — FP workflow snapshots | Medium (new table) | Medium | +| 10 | 1.7 — Category breakdown | Low (data ready) | Low–Medium | + +--- + +## Charting Library Consideration + +The current implementation uses **hand-rolled SVG donut charts** (no external library). For time-series line/bar/area charts, the team should decide: + +| Option | Pros | Cons | +|--------|------|------| +| **Continue hand-rolled SVG** | Zero dependencies, full style control | Significant effort for axes, labels, tooltips | +| **Recharts** (React-native) | Well-matched to React 19, composable, responsive | ~500KB dependency | +| **Chart.js via react-chartjs-2** | Mature, widely documented | Less React-idiomatic | +| **Lightweight: uPlot or Chart.xkcd** | Very small bundle | Less community support | + +**Recommendation:** Recharts aligns best with the React 19 stack and allows declaring charts as JSX components consistent with the existing code style. It supports all chart types listed above. + +--- + +## Notes for Director Review + +- All **Tier 1** recommendations can be implemented with zero database migrations — the data is already there. +- The **single highest-value addition** is `2.1 — Ivanti open/closed count history`, as it captures the most direct remediation progress metric. It only requires one new table and one line added to the sync handler. +- **Compliance charts (1.1–1.5)** will only be meaningful once multiple compliance uploads have been committed. If only 1–2 uploads exist currently, the trend will not show much until more data accumulates — but building the charts now means data will automatically populate them. +- All queries listed above have been validated against the actual database schema. + +--- + +*Next step: Review with director, confirm priority order, then schedule sprint for implementation.* diff --git a/frontend/package.json b/frontend/package.json index cd53864..b0f1df0 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -12,6 +12,7 @@ "react-dom": "^19.2.4", "react-markdown": "^10.1.0", "react-scripts": "5.0.1", + "recharts": "^3.8.1", "web-vitals": "^2.1.4", "xlsx": "^0.18.5" }, diff --git a/frontend/src/components/pages/ComplianceChartsPanel.js b/frontend/src/components/pages/ComplianceChartsPanel.js new file mode 100644 index 0000000..f74436d --- /dev/null +++ b/frontend/src/components/pages/ComplianceChartsPanel.js @@ -0,0 +1,424 @@ +// ComplianceChartsPanel.js +// Tier-1 time-based compliance charts using Recharts. +// Charts rendered: Active Findings Over Time, Change per Cycle, +// Team Health, MTTR by Team, Persistent Findings, Archer Pipeline. + +import React, { useState, useEffect, useMemo } from 'react'; +import { + LineChart, Line, + BarChart, Bar, + XAxis, YAxis, CartesianGrid, + Tooltip, Legend, + ResponsiveContainer, +} from 'recharts'; +import { TrendingUp, ChevronDown, ChevronUp, Loader } from 'lucide-react'; + +const API_BASE = process.env.REACT_APP_API_BASE || 'http://localhost:3001/api'; + +const TEAL = '#14B8A6'; + +const TEAM_COLORS = { + 'STEAM': '#0EA5E9', + 'ACCESS-ENG': '#F59E0B', + 'ACCESS-OPS': '#8B5CF6', + 'INTELDEV': '#10B981', +}; + +const ARCHER_STATUS_COLORS = { + 'Draft': '#475569', + 'Open': '#0EA5E9', + 'Under Review': '#F59E0B', + 'Accepted': '#10B981', +}; + +// --------------------------------------------------------------------------- +// Shared style tokens +// --------------------------------------------------------------------------- +const AXIS_STYLE = { fontFamily: 'monospace', fontSize: '0.6rem', fill: '#475569' }; +const GRID_STYLE = { stroke: 'rgba(255,255,255,0.05)', strokeDasharray: '3 3' }; +const LEGEND_STYLE = { fontFamily: 'monospace', fontSize: '0.62rem', color: '#64748B' }; + +// --------------------------------------------------------------------------- +// Custom dark tooltip +// --------------------------------------------------------------------------- +function DarkTooltip({ active, payload, label }) { + if (!active || !payload?.length) return null; + return ( +