From 1f3833989a63cbd700d0a1862b0e899f9d3a1c1b Mon Sep 17 00:00:00 2001 From: Jordan Ramos Date: Mon, 8 Jun 2026 07:59:56 -0600 Subject: [PATCH] Replace CCP cross-metric aggregates with per-metric summary views Add per-metric stats and trend endpoints to vclMultiVertical.js. Refactor CCPMetricsPage to use a unified MetricSelector that drives StatsBar, TrendChart, DonutChart, and ForecastBurndownChart for the selected metric only. Remove the separate Per-Metric Forecast Burndown section (now integrated). Fix trend query double-counting when multiple uploads exist per vertical per month. Closes #25 --- backend/routes/vclMultiVertical.js | 291 ++++++++++++++++++ .../src/components/pages/CCPMetricsPage.js | 210 +++++++++---- 2 files changed, 439 insertions(+), 62 deletions(-) diff --git a/backend/routes/vclMultiVertical.js b/backend/routes/vclMultiVertical.js index 405d531..4797a9e 100644 --- a/backend/routes/vclMultiVertical.js +++ b/backend/routes/vclMultiVertical.js @@ -1244,6 +1244,144 @@ function createVCLMultiVerticalRouter(upload) { } }); + // ----------------------------------------------------------------------- + // GET /metric/:id/stats — Per-metric summary statistics + donut breakdown + // ----------------------------------------------------------------------- + + /** + * GET /metric/:id/stats + * Returns summary statistics and donut breakdown for a single metric, + * aggregated across all verticals using only ALL: rollup rows. + * + * @method GET + * @route /metric/:id/stats + * @param {string} id — metric identifier (e.g., "2.3.6i") + * + * @response 200 + * { + * metric_id: string, + * metric_desc: string, + * category: string, + * total_devices: number, + * compliant: number, + * non_compliant: number, + * compliance_pct: number, + * target: number, + * donut: { + * blocked: { count: number, pct: number }, + * in_progress: { count: number, pct: number } + * } + * } + * @response 400 { error: string } — metric ID exceeds 50 characters + * @response 500 { error: string } + */ + router.get('/metric/:id/stats', async (req, res) => { + const metricId = req.params.id; + if (!metricId || metricId.length > 50) { + return res.status(400).json({ error: 'Invalid metric ID' }); + } + + try { + // Get latest upload ID per vertical (same pattern as existing /stats endpoint) + const { rows: latestUploads } = await pool.query(` + SELECT DISTINCT ON (vertical) id, vertical + FROM compliance_uploads + WHERE vertical IS NOT NULL + ORDER BY vertical, id DESC + `); + + if (latestUploads.length === 0) { + return res.json({ + metric_id: metricId, + metric_desc: '', + category: '', + total_devices: 0, + compliant: 0, + non_compliant: 0, + compliance_pct: 0, + target: 0, + donut: { + blocked: { count: 0, pct: 0 }, + in_progress: { count: 0, pct: 0 }, + }, + }); + } + + const latestUploadIds = latestUploads.map(u => u.id); + + // Query vcl_multi_vertical_summary for this metric, ALL: rollup rows only + const { rows: summaryRows } = await pool.query(` + SELECT metric_desc, category, total, compliant, non_compliant, target + FROM vcl_multi_vertical_summary + WHERE upload_id = ANY($1) AND metric_id = $2 AND team LIKE 'ALL:%' + `, [latestUploadIds, metricId]); + + // If metric not found, return zero-filled response (HTTP 200, not 404) + if (summaryRows.length === 0) { + return res.json({ + metric_id: metricId, + metric_desc: '', + category: '', + total_devices: 0, + compliant: 0, + non_compliant: 0, + compliance_pct: 0, + target: 0, + donut: { + blocked: { count: 0, pct: 0 }, + in_progress: { count: 0, pct: 0 }, + }, + }); + } + + // Aggregate across verticals + let totalDevices = 0, totalCompliant = 0, totalNonCompliant = 0; + let targetSum = 0, targetCount = 0; + let metricDesc = ''; + let category = ''; + + for (const row of summaryRows) { + totalDevices += row.total || 0; + totalCompliant += row.compliant || 0; + totalNonCompliant += row.non_compliant || 0; + targetSum += parseFloat(row.target) || 0; + targetCount++; + // Derive metric_desc and category from first non-empty value + if (!metricDesc && row.metric_desc) metricDesc = row.metric_desc; + if (!category && row.category) category = row.category; + } + + const target = targetCount > 0 ? Math.round((targetSum / targetCount) * 100) / 100 : 0; + const compliancePct = totalDevices > 0 ? Math.round((totalCompliant / totalDevices) * 100) : 0; + + // Donut breakdown: query compliance_items for this metric + const { rows: donutRows } = await pool.query(` + SELECT hostname, MAX(resolution_date) AS resolution_date + FROM compliance_items + WHERE metric_id = $1 AND status = 'active' AND vertical IS NOT NULL + GROUP BY hostname + `, [metricId]); + + const donutItems = donutRows.map(r => ({ resolution_date: r.resolution_date })); + const donut = categorizeNonCompliant(donutItems); + + res.json({ + metric_id: metricId, + metric_desc: metricDesc, + category: category, + total_devices: totalDevices, + compliant: totalCompliant, + non_compliant: totalNonCompliant, + compliance_pct: compliancePct, + target: target, + donut, + }); + } catch (err) { + console.error('[VCL Multi] GET /metric/:id/stats error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + // ----------------------------------------------------------------------- // GET /metric/:id/verticals — Per-vertical breakdown for a specific metric // ----------------------------------------------------------------------- @@ -1520,6 +1658,159 @@ function createVCLMultiVerticalRouter(upload) { } }); + // ----------------------------------------------------------------------- + // GET /metric/:id/trend — Per-metric monthly compliance trend with forecast + // ----------------------------------------------------------------------- + + /** + * GET /metric/:id/trend + * Returns monthly compliance history with linear regression forecast for a + * single metric. Groups by report month from compliance_uploads, aggregating + * only rollup rows (team LIKE 'ALL:%'). + * + * @method GET + * @route /metric/:id/trend + * @param {string} id — metric identifier (e.g., "2.3.6i") + * + * @response 200 + * { + * months: Array<{ + * month: string, + * compliant_count: number|null, + * non_compliant_count: number|null, + * total_devices: number|null, + * compliance_pct: number|null, + * forecast_pct: number|null, + * target: number + * }> + * } + * @response 400 { error: string } — metric ID exceeds 50 characters + * @response 500 { error: string } + */ + router.get('/metric/:id/trend', async (req, res) => { + const metricId = req.params.id; + if (!metricId || metricId.length > 50) { + return res.status(400).json({ error: 'Invalid metric ID' }); + } + + try { + // Get the metric's target value from the latest uploads (same pattern as stats endpoint) + const { rows: latestUploads } = await pool.query(` + SELECT DISTINCT ON (vertical) id, vertical + FROM compliance_uploads + WHERE vertical IS NOT NULL + ORDER BY vertical, id DESC + `); + + if (latestUploads.length === 0) { + return res.json({ months: [] }); + } + + const latestUploadIds = latestUploads.map(u => u.id); + + // Get target from the latest uploads for this metric + const { rows: targetRows } = await pool.query(` + SELECT ROUND(AVG(target::numeric), 2) AS target + FROM vcl_multi_vertical_summary + WHERE upload_id = ANY($1) AND metric_id = $2 AND team LIKE 'ALL:%' + `, [latestUploadIds, metricId]); + + const metricTarget = targetRows.length > 0 && targetRows[0].target !== null + ? parseFloat(targetRows[0].target) + : 0; + + // Join vcl_multi_vertical_summary with compliance_uploads to group by report month. + // Use only the latest upload per vertical per month to avoid double-counting + // when a vertical is re-uploaded multiple times in the same month. + const { rows: monthlyData } = await pool.query(` + WITH latest_per_vertical_month AS ( + SELECT DISTINCT ON (cu.vertical, COALESCE(SUBSTRING(cu.report_date FROM 1 FOR 7), TO_CHAR(cu.uploaded_at, 'YYYY-MM'))) + cu.id AS upload_id, + cu.vertical, + COALESCE(SUBSTRING(cu.report_date FROM 1 FOR 7), TO_CHAR(cu.uploaded_at, 'YYYY-MM')) AS report_month + FROM compliance_uploads cu + WHERE cu.vertical IS NOT NULL + ORDER BY cu.vertical, COALESCE(SUBSTRING(cu.report_date FROM 1 FOR 7), TO_CHAR(cu.uploaded_at, 'YYYY-MM')), cu.id DESC + ) + SELECT lvm.report_month, + SUM(s.compliant)::int AS compliant, + SUM(s.non_compliant)::int AS non_compliant, + SUM(s.total)::int AS total + FROM vcl_multi_vertical_summary s + JOIN latest_per_vertical_month lvm ON s.upload_id = lvm.upload_id + WHERE s.metric_id = $1 AND s.team LIKE 'ALL:%' + GROUP BY lvm.report_month + ORDER BY lvm.report_month ASC + `, [metricId]); + + // If metric not found in any historical data, return empty months + if (monthlyData.length === 0) { + return res.json({ months: [] }); + } + + // Build historical months with compliance_pct + const months = monthlyData.map(row => { + const pct = row.total > 0 + ? Math.round((row.compliant / row.total) * 100 * 10) / 10 + : 0; + return { + month: row.report_month, + compliant_count: row.compliant, + non_compliant_count: row.non_compliant, + total_devices: row.total, + compliance_pct: pct, + forecast_pct: null, + target: metricTarget, + }; + }); + + // Compute forecast using linear regression if we have 3+ months + if (months.length >= 3) { + const n = months.length; + let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0; + for (let i = 0; i < n; i++) { + sumX += i; + sumY += months[i].compliance_pct; + sumXY += i * months[i].compliance_pct; + sumX2 += i * i; + } + const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX); + const intercept = (sumY - slope * sumX) / n; + + // Set forecast_pct on the last historical month as the transition point + months[n - 1].forecast_pct = months[n - 1].compliance_pct; + + // Project forward 3 months + for (let i = 0; i < 3; i++) { + const futureIdx = n + i; + const forecastPct = Math.min(100.0, Math.max(0.0, + Math.round((slope * futureIdx + intercept) * 10) / 10 + )); + + const lastMonth = months[months.length - 1].month; + const [year, mon] = lastMonth.split('-').map(Number); + const futureDate = new Date(year, mon - 1 + 1, 1); + const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`; + + months.push({ + month: futureMonth, + compliant_count: null, + non_compliant_count: null, + total_devices: null, + compliance_pct: null, + forecast_pct: forecastPct, + target: metricTarget, + }); + } + } + + res.json({ months }); + } catch (err) { + console.error('[VCL Multi] GET /metric/:id/trend error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + // ----------------------------------------------------------------------- // GET /metric/:metricId/forecast-burndown — Per-metric forecast burndown // ----------------------------------------------------------------------- diff --git a/frontend/src/components/pages/CCPMetricsPage.js b/frontend/src/components/pages/CCPMetricsPage.js index 23a43fe..d1886f1 100644 --- a/frontend/src/components/pages/CCPMetricsPage.js +++ b/frontend/src/components/pages/CCPMetricsPage.js @@ -248,7 +248,7 @@ function TrendChart({ months }) { - + @@ -258,6 +258,7 @@ function TrendChart({ months }) { // --------------------------------------------------------------------------- // Aggregated Burndown Chart // --------------------------------------------------------------------------- +// eslint-disable-next-line no-unused-vars function AggregatedBurndownChart({ data, loading, error }) { if (loading) { return ( @@ -1540,22 +1541,30 @@ function ForecastBurndownChart({ metricId }) { // --------------------------------------------------------------------------- export default function CCPMetricsPage() { const { isAdmin, canWrite } = useAuth(); - const [stats, setStats] = useState(null); - const [trend, setTrend] = useState(null); + // Cross-metric aggregate stats — retained only for metric_breakdown (MetricBreakdownPanel) + const [metricBreakdownData, setMetricBreakdownData] = useState(null); const [metricsData, setMetricsData] = useState(null); - const [burndownData, setBurndownData] = useState(null); - const [burndownLoading, setBurndownLoading] = useState(true); - const [burndownError, setBurndownError] = useState(null); const [loading, setLoading] = useState(true); const [error, setError] = useState(null); const [showUpload, setShowUpload] = useState(false); const [showManage, setShowManage] = useState(false); const [showMetricBreakdown, setShowMetricBreakdown] = useState(false); - const [forecastMetric, setForecastMetric] = useState(null); + + // Per-metric CCP Summary state (Task 4.1) + const [selectedCCPMetric, setSelectedCCPMetric] = useState(null); + const [metricStats, setMetricStats] = useState(null); + const [metricTrend, setMetricTrend] = useState(null); + const [metricStatsLoading, setMetricStatsLoading] = useState(false); + const [metricStatsError, setMetricStatsError] = useState(null); + const [metricTrendLoading, setMetricTrendLoading] = useState(false); + const [metricTrendError, setMetricTrendError] = useState(null); + + // Request counter for stale response discarding (Task 4.1 / Requirement 8.5) + const metricRequestCounter = useRef(0); // Drill-down state (metric-first hierarchy: metric → vertical → team) const [selectedMetric, setSelectedMetric] = useState(null); - const [selectedMetricData, setSelectedMetricData] = useState(null); // eslint-disable-line no-unused-vars + const [_selectedMetricData, setSelectedMetricData] = useState(null); const [selectedVertical, setSelectedVertical] = useState(null); const [selectedVerticalData, setSelectedVerticalData] = useState(null); const [selectedTeam, setSelectedTeam] = useState(null); @@ -1563,31 +1572,75 @@ export default function CCPMetricsPage() { const fetchData = useCallback(() => { setLoading(true); setError(null); - setBurndownLoading(true); - setBurndownError(null); Promise.all([ + // Retain /vcl-multi/stats only for metric_breakdown field used by MetricBreakdownPanel fetch(`${API_BASE}/compliance/vcl-multi/stats`, { credentials: 'include' }).then(r => { if (!r.ok) throw new Error('Failed to load stats'); return r.json(); }), - fetch(`${API_BASE}/compliance/vcl-multi/trend`, { credentials: 'include' }).then(r => { if (!r.ok) throw new Error('Failed to load trend'); return r.json(); }), fetch(`${API_BASE}/compliance/vcl-multi/metrics`, { credentials: 'include' }).then(r => { if (!r.ok) throw new Error('Failed to load metrics'); return r.json(); }), - ]).then(([statsData, trendData, metricsResult]) => { - setStats(statsData); - setTrend(trendData); + ]).then(([statsData, metricsResult]) => { + setMetricBreakdownData(statsData?.metric_breakdown || null); setMetricsData(metricsResult); setLoading(false); }).catch(err => { setError(err.message); setLoading(false); }); - - // Fetch burndown independently so a failure doesn't block the rest of the page - fetch(`${API_BASE}/compliance/vcl-multi/burndown`, { credentials: 'include' }) - .then(r => { if (!r.ok) throw new Error('Failed to load burndown'); return r.json(); }) - .then(data => { setBurndownData(data); setBurndownLoading(false); }) - .catch(err => { setBurndownError(err.message); setBurndownLoading(false); }); }, []); useEffect(() => { fetchData(); }, [fetchData]); + // Handle CCP metric selection (Task 4.2) + const handleCCPMetricSelect = useCallback((metricId) => { + setSelectedCCPMetric(metricId); + }, []); + + // Per-metric data fetching effect — will be wired in task 5.1 + // For now, just reset state when metric changes + useEffect(() => { + if (!selectedCCPMetric) { + setMetricStats(null); + setMetricTrend(null); + setMetricStatsLoading(false); + setMetricStatsError(null); + setMetricTrendLoading(false); + setMetricTrendError(null); + return; + } + + const currentRequest = ++metricRequestCounter.current; + setMetricStatsLoading(true); + setMetricStatsError(null); + setMetricTrendLoading(true); + setMetricTrendError(null); + + // Fetch per-metric stats + fetch(`${API_BASE}/compliance/vcl-multi/metric/${encodeURIComponent(selectedCCPMetric)}/stats`, { credentials: 'include' }) + .then(r => { if (!r.ok) throw new Error('Failed to load metric stats'); return r.json(); }) + .then(data => { + if (currentRequest !== metricRequestCounter.current) return; + setMetricStats(data); + setMetricStatsLoading(false); + }) + .catch(err => { + if (currentRequest !== metricRequestCounter.current) return; + setMetricStatsError(err.message); + setMetricStatsLoading(false); + }); + + // Fetch per-metric trend + fetch(`${API_BASE}/compliance/vcl-multi/metric/${encodeURIComponent(selectedCCPMetric)}/trend`, { credentials: 'include' }) + .then(r => { if (!r.ok) throw new Error('Failed to load metric trend'); return r.json(); }) + .then(data => { + if (currentRequest !== metricRequestCounter.current) return; + setMetricTrend(data); + setMetricTrendLoading(false); + }) + .catch(err => { + if (currentRequest !== metricRequestCounter.current) return; + setMetricTrendError(err.message); + setMetricTrendLoading(false); + }); + }, [selectedCCPMetric]); + const handleUploadComplete = () => { setShowUpload(false); fetchData(); @@ -1710,60 +1763,93 @@ export default function CCPMetricsPage() { )} - {!loading && !error && stats && ( + {!loading && !error && metricsData && ( <> - {/* Stats bar */} - setShowMetricBreakdown(!showMetricBreakdown)} - ncExpanded={showMetricBreakdown} - /> + {/* Metric Selector — drives CCP Summary (Task 4.2) */} +
+ +
- {/* Metric breakdown (revealed when Non-Compliant is clicked) */} - {showMetricBreakdown && ( - + {/* CCP Summary section — per-metric stats, trend, donut, burndown */} + {selectedCCPMetric && ( + <> + {/* Stats bar — driven by per-metric stats */} + {metricStatsLoading ? ( +
+ +
Loading metric stats...
+
+ ) : metricStatsError ? ( +
+ + {metricStatsError} +
+ ) : metricStats ? ( + setShowMetricBreakdown(!showMetricBreakdown)} + ncExpanded={showMetricBreakdown} + /> + ) : null} + + {/* Metric breakdown (revealed when Non-Compliant is clicked) — always cross-metric */} + {showMetricBreakdown && ( + + )} + + {/* Charts row — trend and donut */} +
+ {/* TrendChart — driven by per-metric trend */} + {metricTrendLoading ? ( +
+ +
Loading trend data...
+
+ ) : metricTrendError ? ( +
+ + {metricTrendError} +
+ ) : ( + + )} + + {/* DonutChart — driven by per-metric stats donut */} + {metricStatsLoading ? ( +
+ +
+ ) : metricStatsError ? ( +
+ No donut data available +
+ ) : ( + + )} +
+ + {/* Forecast Burndown — driven by selectedCCPMetric */} +
+ +
+ )} - {/* Charts row */} -
- - -
- - {/* Aggregated burndown forecast */} - - - {/* Per-Metric Forecast Burndown */} -
-

- Per-Metric Forecast Burndown -

- -
- -
-
- {/* Metrics overview table (metric-first model) */} setSelectedMetric(metricId)} /> - - {/* Last upload info */} - {stats.last_upload_date && ( -
- Last upload: {stats.last_upload_date} -
- )} )} - {!loading && !error && (!stats || !stats.vertical_breakdown || stats.vertical_breakdown.length === 0) && ( + {!loading && !error && !metricsData && (
No multi-vertical data yet