From 55238ec71e517e979c7600d65ffca2699d82f38a Mon Sep 17 00:00:00 2001 From: Jordan Ramos Date: Thu, 14 May 2026 12:01:19 -0600 Subject: [PATCH] Fix compliance stats to use Summary sheet data instead of item counts MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The compliance_items table only contains non-compliant devices (detail sheet rows). Compliant devices are never inserted — they only exist in the Summary sheet totals. This caused Compliant to show 0 and Compliance % to show 0% for all verticals. Fix: stats endpoint now reads from vcl_multi_vertical_summary (parsed Summary sheet data) for total/compliant/non-compliant counts. Snapshot creation also uses summary data for accurate trend charting. The compliance_items table is still used for: - Donut chart (blocked vs in-progress based on resolution_date) - Burndown forecast (devices with/without resolution dates) - Device drill-down (actual non-compliant device list) --- backend/routes/vclMultiVertical.js | 113 +++++++++++++++++------------ 1 file changed, 65 insertions(+), 48 deletions(-) diff --git a/backend/routes/vclMultiVertical.js b/backend/routes/vclMultiVertical.js index b62ab94..e7338b9 100644 --- a/backend/routes/vclMultiVertical.js +++ b/backend/routes/vclMultiVertical.js @@ -141,26 +141,26 @@ async function persistMultiVerticalUpload({ items, summary, reportDate, filename } // 6. Create/update compliance_snapshots for this vertical + // Use summary data for accurate totals (compliance_items only has non-compliant devices) const currentMonth = new Date().toISOString().slice(0, 7); - const { rows: verticalStats } = await client.query( - `SELECT - COUNT(DISTINCT hostname)::int AS total_devices, - COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant - FROM compliance_items - WHERE vertical = $1`, - [vertical] - ); - const vs = verticalStats[0] || { total_devices: 0, non_compliant: 0 }; - const totalDevices = vs.total_devices; - const compliant = totalDevices - vs.non_compliant; - const compPct = totalDevices > 0 ? Math.round((compliant / totalDevices) * 100 * 100) / 100 : 0; + let totalDevices = 0, snapshotCompliant = 0, snapshotNonCompliant = 0; + + if (summary && summary.entries && summary.entries.length > 0) { + for (const entry of summary.entries) { + totalDevices += entry.total || 0; + snapshotCompliant += entry.compliant || 0; + snapshotNonCompliant += entry.non_compliant || 0; + } + } + + const compPct = totalDevices > 0 ? Math.round((snapshotCompliant / totalDevices) * 100 * 100) / 100 : 0; await client.query( `INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct) VALUES ($1, $2, $3, $4, $5, $6) ON CONFLICT (snapshot_month, vertical) DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`, - [currentMonth, vertical, totalDevices, compliant, vs.non_compliant, compPct] + [currentMonth, vertical, totalDevices, snapshotCompliant, snapshotNonCompliant, compPct] ); return { uploadId, newCount, recurringCount, resolvedCount }; @@ -437,22 +437,53 @@ function createVCLMultiVerticalRouter(upload) { */ router.get('/stats', async (req, res) => { try { - // Aggregate device-level stats across all multi-vertical items - const { rows: statsRows } = await pool.query(` - SELECT - COUNT(DISTINCT hostname)::int AS total_devices, - COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant - FROM compliance_items + // Use Summary sheet data (vcl_multi_vertical_summary) for accurate totals. + // The compliance_items table only contains NON-COMPLIANT devices, so counting + // hostnames there gives 0 compliant. The Summary sheet has the real numbers. + + // Get the latest upload per vertical to pull summary data from + const { rows: latestUploads } = await pool.query(` + SELECT DISTINCT ON (vertical) id, vertical + FROM compliance_uploads WHERE vertical IS NOT NULL + ORDER BY vertical, id DESC `); - const raw = statsRows[0] || { total_devices: 0, non_compliant: 0 }; - const total_devices = raw.total_devices; - const non_compliant = raw.non_compliant; - const compliant = total_devices - non_compliant; - const compliance_pct = total_devices > 0 ? Math.round((compliant / total_devices) * 100) : 0; + if (latestUploads.length === 0) { + return res.json({ + stats: { total_devices: 0, compliant: 0, non_compliant: 0, compliance_pct: 0, target_pct: VCL_TARGET_PCT }, + donut: { blocked: { count: 0, pct: 0 }, in_progress: { count: 0, pct: 0 } }, + vertical_breakdown: [], + last_upload_date: null, + }); + } - // Donut: blocked vs in-progress across all verticals + const latestUploadIds = latestUploads.map(u => u.id); + + // Aggregate summary data from the latest upload per vertical + // Each row in vcl_multi_vertical_summary is one metric for one vertical. + // We sum across metrics per vertical to get vertical-level totals. + const { rows: verticalSummary } = await pool.query(` + SELECT vertical, + SUM(total)::int AS total_devices, + SUM(compliant)::int AS compliant, + SUM(non_compliant)::int AS non_compliant + FROM vcl_multi_vertical_summary + WHERE upload_id = ANY($1) + GROUP BY vertical + ORDER BY vertical + `, [latestUploadIds]); + + // Compute aggregated stats across all verticals + let aggTotal = 0, aggCompliant = 0, aggNonCompliant = 0; + for (const v of verticalSummary) { + aggTotal += v.total_devices; + aggCompliant += v.compliant; + aggNonCompliant += v.non_compliant; + } + const compliance_pct = aggTotal > 0 ? Math.round((aggCompliant / aggTotal) * 100) : 0; + + // Donut: blocked vs in-progress (from compliance_items — devices with/without resolution dates) const { rows: donutRows } = await pool.query(` SELECT hostname, MAX(resolution_date) AS resolution_date FROM compliance_items @@ -462,18 +493,6 @@ function createVCLMultiVerticalRouter(upload) { const donutItems = donutRows.map(r => ({ resolution_date: r.resolution_date })); const donut = categorizeNonCompliant(donutItems); - // Per-vertical breakdown - const { rows: verticalRows } = await pool.query(` - SELECT - vertical, - COUNT(DISTINCT hostname)::int AS total_devices, - COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant - FROM compliance_items - WHERE vertical IS NOT NULL - GROUP BY vertical - ORDER BY vertical - `); - // Get last upload date per vertical const { rows: uploadDates } = await pool.query(` SELECT vertical, MAX(report_date) AS last_upload @@ -484,30 +503,28 @@ function createVCLMultiVerticalRouter(upload) { const uploadDateMap = {}; uploadDates.forEach(r => { uploadDateMap[r.vertical] = r.last_upload; }); - // Get burndown data per vertical + // Get burndown data per vertical (from compliance_items — actual device records) const { rows: burndownRows } = await pool.query(` SELECT vertical, hostname, resolution_date FROM compliance_items WHERE vertical IS NOT NULL AND status = 'active' `); - // Group by vertical for burndown computation const burndownByVertical = {}; for (const row of burndownRows) { if (!burndownByVertical[row.vertical]) burndownByVertical[row.vertical] = []; burndownByVertical[row.vertical].push(row); } - const vertical_breakdown = verticalRows.map(v => { - const totalDev = v.total_devices; - const comp = totalDev - v.non_compliant; - const pct = totalDev > 0 ? Math.round((comp / totalDev) * 100) : 0; + // Build per-vertical breakdown using summary data for totals + const vertical_breakdown = verticalSummary.map(v => { + const pct = v.total_devices > 0 ? Math.round((v.compliant / v.total_devices) * 100) : 0; const items = burndownByVertical[v.vertical] || []; const burndown = computeVerticalBurndown(items); return { vertical: v.vertical, - total_devices: totalDev, - compliant: comp, + total_devices: v.total_devices, + compliant: v.compliant, non_compliant: v.non_compliant, compliance_pct: pct, blockers: burndown.blockers, @@ -518,9 +535,9 @@ function createVCLMultiVerticalRouter(upload) { res.json({ stats: { - total_devices, - compliant, - non_compliant, + total_devices: aggTotal, + compliant: aggCompliant, + non_compliant: aggNonCompliant, compliance_pct, target_pct: VCL_TARGET_PCT, },