Fix compliance stats to use Summary sheet data instead of item counts
The compliance_items table only contains non-compliant devices (detail sheet rows). Compliant devices are never inserted — they only exist in the Summary sheet totals. This caused Compliant to show 0 and Compliance % to show 0% for all verticals. Fix: stats endpoint now reads from vcl_multi_vertical_summary (parsed Summary sheet data) for total/compliant/non-compliant counts. Snapshot creation also uses summary data for accurate trend charting. The compliance_items table is still used for: - Donut chart (blocked vs in-progress based on resolution_date) - Burndown forecast (devices with/without resolution dates) - Device drill-down (actual non-compliant device list)
This commit is contained in:
Jordan Ramos
@@ -141,26 +141,26 @@ async function persistMultiVerticalUpload({ items, summary, reportDate, filename
|
||||
}
|
||||
|
||||
// 6. Create/update compliance_snapshots for this vertical
|
||||
// Use summary data for accurate totals (compliance_items only has non-compliant devices)
|
||||
const currentMonth = new Date().toISOString().slice(0, 7);
|
||||
const { rows: verticalStats } = await client.query(
|
||||
`SELECT
|
||||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||||
FROM compliance_items
|
||||
WHERE vertical = $1`,
|
||||
[vertical]
|
||||
);
|
||||
const vs = verticalStats[0] || { total_devices: 0, non_compliant: 0 };
|
||||
const totalDevices = vs.total_devices;
|
||||
const compliant = totalDevices - vs.non_compliant;
|
||||
const compPct = totalDevices > 0 ? Math.round((compliant / totalDevices) * 100 * 100) / 100 : 0;
|
||||
let totalDevices = 0, snapshotCompliant = 0, snapshotNonCompliant = 0;
|
||||
|
||||
if (summary && summary.entries && summary.entries.length > 0) {
|
||||
for (const entry of summary.entries) {
|
||||
totalDevices += entry.total || 0;
|
||||
snapshotCompliant += entry.compliant || 0;
|
||||
snapshotNonCompliant += entry.non_compliant || 0;
|
||||
}
|
||||
}
|
||||
|
||||
const compPct = totalDevices > 0 ? Math.round((snapshotCompliant / totalDevices) * 100 * 100) / 100 : 0;
|
||||
|
||||
await client.query(
|
||||
`INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (snapshot_month, vertical)
|
||||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||||
[currentMonth, vertical, totalDevices, compliant, vs.non_compliant, compPct]
|
||||
[currentMonth, vertical, totalDevices, snapshotCompliant, snapshotNonCompliant, compPct]
|
||||
);
|
||||
|
||||
return { uploadId, newCount, recurringCount, resolvedCount };
|
||||
@@ -437,22 +437,53 @@ function createVCLMultiVerticalRouter(upload) {
|
||||
*/
|
||||
router.get('/stats', async (req, res) => {
|
||||
try {
|
||||
// Aggregate device-level stats across all multi-vertical items
|
||||
const { rows: statsRows } = await pool.query(`
|
||||
SELECT
|
||||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||||
FROM compliance_items
|
||||
// Use Summary sheet data (vcl_multi_vertical_summary) for accurate totals.
|
||||
// The compliance_items table only contains NON-COMPLIANT devices, so counting
|
||||
// hostnames there gives 0 compliant. The Summary sheet has the real numbers.
|
||||
|
||||
// Get the latest upload per vertical to pull summary data from
|
||||
const { rows: latestUploads } = await pool.query(`
|
||||
SELECT DISTINCT ON (vertical) id, vertical
|
||||
FROM compliance_uploads
|
||||
WHERE vertical IS NOT NULL
|
||||
ORDER BY vertical, id DESC
|
||||
`);
|
||||
|
||||
const raw = statsRows[0] || { total_devices: 0, non_compliant: 0 };
|
||||
const total_devices = raw.total_devices;
|
||||
const non_compliant = raw.non_compliant;
|
||||
const compliant = total_devices - non_compliant;
|
||||
const compliance_pct = total_devices > 0 ? Math.round((compliant / total_devices) * 100) : 0;
|
||||
if (latestUploads.length === 0) {
|
||||
return res.json({
|
||||
stats: { total_devices: 0, compliant: 0, non_compliant: 0, compliance_pct: 0, target_pct: VCL_TARGET_PCT },
|
||||
donut: { blocked: { count: 0, pct: 0 }, in_progress: { count: 0, pct: 0 } },
|
||||
vertical_breakdown: [],
|
||||
last_upload_date: null,
|
||||
});
|
||||
}
|
||||
|
||||
// Donut: blocked vs in-progress across all verticals
|
||||
const latestUploadIds = latestUploads.map(u => u.id);
|
||||
|
||||
// Aggregate summary data from the latest upload per vertical
|
||||
// Each row in vcl_multi_vertical_summary is one metric for one vertical.
|
||||
// We sum across metrics per vertical to get vertical-level totals.
|
||||
const { rows: verticalSummary } = await pool.query(`
|
||||
SELECT vertical,
|
||||
SUM(total)::int AS total_devices,
|
||||
SUM(compliant)::int AS compliant,
|
||||
SUM(non_compliant)::int AS non_compliant
|
||||
FROM vcl_multi_vertical_summary
|
||||
WHERE upload_id = ANY($1)
|
||||
GROUP BY vertical
|
||||
ORDER BY vertical
|
||||
`, [latestUploadIds]);
|
||||
|
||||
// Compute aggregated stats across all verticals
|
||||
let aggTotal = 0, aggCompliant = 0, aggNonCompliant = 0;
|
||||
for (const v of verticalSummary) {
|
||||
aggTotal += v.total_devices;
|
||||
aggCompliant += v.compliant;
|
||||
aggNonCompliant += v.non_compliant;
|
||||
}
|
||||
const compliance_pct = aggTotal > 0 ? Math.round((aggCompliant / aggTotal) * 100) : 0;
|
||||
|
||||
// Donut: blocked vs in-progress (from compliance_items — devices with/without resolution dates)
|
||||
const { rows: donutRows } = await pool.query(`
|
||||
SELECT hostname, MAX(resolution_date) AS resolution_date
|
||||
FROM compliance_items
|
||||
@@ -462,18 +493,6 @@ function createVCLMultiVerticalRouter(upload) {
|
||||
const donutItems = donutRows.map(r => ({ resolution_date: r.resolution_date }));
|
||||
const donut = categorizeNonCompliant(donutItems);
|
||||
|
||||
// Per-vertical breakdown
|
||||
const { rows: verticalRows } = await pool.query(`
|
||||
SELECT
|
||||
vertical,
|
||||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||||
FROM compliance_items
|
||||
WHERE vertical IS NOT NULL
|
||||
GROUP BY vertical
|
||||
ORDER BY vertical
|
||||
`);
|
||||
|
||||
// Get last upload date per vertical
|
||||
const { rows: uploadDates } = await pool.query(`
|
||||
SELECT vertical, MAX(report_date) AS last_upload
|
||||
@@ -484,30 +503,28 @@ function createVCLMultiVerticalRouter(upload) {
|
||||
const uploadDateMap = {};
|
||||
uploadDates.forEach(r => { uploadDateMap[r.vertical] = r.last_upload; });
|
||||
|
||||
// Get burndown data per vertical
|
||||
// Get burndown data per vertical (from compliance_items — actual device records)
|
||||
const { rows: burndownRows } = await pool.query(`
|
||||
SELECT vertical, hostname, resolution_date
|
||||
FROM compliance_items
|
||||
WHERE vertical IS NOT NULL AND status = 'active'
|
||||
`);
|
||||
// Group by vertical for burndown computation
|
||||
const burndownByVertical = {};
|
||||
for (const row of burndownRows) {
|
||||
if (!burndownByVertical[row.vertical]) burndownByVertical[row.vertical] = [];
|
||||
burndownByVertical[row.vertical].push(row);
|
||||
}
|
||||
|
||||
const vertical_breakdown = verticalRows.map(v => {
|
||||
const totalDev = v.total_devices;
|
||||
const comp = totalDev - v.non_compliant;
|
||||
const pct = totalDev > 0 ? Math.round((comp / totalDev) * 100) : 0;
|
||||
// Build per-vertical breakdown using summary data for totals
|
||||
const vertical_breakdown = verticalSummary.map(v => {
|
||||
const pct = v.total_devices > 0 ? Math.round((v.compliant / v.total_devices) * 100) : 0;
|
||||
const items = burndownByVertical[v.vertical] || [];
|
||||
const burndown = computeVerticalBurndown(items);
|
||||
|
||||
return {
|
||||
vertical: v.vertical,
|
||||
total_devices: totalDev,
|
||||
compliant: comp,
|
||||
total_devices: v.total_devices,
|
||||
compliant: v.compliant,
|
||||
non_compliant: v.non_compliant,
|
||||
compliance_pct: pct,
|
||||
blockers: burndown.blockers,
|
||||
@@ -518,9 +535,9 @@ function createVCLMultiVerticalRouter(upload) {
|
||||
|
||||
res.json({
|
||||
stats: {
|
||||
total_devices,
|
||||
compliant,
|
||||
non_compliant,
|
||||
total_devices: aggTotal,
|
||||
compliant: aggCompliant,
|
||||
non_compliant: aggNonCompliant,
|
||||
compliance_pct,
|
||||
target_pct: VCL_TARGET_PCT,
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user