Add VCL compliance reporting: exec report page, device metadata fields, bulk upload
This commit is contained in:
Jordan Ramos
@@ -9,6 +9,7 @@ const { spawn } = require('child_process');
|
||||
const pool = require('../db');
|
||||
const { requireAuth, requireGroup } = require('../middleware/auth');
|
||||
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
|
||||
const { isValidDateString, validateRemediationPlan, computeVCLStats, categorizeNonCompliant, rankHeavyHitters, computeForecastBurndown, matchByHostname, computeBulkDiff, mapColumnHeaders } = require('../helpers/vclHelpers');
|
||||
const logAudit = require('../helpers/auditLog');
|
||||
|
||||
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
|
||||
@@ -147,6 +148,39 @@ async function persistUpload({ items, summary, reportDate, filename, userId }) {
|
||||
);
|
||||
|
||||
await client.query('COMMIT');
|
||||
|
||||
// Task 7: Create/update compliance_snapshots for the current month
|
||||
try {
|
||||
const currentMonth = new Date().toISOString().slice(0, 7); // YYYY-MM
|
||||
|
||||
// Compute per-vertical compliance percentages from current state
|
||||
const { rows: verticalStats } = await pool.query(
|
||||
`SELECT team AS vertical,
|
||||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||||
COUNT(DISTINCT CASE WHEN status = 'resolved' THEN hostname END)::int AS compliant,
|
||||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||||
FROM compliance_items
|
||||
WHERE team IS NOT NULL
|
||||
GROUP BY team`
|
||||
);
|
||||
|
||||
for (const vs of verticalStats) {
|
||||
const total = vs.total_devices;
|
||||
const compPct = total > 0 ? Math.round((vs.compliant / total) * 100 * 100) / 100 : 0;
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (snapshot_month, vertical)
|
||||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||||
[currentMonth, vs.vertical, total, vs.compliant, vs.non_compliant, compPct]
|
||||
);
|
||||
}
|
||||
} catch (snapshotErr) {
|
||||
// Snapshot creation is non-critical — log but don't fail the upload
|
||||
console.error('[Compliance] Snapshot creation error:', snapshotErr.message);
|
||||
}
|
||||
|
||||
return { uploadId, newCount, recurringCount, resolvedCount };
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
@@ -669,6 +703,425 @@ function createComplianceRouter(upload) {
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// PATCH /items/:hostname/metadata — Update resolution_date / remediation_plan
|
||||
// -----------------------------------------------------------------------
|
||||
router.patch('/items/:hostname/metadata', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const hostname = req.params.hostname;
|
||||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||||
|
||||
const { resolution_date, remediation_plan } = req.body;
|
||||
|
||||
// Validate resolution_date: must be a valid ISO date string or null
|
||||
if (resolution_date !== undefined && resolution_date !== null) {
|
||||
if (!isValidDateString(resolution_date)) {
|
||||
return res.status(400).json({ error: 'Invalid resolution_date format' });
|
||||
}
|
||||
}
|
||||
|
||||
// Validate remediation_plan: must be <= 2000 chars or null
|
||||
if (remediation_plan !== undefined && remediation_plan !== null) {
|
||||
const planValidation = validateRemediationPlan(remediation_plan);
|
||||
if (!planValidation.valid) {
|
||||
return res.status(400).json({ error: planValidation.error });
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Build dynamic SET clause for provided fields only
|
||||
const setClauses = [];
|
||||
const values = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (resolution_date !== undefined) {
|
||||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||||
values.push(resolution_date);
|
||||
}
|
||||
if (remediation_plan !== undefined) {
|
||||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||||
values.push(remediation_plan);
|
||||
}
|
||||
|
||||
if (setClauses.length === 0) {
|
||||
return res.status(400).json({ error: 'No fields to update' });
|
||||
}
|
||||
|
||||
values.push(hostname);
|
||||
const result = await pool.query(
|
||||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||||
values
|
||||
);
|
||||
|
||||
if (result.rowCount === 0) {
|
||||
return res.status(404).json({ error: 'Device not found' });
|
||||
}
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id,
|
||||
username: req.user.username,
|
||||
action: 'compliance_metadata_update',
|
||||
entityType: 'compliance_item',
|
||||
entityId: hostname,
|
||||
details: { resolution_date, remediation_plan },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.json({ updated: result.rowCount });
|
||||
} catch (err) {
|
||||
console.error('[Compliance] PATCH /items/:hostname/metadata error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to update device metadata' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// GET /vcl/stats — VCL executive summary statistics
|
||||
// -----------------------------------------------------------------------
|
||||
const VCL_TARGET_PCT = parseInt(process.env.VCL_TARGET_PCT, 10) || 95;
|
||||
|
||||
router.get('/vcl/stats', async (req, res) => {
|
||||
try {
|
||||
// Fetch all active compliance items
|
||||
const { rows: items } = await pool.query(
|
||||
`SELECT hostname, team, status, resolution_date, remediation_plan,
|
||||
CASE WHEN status = 'resolved' THEN true ELSE false END AS is_compliant,
|
||||
true AS in_scope
|
||||
FROM compliance_items WHERE status = 'active'`
|
||||
);
|
||||
|
||||
// For stats computation, all active items are non-compliant (they are findings)
|
||||
// We need total in-scope devices (active + resolved from latest upload)
|
||||
const { rows: latestUploadRows } = await pool.query(
|
||||
`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
|
||||
);
|
||||
|
||||
let allDeviceItems = [];
|
||||
if (latestUploadRows.length > 0) {
|
||||
const { rows: allItems } = await pool.query(
|
||||
`SELECT hostname, team, status, resolution_date, remediation_plan,
|
||||
CASE WHEN status = 'resolved' THEN true ELSE false END AS is_compliant,
|
||||
true AS in_scope
|
||||
FROM compliance_items`
|
||||
);
|
||||
// Deduplicate by hostname — a device is compliant if it has no active findings
|
||||
const deviceMap = new Map();
|
||||
for (const item of allItems) {
|
||||
const existing = deviceMap.get(item.hostname);
|
||||
if (!existing) {
|
||||
deviceMap.set(item.hostname, { ...item, is_compliant: item.status !== 'active', in_scope: true });
|
||||
} else if (item.status === 'active') {
|
||||
existing.is_compliant = false;
|
||||
}
|
||||
}
|
||||
allDeviceItems = Array.from(deviceMap.values());
|
||||
}
|
||||
|
||||
const stats = computeVCLStats(allDeviceItems, VCL_TARGET_PCT);
|
||||
|
||||
// Donut: categorize non-compliant items by resolution_date presence
|
||||
const nonCompliantItems = items.filter(i => i.status === 'active');
|
||||
const donut = categorizeNonCompliant(nonCompliantItems);
|
||||
|
||||
// Heavy hitters: group by team, count non-compliant per team
|
||||
const teamCounts = {};
|
||||
for (const item of nonCompliantItems) {
|
||||
const team = item.team || 'Unknown';
|
||||
if (!teamCounts[team]) {
|
||||
teamCounts[team] = { vertical: team, team: team, non_compliant: 0, compliance_date: null, notes: '' };
|
||||
}
|
||||
teamCounts[team].non_compliant++;
|
||||
// Use the latest resolution_date as the team's compliance_date
|
||||
if (item.resolution_date && (!teamCounts[team].compliance_date || item.resolution_date > teamCounts[team].compliance_date)) {
|
||||
teamCounts[team].compliance_date = item.resolution_date;
|
||||
}
|
||||
}
|
||||
const heavy_hitters = rankHeavyHitters(Object.values(teamCounts));
|
||||
|
||||
// Vertical breakdown with burndown
|
||||
const verticalBreakdown = [];
|
||||
for (const team of Object.keys(teamCounts)) {
|
||||
const teamItems = nonCompliantItems.filter(i => (i.team || 'Unknown') === team);
|
||||
const teamAllDevices = allDeviceItems.filter(i => (i.team || 'Unknown') === team);
|
||||
const teamTotal = teamAllDevices.length;
|
||||
const teamCompliant = teamAllDevices.filter(i => i.is_compliant).length;
|
||||
const compliance_pct = teamTotal > 0 ? Math.round((teamCompliant / teamTotal) * 100) : 0;
|
||||
|
||||
const actual_burndown = computeForecastBurndown(teamItems.filter(i => i.resolution_date));
|
||||
const forecast_burndown = computeForecastBurndown(teamItems.filter(i => i.resolution_date));
|
||||
const blockers = teamItems.filter(i => !i.resolution_date).length;
|
||||
|
||||
verticalBreakdown.push({
|
||||
vertical: team,
|
||||
compliance_pct,
|
||||
team: team,
|
||||
non_compliant: teamItems.length,
|
||||
actual_burndown,
|
||||
forecast_burndown,
|
||||
blockers,
|
||||
risk_acceptances: 0,
|
||||
notes: '',
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ stats, donut, heavy_hitters, vertical_breakdown: verticalBreakdown });
|
||||
} catch (err) {
|
||||
console.error('[Compliance] GET /vcl/stats error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// GET /vcl/trend — Monthly compliance trend with forecast
|
||||
// -----------------------------------------------------------------------
|
||||
router.get('/vcl/trend', async (req, res) => {
|
||||
try {
|
||||
const { rows: snapshots } = await pool.query(
|
||||
`SELECT snapshot_month, SUM(compliant)::int AS compliant_count,
|
||||
CASE WHEN SUM(total_devices) > 0
|
||||
THEN ROUND((SUM(compliant)::numeric / SUM(total_devices)::numeric) * 100, 1)
|
||||
ELSE 0 END AS compliance_pct
|
||||
FROM compliance_snapshots
|
||||
GROUP BY snapshot_month
|
||||
ORDER BY snapshot_month ASC`
|
||||
);
|
||||
|
||||
// Build months array with actuals
|
||||
const months = snapshots.map(s => ({
|
||||
month: s.snapshot_month,
|
||||
compliant_count: s.compliant_count,
|
||||
compliance_pct: parseFloat(s.compliance_pct),
|
||||
forecast_pct: null,
|
||||
target_pct: VCL_TARGET_PCT,
|
||||
}));
|
||||
|
||||
// Compute forecast using linear regression if we have 3+ months
|
||||
if (months.length >= 3) {
|
||||
const n = months.length;
|
||||
// Use last data points for regression
|
||||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
sumX += i;
|
||||
sumY += months[i].compliance_pct;
|
||||
sumXY += i * months[i].compliance_pct;
|
||||
sumX2 += i * i;
|
||||
}
|
||||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
|
||||
const intercept = (sumY - slope * sumX) / n;
|
||||
|
||||
// Project forward 3 months
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const futureIdx = n + i;
|
||||
const forecastPct = Math.min(100, Math.max(0, Math.round((slope * futureIdx + intercept) * 10) / 10));
|
||||
|
||||
// Compute the future month string
|
||||
const lastMonth = months[months.length - 1].month;
|
||||
const [year, mon] = lastMonth.split('-').map(Number);
|
||||
const futureDate = new Date(year, mon - 1 + i + 1, 1);
|
||||
const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`;
|
||||
|
||||
months.push({
|
||||
month: futureMonth,
|
||||
compliant_count: null,
|
||||
compliance_pct: null,
|
||||
forecast_pct: forecastPct,
|
||||
target_pct: VCL_TARGET_PCT,
|
||||
});
|
||||
}
|
||||
|
||||
// Also add forecast_pct to the last actual month as the starting point
|
||||
if (months.length > 0 && n > 0) {
|
||||
months[n - 1].forecast_pct = months[n - 1].compliance_pct;
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ months });
|
||||
} catch (err) {
|
||||
console.error('[Compliance] GET /vcl/trend error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// POST /vcl/bulk-preview — Bulk upload diff preview
|
||||
// -----------------------------------------------------------------------
|
||||
router.post('/vcl/bulk-preview', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { rows, headers } = req.body;
|
||||
|
||||
// Validate: require rows array
|
||||
if (!rows || !Array.isArray(rows)) {
|
||||
return res.status(400).json({ error: 'rows array is required' });
|
||||
}
|
||||
|
||||
// Enforce 2000 row limit
|
||||
if (rows.length === 0) {
|
||||
return res.status(400).json({ error: 'File contains no data rows' });
|
||||
}
|
||||
if (rows.length > 2000) {
|
||||
return res.status(400).json({ error: 'File exceeds maximum of 2000 rows' });
|
||||
}
|
||||
|
||||
// Map column headers if provided
|
||||
let columnMapping = {};
|
||||
if (headers && Array.isArray(headers)) {
|
||||
columnMapping = mapColumnHeaders(headers);
|
||||
}
|
||||
|
||||
// Require hostname field
|
||||
const hasHostname = rows.every(r => r.hostname != null && r.hostname !== '');
|
||||
if (!hasHostname) {
|
||||
return res.status(400).json({ error: 'File must contain a Hostname column' });
|
||||
}
|
||||
|
||||
// Check for updatable fields (resolution_date, remediation_plan, or notes)
|
||||
const sampleRow = rows[0] || {};
|
||||
const updatableFields = ['resolution_date', 'remediation_plan', 'notes'];
|
||||
const hasUpdatableFields = updatableFields.some(f => f in sampleRow);
|
||||
if (!hasUpdatableFields && headers) {
|
||||
// Check via column mapping
|
||||
const mappedFields = Object.keys(columnMapping).filter(k => k !== 'hostname');
|
||||
if (mappedFields.length === 0) {
|
||||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||||
}
|
||||
} else if (!hasUpdatableFields && !headers) {
|
||||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||||
}
|
||||
|
||||
try {
|
||||
// Get existing hostnames from DB
|
||||
const { rows: existingRows } = await pool.query(
|
||||
`SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active'`
|
||||
);
|
||||
const existingHostnames = new Set(existingRows.map(r => r.hostname));
|
||||
|
||||
// Match by hostname
|
||||
const { matched, unmatched } = matchByHostname(rows, existingHostnames);
|
||||
|
||||
// Validate fields on matched rows
|
||||
const validRows = [];
|
||||
const invalidRows = [];
|
||||
|
||||
for (const row of matched) {
|
||||
const errors = [];
|
||||
|
||||
if (row.resolution_date !== undefined && row.resolution_date !== null && row.resolution_date !== '') {
|
||||
if (!isValidDateString(row.resolution_date)) {
|
||||
errors.push('resolution_date: invalid date format');
|
||||
}
|
||||
}
|
||||
|
||||
if (row.remediation_plan !== undefined && row.remediation_plan !== null) {
|
||||
const planCheck = validateRemediationPlan(row.remediation_plan);
|
||||
if (!planCheck.valid) {
|
||||
errors.push('remediation_plan: ' + planCheck.error);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
invalidRows.push({ hostname: row.hostname, errors });
|
||||
} else {
|
||||
validRows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
// Get current data for diff computation
|
||||
const { rows: currentRows } = await pool.query(
|
||||
`SELECT DISTINCT ON (hostname) hostname, resolution_date, remediation_plan
|
||||
FROM compliance_items WHERE status = 'active' AND hostname = ANY($1)
|
||||
ORDER BY hostname, id DESC`,
|
||||
[validRows.map(r => r.hostname)]
|
||||
);
|
||||
const currentData = new Map();
|
||||
for (const row of currentRows) {
|
||||
currentData.set(row.hostname, {
|
||||
resolution_date: row.resolution_date ? row.resolution_date.toISOString?.().slice(0, 10) || String(row.resolution_date).slice(0, 10) : null,
|
||||
remediation_plan: row.remediation_plan || null,
|
||||
notes: null,
|
||||
});
|
||||
}
|
||||
|
||||
// Compute diff
|
||||
const diffResults = computeBulkDiff(validRows, currentData);
|
||||
const changedRows = diffResults.filter(r => r.status === 'changed');
|
||||
|
||||
res.json({
|
||||
matched: matched.length,
|
||||
unmatched: unmatched.length,
|
||||
changes: changedRows.length,
|
||||
invalid: invalidRows.length,
|
||||
details: diffResults,
|
||||
unmatched_rows: unmatched.map(r => r.hostname),
|
||||
invalid_rows: invalidRows,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Compliance] POST /vcl/bulk-preview error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to process bulk preview' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// POST /vcl/bulk-commit — Commit validated bulk changes
|
||||
// -----------------------------------------------------------------------
|
||||
router.post('/vcl/bulk-commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { changes } = req.body;
|
||||
|
||||
if (!changes || !Array.isArray(changes) || changes.length === 0) {
|
||||
return res.status(400).json({ error: 'changes array is required' });
|
||||
}
|
||||
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
let committedCount = 0;
|
||||
for (const change of changes) {
|
||||
const setClauses = [];
|
||||
const values = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (change.resolution_date !== undefined) {
|
||||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||||
values.push(change.resolution_date);
|
||||
}
|
||||
if (change.remediation_plan !== undefined) {
|
||||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||||
values.push(change.remediation_plan);
|
||||
}
|
||||
if (change.notes !== undefined) {
|
||||
// Notes are stored separately in compliance_notes, but we can update a field if it exists
|
||||
// For now, skip notes in the direct update
|
||||
}
|
||||
|
||||
if (setClauses.length === 0) continue;
|
||||
|
||||
values.push(change.hostname);
|
||||
const result = await client.query(
|
||||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||||
values
|
||||
);
|
||||
if (result.rowCount > 0) committedCount++;
|
||||
}
|
||||
|
||||
await client.query('COMMIT');
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id,
|
||||
username: req.user.username,
|
||||
action: 'compliance_bulk_update',
|
||||
entityType: 'compliance_items',
|
||||
entityId: null,
|
||||
details: { rows_updated: committedCount, total_changes: changes.length },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.json({ committed: committedCount });
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
console.error('[Compliance] POST /vcl/bulk-commit error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to commit changes' });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user