97e5d68d8e
The /summary endpoint was fetching the most recent upload regardless of vertical, which on dev was a PRDCT_VSO multi-vertical upload. Now it looks for AEO uploads (vertical IS NULL) first, then falls back to the NTS_AEO multi-vertical upload. The /items endpoint now includes items from both vertical IS NULL and vertical = 'NTS_AEO' so the AEO compliance page shows devices uploaded through either flow.
1420 lines
68 KiB
JavaScript
1420 lines
68 KiB
JavaScript
// Compliance Routes — AEO metric tracking
|
||
// Handles xlsx upload/parse, non-compliant item history, and notes.
|
||
|
||
const express = require('express');
|
||
const path = require('path');
|
||
const fs = require('fs');
|
||
const crypto = require('crypto');
|
||
const { spawn } = require('child_process');
|
||
const pool = require('../db');
|
||
const { requireAuth, requireGroup } = require('../middleware/auth');
|
||
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
|
||
const { isValidDateString, validateRemediationPlan, computeVCLStats, categorizeNonCompliant, rankHeavyHitters, computeForecastBurndown, matchByHostname, computeBulkDiff, mapColumnHeaders } = require('../helpers/vclHelpers');
|
||
const logAudit = require('../helpers/auditLog');
|
||
|
||
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
|
||
const SCHEMA_SCRIPT = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
|
||
const CONFIG_PATH = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
|
||
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
|
||
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
|
||
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Run Python parser, return parsed object
|
||
// ---------------------------------------------------------------------------
|
||
function parseXlsx(filePath) {
|
||
return new Promise((resolve, reject) => {
|
||
const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
|
||
let out = '';
|
||
let err = '';
|
||
py.stdout.on('data', d => { out += d; });
|
||
py.stderr.on('data', d => { err += d; });
|
||
py.on('close', code => {
|
||
if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
|
||
try { resolve(JSON.parse(out)); }
|
||
catch (e) { reject(new Error('Parser returned invalid JSON')); }
|
||
});
|
||
py.on('error', reject);
|
||
});
|
||
}
|
||
|
||
function extractXlsxSchema(filePath) {
|
||
return new Promise((resolve, reject) => {
|
||
const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
|
||
let out = '';
|
||
let err = '';
|
||
py.stdout.on('data', d => { out += d; });
|
||
py.stderr.on('data', d => { err += d; });
|
||
py.on('close', code => {
|
||
if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
|
||
try { resolve(JSON.parse(out)); }
|
||
catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
|
||
});
|
||
py.on('error', reject);
|
||
});
|
||
}
|
||
|
||
function isSafeTempPath(filePath) {
|
||
const resolved = path.resolve(filePath);
|
||
return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Compute diff: new / recurring / resolved
|
||
// ---------------------------------------------------------------------------
|
||
async function computeDiff(incomingItems) {
|
||
const { rows: activeRows } = await pool.query(
|
||
`SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
|
||
);
|
||
const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
|
||
const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));
|
||
|
||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
|
||
for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; }
|
||
|
||
return { newCount, recurringCount, resolvedCount };
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Write a parsed upload to the DB (within a transaction)
|
||
// ---------------------------------------------------------------------------
|
||
async function persistUpload({ items, summary, reportDate, filename, userId }) {
|
||
const { rows: activeRows } = await pool.query(
|
||
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
|
||
);
|
||
const activeMap = {};
|
||
activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });
|
||
|
||
const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
// 1. Insert the upload record
|
||
const uploadResult = await client.query(
|
||
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
|
||
VALUES ($1, $2, $3, NOW(), $4)
|
||
RETURNING id`,
|
||
[filename, reportDate || null, userId || null, JSON.stringify(summary)]
|
||
);
|
||
const uploadId = uploadResult.rows[0].id;
|
||
|
||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||
|
||
// 2. Upsert each incoming non-compliant item
|
||
for (const item of items) {
|
||
const key = `${item.hostname}|||${item.metric_id}`;
|
||
const existing = activeMap[key];
|
||
const extraStr = JSON.stringify(item.extra_json || {});
|
||
|
||
if (existing) {
|
||
await client.query(
|
||
`UPDATE compliance_items
|
||
SET upload_id = $1, seen_count = $2, ip_address = $3, device_type = $4, extra_json = $5
|
||
WHERE id = $6`,
|
||
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
|
||
);
|
||
recurringCount++;
|
||
} else {
|
||
await client.query(
|
||
`INSERT INTO compliance_items
|
||
(upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
|
||
category, extra_json, status, first_seen_upload_id, seen_count)
|
||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, 1)`,
|
||
[uploadId, item.hostname, item.ip_address, item.device_type, item.team,
|
||
item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
|
||
);
|
||
newCount++;
|
||
}
|
||
}
|
||
|
||
// 3. Mark items not present in this upload as resolved
|
||
for (const [key, row] of Object.entries(activeMap)) {
|
||
if (!newKeys.has(key)) {
|
||
await client.query(
|
||
`UPDATE compliance_items SET status = 'resolved', resolved_upload_id = $1 WHERE id = $2`,
|
||
[uploadId, row.id]
|
||
);
|
||
resolvedCount++;
|
||
}
|
||
}
|
||
|
||
// 4. Update upload with final counts
|
||
await client.query(
|
||
`UPDATE compliance_uploads SET new_count = $1, resolved_count = $2, recurring_count = $3 WHERE id = $4`,
|
||
[newCount, resolvedCount, recurringCount, uploadId]
|
||
);
|
||
|
||
await client.query('COMMIT');
|
||
|
||
// Task 7: Create/update compliance_snapshots for the current month
|
||
try {
|
||
const currentMonth = new Date().toISOString().slice(0, 7); // YYYY-MM
|
||
|
||
// Compute per-vertical compliance percentages from current state
|
||
const { rows: verticalStats } = await pool.query(
|
||
`SELECT team AS vertical,
|
||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||
COUNT(DISTINCT CASE WHEN status = 'resolved' THEN hostname END)::int AS compliant,
|
||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||
FROM compliance_items
|
||
WHERE team IS NOT NULL
|
||
GROUP BY team`
|
||
);
|
||
|
||
for (const vs of verticalStats) {
|
||
const total = vs.total_devices;
|
||
const compPct = total > 0 ? Math.round((vs.compliant / total) * 100 * 100) / 100 : 0;
|
||
|
||
await pool.query(
|
||
`INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||
VALUES ($1, $2, $3, $4, $5, $6)
|
||
ON CONFLICT (snapshot_month, vertical)
|
||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||
[currentMonth, vs.vertical, total, vs.compliant, vs.non_compliant, compPct]
|
||
);
|
||
}
|
||
} catch (snapshotErr) {
|
||
// Snapshot creation is non-critical — log but don't fail the upload
|
||
console.error('[Compliance] Snapshot creation error:', snapshotErr.message);
|
||
}
|
||
|
||
return { uploadId, newCount, recurringCount, resolvedCount };
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
throw err;
|
||
} finally {
|
||
client.release();
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Group flat compliance_items rows into per-device objects
|
||
// ---------------------------------------------------------------------------
|
||
function groupByHostname(rows, noteHostnames) {
|
||
const deviceMap = {};
|
||
for (const row of rows) {
|
||
if (!deviceMap[row.hostname]) {
|
||
deviceMap[row.hostname] = {
|
||
hostname: row.hostname, ip_address: row.ip_address || '', device_type: row.device_type || '',
|
||
team: row.team || '', status: row.status, failing_metrics: [],
|
||
seen_count: row.seen_count || 1, first_seen: row.first_seen || null,
|
||
last_seen: row.last_seen || null, resolved_on: row.resolved_on || null,
|
||
has_notes: noteHostnames.has(row.hostname),
|
||
};
|
||
}
|
||
const dev = deviceMap[row.hostname];
|
||
dev.failing_metrics.push({ metric_id: row.metric_id, metric_desc: row.metric_desc || '', category: row.category || '' });
|
||
if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count;
|
||
if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen)) dev.first_seen = row.first_seen;
|
||
if (row.last_seen && (!dev.last_seen || row.last_seen > dev.last_seen)) dev.last_seen = row.last_seen;
|
||
}
|
||
return Object.values(deviceMap);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Pure helpers
|
||
// ---------------------------------------------------------------------------
|
||
const BUCKET_ORDER = ['1 cycle', '2–3 cycles', '4–6 cycles', '7+ cycles'];
|
||
|
||
function bucketAgingItems(items) {
|
||
const teams = ['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV'];
|
||
const buckets = {};
|
||
for (const b of BUCKET_ORDER) {
|
||
buckets[b] = { bucket: b, total: 0 };
|
||
for (const t of teams) buckets[b][t] = 0;
|
||
}
|
||
for (const item of items) {
|
||
const sc = item.seen_count;
|
||
let label;
|
||
if (sc === 1) label = '1 cycle';
|
||
else if (sc >= 2 && sc <= 3) label = '2–3 cycles';
|
||
else if (sc >= 4 && sc <= 6) label = '4–6 cycles';
|
||
else label = '7+ cycles';
|
||
buckets[label].total += 1;
|
||
if (item.team in buckets[label]) buckets[label][item.team] += 1;
|
||
}
|
||
return BUCKET_ORDER.map(b => buckets[b]);
|
||
}
|
||
|
||
function computeWaterfall(uploads) {
|
||
let start = 0;
|
||
return uploads.map((row) => {
|
||
const end = start + row.new_count + row.recurring_count - row.resolved_count;
|
||
const entry = { date: row.report_date, start, new_count: row.new_count, recurring_count: row.recurring_count, resolved_count: row.resolved_count, end };
|
||
start = end;
|
||
return entry;
|
||
});
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Router factory
|
||
// ---------------------------------------------------------------------------
|
||
function createComplianceRouter(upload) {
|
||
const router = express.Router();
|
||
|
||
// All compliance routes require authentication
|
||
router.use(requireAuth());
|
||
|
||
/**
|
||
* POST /preview
|
||
* Uploads an xlsx file, parses it, computes a diff against current active items,
|
||
* performs schema drift detection, and stores parsed data in a temp file for later commit.
|
||
*
|
||
* @body multipart/form-data — field "file" (xlsx spreadsheet, max 10MB)
|
||
* @response 200 { drift, drift_error, schema, diff: { new_count, recurring_count, resolved_count }, tempFile, filename, report_date, total_items }
|
||
* @response 400 { error } — no file, wrong extension, or upload error
|
||
* @response 422 { error } — parser returned an error
|
||
* @response 500 { error } — config load failure or parse failure
|
||
*/
|
||
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
|
||
upload.single('file')(req, res, async (uploadErr) => {
|
||
if (uploadErr) return res.status(400).json({ error: uploadErr.message });
|
||
if (!req.file) return res.status(400).json({ error: 'No file uploaded' });
|
||
if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
|
||
fs.unlink(req.file.path, () => {});
|
||
return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
|
||
}
|
||
|
||
try {
|
||
let drift = null, drift_error = null;
|
||
let config;
|
||
try { config = loadConfig(CONFIG_PATH); } catch (configErr) {
|
||
fs.unlink(req.file.path, () => {});
|
||
return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
|
||
}
|
||
|
||
let xlsxSchema = null;
|
||
try {
|
||
xlsxSchema = await extractXlsxSchema(req.file.path);
|
||
if (xlsxSchema.error) throw new Error(xlsxSchema.error);
|
||
drift = compareSchemaToDrift(xlsxSchema, config);
|
||
} catch (driftErr) {
|
||
drift = null;
|
||
drift_error = driftErr.message || 'Drift check failed';
|
||
}
|
||
|
||
const parsed = await parseXlsx(req.file.path);
|
||
if (parsed.error) {
|
||
fs.unlink(req.file.path, () => {});
|
||
return res.status(422).json({ error: parsed.error });
|
||
}
|
||
|
||
const diff = await computeDiff(parsed.items);
|
||
|
||
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||
const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
|
||
const tempFilePath = path.join(TEMP_DIR, tempFilename);
|
||
|
||
fs.writeFileSync(tempFilePath, JSON.stringify({
|
||
items: parsed.items, summary: parsed.summary,
|
||
report_date: parsed.report_date,
|
||
filename: req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
|
||
}));
|
||
|
||
fs.unlink(req.file.path, () => {});
|
||
|
||
res.json({
|
||
drift, drift_error, schema: xlsxSchema,
|
||
diff: { new_count: diff.newCount, recurring_count: diff.recurringCount, resolved_count: diff.resolvedCount },
|
||
tempFile: tempFilePath, filename: req.file.originalname,
|
||
report_date: parsed.report_date, total_items: parsed.total,
|
||
});
|
||
} catch (err) {
|
||
fs.unlink(req.file.path, () => {});
|
||
console.error('[Compliance] Preview error:', err.message);
|
||
res.status(500).json({ error: 'Failed to parse file: ' + err.message });
|
||
}
|
||
});
|
||
});
|
||
|
||
/**
|
||
* POST /reconcile-config
|
||
* Applies schema drift reconciliation to the compliance config file based on detected drift findings.
|
||
*
|
||
* @body { drift: object, schema?: object }
|
||
* @response 200 { changes: Array<{ action, key, value, detail }>, message }
|
||
* @response 400 { error } — missing drift or no findings to reconcile
|
||
* @response 500 { error } — reconciliation failure
|
||
*/
|
||
router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
|
||
const { drift, schema } = req.body;
|
||
if (!drift || typeof drift !== 'object') return res.status(400).json({ error: 'drift report is required in request body' });
|
||
const hasFindings = (drift.breaking && drift.breaking.length > 0) || (drift.silent_miss && drift.silent_miss.length > 0);
|
||
if (!hasFindings) return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });
|
||
|
||
try {
|
||
const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);
|
||
if (changes.length === 0) return res.json({ changes: [], message: 'No changes needed' });
|
||
|
||
for (const change of changes) {
|
||
logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_config_reconcile', entityType: 'compliance_config', entityId: change.value, details: { action: change.action, key: change.key, detail: change.detail }, ipAddress: req.ip });
|
||
}
|
||
res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });
|
||
} catch (err) {
|
||
console.error('[Compliance] Reconcile config error:', err.message);
|
||
res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /commit
|
||
* Commits a previously previewed compliance upload to the database. Resolves items no longer
|
||
* present, upserts recurring/new items, and creates a compliance snapshot for the current month.
|
||
*
|
||
* @body { tempFile: string, filename?: string, report_date?: string }
|
||
* @response 200 { upload: { id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count } }
|
||
* @response 400 { error } — missing/invalid tempFile or expired preview session
|
||
* @response 500 { error } — commit failure
|
||
*/
|
||
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { tempFile, filename, report_date } = req.body;
|
||
if (!tempFile || typeof tempFile !== 'string') return res.status(400).json({ error: 'tempFile is required' });
|
||
if (!isSafeTempPath(tempFile)) return res.status(400).json({ error: 'Invalid tempFile path' });
|
||
if (!fs.existsSync(tempFile)) return res.status(400).json({ error: 'Preview session expired — please upload again' });
|
||
|
||
let parsed;
|
||
try { parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8')); }
|
||
catch { return res.status(400).json({ error: 'Could not read preview data — please upload again' }); }
|
||
|
||
try {
|
||
const result = await persistUpload({
|
||
items: parsed.items, summary: parsed.summary,
|
||
reportDate: report_date || parsed.report_date,
|
||
filename: filename || parsed.filename,
|
||
userId: req.user?.id || null,
|
||
});
|
||
fs.unlink(tempFile, () => {});
|
||
|
||
const { rows } = await pool.query(
|
||
`SELECT id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count
|
||
FROM compliance_uploads WHERE id = $1`, [result.uploadId]
|
||
);
|
||
res.json({ upload: rows[0] });
|
||
} catch (err) {
|
||
console.error('[Compliance] Commit error:', err.message);
|
||
res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /uploads
|
||
* Returns all compliance upload records ordered by most recent first.
|
||
*
|
||
* @response 200 { uploads: Array<{ id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/uploads', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count
|
||
FROM compliance_uploads ORDER BY id DESC`
|
||
);
|
||
res.json({ uploads: rows });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /uploads error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /rollback/:uploadId
|
||
* Rolls back the most recent compliance upload — deletes new items introduced by that upload,
|
||
* reactivates items it resolved, and removes the upload record.
|
||
*
|
||
* @param uploadId — numeric ID of the upload to roll back (must be the latest)
|
||
* @response 200 { message, rolled_back: { upload_id, filename, report_date, items_deleted, items_reactivated } }
|
||
* @response 400 { error } — invalid ID or not the latest upload
|
||
* @response 404 { error } — upload not found
|
||
* @response 500 { error } — rollback failure
|
||
*/
|
||
router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
|
||
const uploadId = parseInt(req.params.uploadId, 10);
|
||
if (isNaN(uploadId)) return res.status(400).json({ error: 'Invalid upload ID' });
|
||
|
||
try {
|
||
const { rows: uploadRows } = await pool.query(
|
||
`SELECT id, filename, report_date, new_count, resolved_count, recurring_count FROM compliance_uploads WHERE id = $1`, [uploadId]
|
||
);
|
||
const upload = uploadRows[0];
|
||
if (!upload) return res.status(404).json({ error: 'Upload not found' });
|
||
|
||
const { rows: latestRows } = await pool.query(`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`);
|
||
if (latestRows[0].id !== uploadId) {
|
||
return res.status(400).json({ error: 'Only the most recent upload can be rolled back', latest_upload_id: latestRows[0].id });
|
||
}
|
||
|
||
const { rows: prevRows } = await pool.query(`SELECT id FROM compliance_uploads WHERE id < $1 ORDER BY id DESC LIMIT 1`, [uploadId]);
|
||
const previousUpload = prevRows[0];
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
const deleteNew = await client.query(
|
||
`DELETE FROM compliance_items WHERE first_seen_upload_id = $1 AND upload_id = $1`, [uploadId]
|
||
);
|
||
const reactivate = await client.query(
|
||
`UPDATE compliance_items SET status = 'active', resolved_upload_id = NULL WHERE resolved_upload_id = $1`, [uploadId]
|
||
);
|
||
if (previousUpload) {
|
||
await client.query(
|
||
`UPDATE compliance_items SET upload_id = $1, seen_count = GREATEST(seen_count - 1, 1) WHERE upload_id = $2 AND first_seen_upload_id != $2`,
|
||
[previousUpload.id, uploadId]
|
||
);
|
||
}
|
||
await client.query(`DELETE FROM compliance_uploads WHERE id = $1`, [uploadId]);
|
||
await client.query('COMMIT');
|
||
|
||
logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_upload_rollback', entityType: 'compliance_upload', entityId: String(uploadId), details: { filename: upload.filename, report_date: upload.report_date, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount }, ipAddress: req.ip });
|
||
|
||
res.json({ message: `Rolled back upload "${upload.filename}"`, rolled_back: { upload_id: uploadId, filename: upload.filename, report_date: upload.report_date, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount } });
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
throw err;
|
||
} finally {
|
||
client.release();
|
||
}
|
||
} catch (err) {
|
||
console.error('[Compliance] Rollback error:', err.message);
|
||
res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /summary
|
||
* Returns the summary data from the most recent compliance upload, optionally filtered by team.
|
||
*
|
||
* @query team — optional, one of STEAM | ACCESS-ENG | ACCESS-OPS | INTELDEV
|
||
* @response 200 { entries: Array, overall_scores: object, upload: { id, report_date, uploaded_at } | null }
|
||
* @response 400 { error } — invalid team
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/summary', async (req, res) => {
|
||
const team = req.query.team;
|
||
if (team && !ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
||
|
||
try {
|
||
// Try AEO uploads first (vertical IS NULL), fall back to NTS_AEO multi-vertical upload
|
||
let { rows: latestRows } = await pool.query(
|
||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical IS NULL ORDER BY id DESC LIMIT 1`
|
||
);
|
||
if (latestRows.length === 0 || !latestRows[0].summary_json) {
|
||
({ rows: latestRows } = await pool.query(
|
||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical = 'NTS_AEO' ORDER BY id DESC LIMIT 1`
|
||
));
|
||
}
|
||
const latestUpload = latestRows[0];
|
||
if (!latestUpload || !latestUpload.summary_json) return res.json({ entries: [], overall_scores: {}, upload: null });
|
||
|
||
let summary;
|
||
try { summary = JSON.parse(latestUpload.summary_json); } catch { return res.json({ entries: [], overall_scores: {}, upload: null }); }
|
||
|
||
let entries = summary.entries || [];
|
||
if (team) entries = entries.filter(e => e.team === team);
|
||
|
||
res.json({ entries, overall_scores: summary.overall_scores || {}, upload: { id: latestUpload.id, report_date: latestUpload.report_date, uploaded_at: latestUpload.uploaded_at } });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /summary error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /items
|
||
* Returns compliance items grouped by hostname for a given team and status.
|
||
*
|
||
* @query team — required, one of STEAM | ACCESS-ENG | ACCESS-OPS | INTELDEV
|
||
* @query status — optional, "active" (default) or "resolved"
|
||
* @response 200 { devices: Array<{ hostname, ip_address, device_type, team, status, failing_metrics, seen_count, first_seen, last_seen, resolved_on, has_notes }>, team, status }
|
||
* @response 400 { error } — missing/invalid team or invalid status
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/items', async (req, res) => {
|
||
const { team, status = 'active' } = req.query;
|
||
if (!team) return res.status(400).json({ error: 'team is required' });
|
||
if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
||
if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' });
|
||
|
||
try {
|
||
// Include items from both AEO uploads (vertical IS NULL) and NTS_AEO multi-vertical uploads
|
||
const { rows } = await pool.query(
|
||
`SELECT ci.hostname, ci.ip_address, ci.device_type, ci.team, ci.metric_id, ci.metric_desc, ci.category, ci.status, ci.seen_count,
|
||
fu.report_date AS first_seen, lu.report_date AS last_seen, ru.report_date AS resolved_on
|
||
FROM compliance_items ci
|
||
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
||
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
||
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
||
WHERE ci.team = $1 AND ci.status = $2 AND (ci.vertical IS NULL OR ci.vertical = 'NTS_AEO')
|
||
ORDER BY ci.hostname, ci.metric_id`,
|
||
[team, status]
|
||
);
|
||
|
||
const { rows: noteRows } = await pool.query(`SELECT DISTINCT hostname FROM compliance_notes`);
|
||
const noteHostnames = new Set(noteRows.map(r => r.hostname));
|
||
const devices = groupByHostname(rows, noteHostnames);
|
||
res.json({ devices, team, status });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /items error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /items/:hostname
|
||
* Returns detailed information for a single device including all metrics and notes.
|
||
*
|
||
* @param hostname — the device hostname
|
||
* @response 200 { hostname, ip_address, device_type, team, metrics: Array<{ metric_id, metric_desc, category, status, ... }>, notes: Array<{ id, metric_id, note, group_id, created_at, created_by }> }
|
||
* @response 400 { error } — invalid hostname
|
||
* @response 404 { error } — device not found
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/items/:hostname', async (req, res) => {
|
||
const hostname = req.params.hostname;
|
||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||
|
||
try {
|
||
const { rows: metricRows } = await pool.query(
|
||
`SELECT ci.metric_id, ci.metric_desc, ci.category, ci.status, ci.ip_address, ci.device_type, ci.team, ci.seen_count, ci.extra_json,
|
||
ci.resolution_date, ci.remediation_plan,
|
||
fu.report_date AS first_seen, fu.uploaded_at AS first_seen_at, lu.report_date AS last_seen, lu.uploaded_at AS last_seen_at, ru.report_date AS resolved_on
|
||
FROM compliance_items ci
|
||
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
||
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
||
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
||
WHERE ci.hostname = $1
|
||
ORDER BY ci.status DESC, ci.metric_id`, [hostname]
|
||
);
|
||
if (metricRows.length === 0) return res.status(404).json({ error: 'Device not found' });
|
||
|
||
const metrics = metricRows.map(r => ({ ...r, extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(), extra_json: undefined }));
|
||
|
||
const { rows: notes } = await pool.query(
|
||
`SELECT cn.id, cn.metric_id, cn.note, cn.group_id, cn.created_at, u.username AS created_by
|
||
FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
|
||
WHERE cn.hostname = $1 ORDER BY cn.created_at DESC`, [hostname]
|
||
);
|
||
|
||
const identity = metricRows.find(r => r.status === 'active') || metricRows[0];
|
||
// Return resolution_date and remediation_plan from the first active item (or any item)
|
||
const resDate = identity.resolution_date ? (typeof identity.resolution_date === 'string' ? identity.resolution_date : identity.resolution_date.toISOString().slice(0, 10)) : null;
|
||
res.json({ hostname, ip_address: identity.ip_address || '', device_type: identity.device_type || '', team: identity.team || '', resolution_date: resDate, remediation_plan: identity.remediation_plan || '', metrics, notes });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /items/:hostname error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /notes
|
||
* Creates one or more compliance notes for a device, linked to specific metric IDs.
|
||
* All notes in a single call share a group_id for batch operations.
|
||
*
|
||
* @body { hostname: string, metric_id?: string, metric_ids?: string[], note: string }
|
||
* @response 201 { notes: Array<{ id, hostname, metric_id, note, group_id, created_at, created_by }> }
|
||
* @response 400 { error } — invalid hostname, missing/invalid metric_id(s), or empty note
|
||
* @response 500 { error } — save failure
|
||
*/
|
||
router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { hostname, metric_id, metric_ids, note } = req.body;
|
||
if (!hostname || typeof hostname !== 'string' || hostname.length > 300 || !/^[a-zA-Z0-9._-]+$/.test(hostname)) return res.status(400).json({ error: 'Invalid hostname format' });
|
||
|
||
let resolvedIds;
|
||
if (metric_ids !== undefined) {
|
||
if (!Array.isArray(metric_ids)) return res.status(400).json({ error: 'metric_ids must be an array' });
|
||
resolvedIds = metric_ids;
|
||
} else if (metric_id !== undefined && metric_id !== null && metric_id !== '') {
|
||
if (typeof metric_id !== 'string' || metric_id.length > 50) return res.status(400).json({ error: 'Invalid metric_id' });
|
||
resolvedIds = [metric_id];
|
||
} else {
|
||
return res.status(400).json({ error: 'metric_id or metric_ids is required' });
|
||
}
|
||
|
||
if (resolvedIds.length === 0) return res.status(400).json({ error: 'At least one metric ID is required' });
|
||
for (let i = 0; i < resolvedIds.length; i++) {
|
||
const mid = resolvedIds[i];
|
||
if (!mid || typeof mid !== 'string' || mid.length === 0 || mid.length > 50) return res.status(400).json({ error: `Invalid metric_id at index ${i}` });
|
||
}
|
||
|
||
const noteText = String(note || '').trim().slice(0, 1000);
|
||
if (!noteText) return res.status(400).json({ error: 'Note cannot be empty' });
|
||
|
||
const groupId = crypto.randomUUID();
|
||
const userId = req.user?.id || null;
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
const insertedIds = [];
|
||
for (const mid of resolvedIds) {
|
||
const { rows } = await client.query(
|
||
`INSERT INTO compliance_notes (hostname, metric_id, note, group_id, created_by, created_at) VALUES ($1, $2, $3, $4, $5, NOW()) RETURNING id`,
|
||
[hostname, mid, noteText, groupId, userId]
|
||
);
|
||
insertedIds.push(rows[0].id);
|
||
}
|
||
await client.query('COMMIT');
|
||
|
||
const { rows: notes } = await pool.query(
|
||
`SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.group_id, cn.created_at, u.username AS created_by
|
||
FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
|
||
WHERE cn.id = ANY($1) ORDER BY cn.id ASC`, [insertedIds]
|
||
);
|
||
res.status(201).json({ notes });
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
console.error('[Compliance] POST /notes error:', err.message);
|
||
res.status(500).json({ error: 'Failed to save note' });
|
||
} finally {
|
||
client.release();
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /notes/:hostname/:metricId
|
||
* Returns all notes for a specific device and metric combination.
|
||
*
|
||
* @param hostname — the device hostname
|
||
* @param metricId — the metric identifier
|
||
* @response 200 { notes: Array<{ id, note, created_at, created_by }> }
|
||
* @response 400 { error } — invalid hostname or metricId
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/notes/:hostname/:metricId', async (req, res) => {
|
||
const { hostname, metricId } = req.params;
|
||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||
if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metricId' });
|
||
|
||
try {
|
||
const { rows: notes } = await pool.query(
|
||
`SELECT cn.id, cn.note, cn.created_at, u.username AS created_by
|
||
FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
|
||
WHERE cn.hostname = $1 AND cn.metric_id = $2 ORDER BY cn.created_at DESC`, [hostname, metricId]
|
||
);
|
||
res.json({ notes });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /notes error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* DELETE /notes/:id
|
||
* Deletes a compliance note by ID. Only the note author or an Admin can delete.
|
||
* Optionally deletes all notes in the same group when ?group=true.
|
||
*
|
||
* @param id — numeric note ID
|
||
* @query group — optional, "true" to delete all notes sharing the same group_id
|
||
* @response 200 { deleted: number }
|
||
* @response 400 { error } — invalid note ID
|
||
* @response 403 { error } — not the author and not Admin
|
||
* @response 404 { error } — note not found
|
||
* @response 500 { error } — delete failure
|
||
*/
|
||
router.delete('/notes/:id', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const noteId = parseInt(req.params.id, 10);
|
||
if (isNaN(noteId)) return res.status(400).json({ error: 'Invalid note ID' });
|
||
const deleteGroup = req.query.group === 'true';
|
||
|
||
try {
|
||
const { rows } = await pool.query(`SELECT id, hostname, metric_id, note, group_id, created_by FROM compliance_notes WHERE id = $1`, [noteId]);
|
||
const noteRow = rows[0];
|
||
if (!noteRow) return res.status(404).json({ error: 'Note not found' });
|
||
|
||
const isAuthor = req.user && String(req.user.id) === String(noteRow.created_by);
|
||
const isAdminUser = req.user && req.user.group === 'Admin';
|
||
if (!isAuthor && !isAdminUser) return res.status(403).json({ error: 'You can only delete your own notes' });
|
||
|
||
let deleted = 0;
|
||
if (deleteGroup && noteRow.group_id) {
|
||
const result = await pool.query(`DELETE FROM compliance_notes WHERE group_id = $1`, [noteRow.group_id]);
|
||
deleted = result.rowCount;
|
||
} else {
|
||
const result = await pool.query(`DELETE FROM compliance_notes WHERE id = $1`, [noteId]);
|
||
deleted = result.rowCount;
|
||
}
|
||
|
||
logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_note_delete', entityType: 'compliance_note', entityId: String(noteId), details: JSON.stringify({ hostname: noteRow.hostname, group_id: noteRow.group_id, deleted_count: deleted }), ipAddress: req.ip });
|
||
res.json({ deleted });
|
||
} catch (err) {
|
||
console.error('[Compliance] DELETE /notes error:', err.message);
|
||
res.status(500).json({ error: 'Failed to delete note' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /trends
|
||
* Returns historical compliance upload trends with per-team breakdowns for charting.
|
||
*
|
||
* @response 200 { trends: Array<{ report_date, new_count, recurring_count, resolved_count, total_active, STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/trends', async (req, res) => {
|
||
try {
|
||
const { rows: uploads } = await pool.query(
|
||
`SELECT id, report_date, COALESCE(new_count, 0) AS new_count, COALESCE(recurring_count, 0) AS recurring_count, COALESCE(resolved_count, 0) AS resolved_count, COALESCE(new_count, 0) + COALESCE(recurring_count, 0) AS total_active FROM compliance_uploads ORDER BY report_date ASC`
|
||
);
|
||
if (uploads.length === 0) return res.json({ trends: [] });
|
||
|
||
const { rows: teamRows } = await pool.query(
|
||
`SELECT ci.upload_id, ci.team, COUNT(ci.id)::int AS count FROM compliance_items ci WHERE ci.team IS NOT NULL GROUP BY ci.upload_id, ci.team`
|
||
);
|
||
const teamMap = {};
|
||
teamRows.forEach(r => { if (!teamMap[r.upload_id]) teamMap[r.upload_id] = {}; teamMap[r.upload_id][r.team] = r.count; });
|
||
|
||
const trends = uploads.map(u => ({
|
||
report_date: u.report_date, new_count: u.new_count, recurring_count: u.recurring_count, resolved_count: u.resolved_count, total_active: u.total_active,
|
||
STEAM: teamMap[u.id]?.STEAM || 0, 'ACCESS-ENG': teamMap[u.id]?.['ACCESS-ENG'] || 0, 'ACCESS-OPS': teamMap[u.id]?.['ACCESS-OPS'] || 0, INTELDEV: teamMap[u.id]?.INTELDEV || 0,
|
||
}));
|
||
res.json({ trends });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /trends error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /mttr
|
||
* Returns aging bucket distribution of active compliance items (1 cycle, 2–3, 4–6, 7+) with per-team counts.
|
||
*
|
||
* @response 200 { aging: Array<{ bucket, total, STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/mttr', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(`SELECT COALESCE(seen_count, 1) AS seen_count, team FROM compliance_items WHERE status = 'active'`);
|
||
if (rows.length === 0) return res.json({ aging: [] });
|
||
const aging = bucketAgingItems(rows);
|
||
res.json({ aging });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /mttr error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /top-recurring
|
||
* Returns waterfall chart data computed from upload history (start, new, recurring, resolved, end per upload).
|
||
*
|
||
* @response 200 { waterfall: Array<{ date, start, new_count, recurring_count, resolved_count, end }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/top-recurring', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT id, report_date, COALESCE(new_count, 0) AS new_count, COALESCE(recurring_count, 0) AS recurring_count, COALESCE(resolved_count, 0) AS resolved_count FROM compliance_uploads ORDER BY report_date ASC`
|
||
);
|
||
const waterfall = computeWaterfall(rows);
|
||
res.json({ waterfall });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /top-recurring error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /category-trend
|
||
* Returns per-upload category breakdown counts for trend charting.
|
||
*
|
||
* @response 200 { categoryTrend: Array<{ report_date, category, count }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/category-trend', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT cu.report_date, COALESCE(ci.category, 'Unknown') AS category, COUNT(ci.id)::int AS count
|
||
FROM compliance_uploads cu JOIN compliance_items ci ON ci.upload_id = cu.id
|
||
GROUP BY cu.id, cu.report_date, category ORDER BY cu.report_date ASC`
|
||
);
|
||
res.json({ categoryTrend: rows });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /category-trend error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* PATCH /items/:hostname/metadata
|
||
* Updates resolution_date and/or remediation_plan for all active compliance items matching a hostname.
|
||
*
|
||
* @param hostname — the device hostname
|
||
* @body { resolution_date?: string|null, remediation_plan?: string|null }
|
||
* @response 200 { updated: number }
|
||
* @response 400 { error } — invalid hostname, invalid date format, plan exceeds 2000 chars, or no fields provided
|
||
* @response 404 { error } — device not found
|
||
* @response 500 { error } — update failure
|
||
*/
|
||
router.patch('/items/:hostname/metadata', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const hostname = req.params.hostname;
|
||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||
|
||
const { resolution_date, remediation_plan } = req.body;
|
||
|
||
// Validate resolution_date: must be a valid ISO date string or null
|
||
if (resolution_date !== undefined && resolution_date !== null) {
|
||
if (!isValidDateString(resolution_date)) {
|
||
return res.status(400).json({ error: 'Invalid resolution_date format' });
|
||
}
|
||
}
|
||
|
||
// Validate remediation_plan: must be <= 2000 chars or null
|
||
if (remediation_plan !== undefined && remediation_plan !== null) {
|
||
const planValidation = validateRemediationPlan(remediation_plan);
|
||
if (!planValidation.valid) {
|
||
return res.status(400).json({ error: planValidation.error });
|
||
}
|
||
}
|
||
|
||
try {
|
||
// Build dynamic SET clause for provided fields only
|
||
const setClauses = [];
|
||
const values = [];
|
||
let paramIdx = 1;
|
||
|
||
if (resolution_date !== undefined) {
|
||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||
values.push(resolution_date);
|
||
}
|
||
if (remediation_plan !== undefined) {
|
||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||
values.push(remediation_plan);
|
||
}
|
||
|
||
if (setClauses.length === 0) {
|
||
return res.status(400).json({ error: 'No fields to update' });
|
||
}
|
||
|
||
values.push(hostname);
|
||
const result = await pool.query(
|
||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx}`,
|
||
values
|
||
);
|
||
|
||
if (result.rowCount === 0) {
|
||
return res.status(404).json({ error: 'Device not found' });
|
||
}
|
||
|
||
logAudit({
|
||
userId: req.user.id,
|
||
username: req.user.username,
|
||
action: 'compliance_metadata_update',
|
||
entityType: 'compliance_item',
|
||
entityId: hostname,
|
||
details: { resolution_date, remediation_plan },
|
||
ipAddress: req.ip,
|
||
});
|
||
|
||
res.json({ updated: result.rowCount });
|
||
} catch (err) {
|
||
console.error('[Compliance] PATCH /items/:hostname/metadata error:', err.message);
|
||
res.status(500).json({ error: 'Failed to update device metadata' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /vcl/stats
|
||
* Returns VCL executive summary statistics including device counts, compliance percentage,
|
||
* non-compliant asset categorization (donut), heavy hitters by team, and vertical breakdown with burndown.
|
||
*
|
||
* @response 200 { stats: { total_devices, in_scope, compliant, non_compliant, remediations_required, compliance_pct, target_pct }, donut: { blocked: { count, pct }, in_progress: { count, pct } }, heavy_hitters: Array<{ vertical, team, non_compliant, compliance_date, notes }>, vertical_breakdown: Array<{ vertical, compliance_pct, team, non_compliant, actual_burndown, forecast_burndown, blockers, risk_acceptances, notes }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
const VCL_TARGET_PCT = parseInt(process.env.VCL_TARGET_PCT, 10) || 95;
|
||
|
||
router.get('/vcl/stats', async (req, res) => {
|
||
try {
|
||
// Compute device-level stats using DISTINCT hostname
|
||
// A device is "compliant" if it has NO active findings
|
||
const { rows: statsRows } = await pool.query(`
|
||
SELECT
|
||
COUNT(DISTINCT hostname) AS total_devices,
|
||
COUNT(DISTINCT hostname) AS in_scope,
|
||
COUNT(DISTINCT CASE
|
||
WHEN hostname NOT IN (SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active')
|
||
THEN hostname END) AS compliant,
|
||
COUNT(DISTINCT CASE
|
||
WHEN hostname IN (SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active')
|
||
THEN hostname END) AS non_compliant
|
||
FROM compliance_items
|
||
`);
|
||
|
||
const raw = statsRows[0] || {};
|
||
const total_devices = parseInt(raw.total_devices) || 0;
|
||
const in_scope = parseInt(raw.in_scope) || 0;
|
||
const compliant = parseInt(raw.compliant) || 0;
|
||
const non_compliant = parseInt(raw.non_compliant) || 0;
|
||
const compliance_pct = in_scope > 0 ? Math.round((compliant / in_scope) * 100) : 0;
|
||
|
||
const stats = {
|
||
total_devices,
|
||
in_scope,
|
||
compliant,
|
||
non_compliant,
|
||
remediations_required: non_compliant,
|
||
compliance_pct,
|
||
target_pct: VCL_TARGET_PCT,
|
||
};
|
||
|
||
// Donut: categorize non-compliant DEVICES by resolution_date presence
|
||
// A device is "blocked" if it has no resolution_date on any of its active findings
|
||
// A device is "in_progress" if at least one active finding has a resolution_date
|
||
const { rows: donutRows } = await pool.query(`
|
||
SELECT
|
||
hostname,
|
||
MAX(resolution_date) AS resolution_date
|
||
FROM compliance_items
|
||
WHERE status = 'active'
|
||
GROUP BY hostname
|
||
`);
|
||
const donut = categorizeNonCompliant(donutRows);
|
||
|
||
// Heavy hitters: group by team, count non-compliant DEVICES per team
|
||
const { rows: teamRows } = await pool.query(`
|
||
SELECT
|
||
COALESCE(team, 'Unknown') AS team,
|
||
COUNT(DISTINCT hostname) AS non_compliant,
|
||
MAX(resolution_date) AS compliance_date
|
||
FROM compliance_items
|
||
WHERE status = 'active'
|
||
GROUP BY team
|
||
ORDER BY COUNT(DISTINCT hostname) DESC
|
||
`);
|
||
const heavy_hitters = teamRows.map(r => ({
|
||
vertical: r.team,
|
||
team: r.team,
|
||
non_compliant: parseInt(r.non_compliant),
|
||
compliance_date: r.compliance_date ? r.compliance_date.toISOString().slice(0, 10) : null,
|
||
notes: '',
|
||
}));
|
||
|
||
// Vertical breakdown with burndown
|
||
const verticalBreakdown = [];
|
||
for (const teamRow of teamRows) {
|
||
const team = teamRow.team;
|
||
const teamNonCompliant = parseInt(teamRow.non_compliant);
|
||
|
||
// Get total devices for this team (all statuses)
|
||
const { rows: teamTotalRows } = await pool.query(
|
||
`SELECT COUNT(DISTINCT hostname) AS total FROM compliance_items WHERE COALESCE(team, 'Unknown') = $1`,
|
||
[team]
|
||
);
|
||
const teamTotal = parseInt(teamTotalRows[0]?.total) || 0;
|
||
const teamCompliant = teamTotal - teamNonCompliant;
|
||
const compliance_pct_team = teamTotal > 0 ? Math.round((teamCompliant / teamTotal) * 100) : 0;
|
||
|
||
// Forecast burndown from resolution_dates
|
||
const { rows: forecastItems } = await pool.query(
|
||
`SELECT resolution_date FROM compliance_items WHERE status = 'active' AND COALESCE(team, 'Unknown') = $1 AND resolution_date IS NOT NULL`,
|
||
[team]
|
||
);
|
||
const forecast_burndown = computeForecastBurndown(forecastItems);
|
||
const blockers = teamNonCompliant - forecastItems.length;
|
||
|
||
verticalBreakdown.push({
|
||
vertical: team,
|
||
compliance_pct: compliance_pct_team,
|
||
team: team,
|
||
non_compliant: teamNonCompliant,
|
||
actual_burndown: {},
|
||
forecast_burndown,
|
||
blockers: blockers > 0 ? blockers : 0,
|
||
risk_acceptances: 0,
|
||
notes: '',
|
||
});
|
||
}
|
||
|
||
// Merge vertical metadata (notes, risk_acceptances, compliance_date)
|
||
try {
|
||
const { rows: metaRows } = await pool.query(`SELECT team, notes, risk_acceptances, compliance_date FROM vcl_vertical_metadata`);
|
||
const metaMap = {};
|
||
metaRows.forEach(r => { metaMap[r.team] = r; });
|
||
|
||
for (const hh of heavy_hitters) {
|
||
const meta = metaMap[hh.vertical] || metaMap[hh.team];
|
||
if (meta) {
|
||
hh.notes = meta.notes || '';
|
||
hh.compliance_date = meta.compliance_date || hh.compliance_date;
|
||
}
|
||
}
|
||
for (const vb of verticalBreakdown) {
|
||
const meta = metaMap[vb.vertical] || metaMap[vb.team];
|
||
if (meta) {
|
||
vb.notes = meta.notes || '';
|
||
vb.risk_acceptances = meta.risk_acceptances || 0;
|
||
vb.compliance_date = meta.compliance_date || null;
|
||
}
|
||
}
|
||
} catch (metaErr) {
|
||
// Non-critical — continue without metadata
|
||
console.error('[Compliance] VCL metadata merge error:', metaErr.message);
|
||
}
|
||
|
||
res.json({ stats, donut, heavy_hitters, vertical_breakdown: verticalBreakdown });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /vcl/stats error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /vcl/trend
|
||
* Returns monthly compliance trend data with actual percentages and linear regression forecast.
|
||
* Forecast is computed when 3+ months of historical data exist, projecting 3 months forward.
|
||
*
|
||
* @response 200 { months: Array<{ month, compliant_count, compliance_pct, forecast_pct, target_pct }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/vcl/trend', async (req, res) => {
|
||
try {
|
||
const { rows: snapshots } = await pool.query(
|
||
`SELECT snapshot_month, SUM(compliant)::int AS compliant_count,
|
||
CASE WHEN SUM(total_devices) > 0
|
||
THEN ROUND((SUM(compliant)::numeric / SUM(total_devices)::numeric) * 100, 1)
|
||
ELSE 0 END AS compliance_pct
|
||
FROM compliance_snapshots
|
||
GROUP BY snapshot_month
|
||
ORDER BY snapshot_month ASC`
|
||
);
|
||
|
||
// Build months array with actuals
|
||
const months = snapshots.map(s => ({
|
||
month: s.snapshot_month,
|
||
compliant_count: s.compliant_count,
|
||
compliance_pct: parseFloat(s.compliance_pct),
|
||
forecast_pct: null,
|
||
target_pct: VCL_TARGET_PCT,
|
||
}));
|
||
|
||
// Compute forecast using linear regression if we have 3+ months
|
||
if (months.length >= 3) {
|
||
const n = months.length;
|
||
// Use last data points for regression
|
||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
|
||
for (let i = 0; i < n; i++) {
|
||
sumX += i;
|
||
sumY += months[i].compliance_pct;
|
||
sumXY += i * months[i].compliance_pct;
|
||
sumX2 += i * i;
|
||
}
|
||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
|
||
const intercept = (sumY - slope * sumX) / n;
|
||
|
||
// Project forward 3 months
|
||
for (let i = 0; i < 3; i++) {
|
||
const futureIdx = n + i;
|
||
const forecastPct = Math.min(100, Math.max(0, Math.round((slope * futureIdx + intercept) * 10) / 10));
|
||
|
||
// Compute the future month string
|
||
const lastMonth = months[months.length - 1].month;
|
||
const [year, mon] = lastMonth.split('-').map(Number);
|
||
const futureDate = new Date(year, mon - 1 + i + 1, 1);
|
||
const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`;
|
||
|
||
months.push({
|
||
month: futureMonth,
|
||
compliant_count: null,
|
||
compliance_pct: null,
|
||
forecast_pct: forecastPct,
|
||
target_pct: VCL_TARGET_PCT,
|
||
});
|
||
}
|
||
|
||
// Also add forecast_pct to the last actual month as the starting point
|
||
if (months.length > 0 && n > 0) {
|
||
months[n - 1].forecast_pct = months[n - 1].compliance_pct;
|
||
}
|
||
}
|
||
|
||
res.json({ months });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /vcl/trend error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /vcl/bulk-preview
|
||
* Accepts parsed bulk upload rows, matches hostnames against active devices, validates fields,
|
||
* and returns a diff preview showing matched/unmatched/changed/invalid row counts.
|
||
*
|
||
* @body { rows: Array<{ hostname, resolution_date?, remediation_plan?, notes? }>, headers?: string[] }
|
||
* @response 200 { matched, unmatched, changes, invalid, details: Array<{ hostname, status, fields? }>, unmatched_rows: string[], invalid_rows: Array<{ hostname, errors }> }
|
||
* @response 400 { error } — missing rows, exceeds 2000 rows, no Hostname column, or no updatable fields
|
||
* @response 500 { error } — processing failure
|
||
*/
|
||
router.post('/vcl/bulk-preview', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { rows, headers } = req.body;
|
||
|
||
// Validate: require rows array
|
||
if (!rows || !Array.isArray(rows)) {
|
||
return res.status(400).json({ error: 'rows array is required' });
|
||
}
|
||
|
||
// Enforce 2000 row limit
|
||
if (rows.length === 0) {
|
||
return res.status(400).json({ error: 'File contains no data rows' });
|
||
}
|
||
if (rows.length > 2000) {
|
||
return res.status(400).json({ error: 'File exceeds maximum of 2000 rows' });
|
||
}
|
||
|
||
// Map column headers if provided
|
||
let columnMapping = {};
|
||
if (headers && Array.isArray(headers)) {
|
||
columnMapping = mapColumnHeaders(headers);
|
||
}
|
||
|
||
// Require hostname field
|
||
const hasHostname = rows.every(r => r.hostname != null && r.hostname !== '');
|
||
if (!hasHostname) {
|
||
return res.status(400).json({ error: 'File must contain a Hostname column' });
|
||
}
|
||
|
||
// Check for updatable fields (resolution_date, remediation_plan, or notes)
|
||
const sampleRow = rows[0] || {};
|
||
const updatableFields = ['resolution_date', 'remediation_plan', 'notes'];
|
||
const hasUpdatableFields = updatableFields.some(f => f in sampleRow);
|
||
if (!hasUpdatableFields && headers) {
|
||
// Check via column mapping
|
||
const mappedFields = Object.keys(columnMapping).filter(k => k !== 'hostname');
|
||
if (mappedFields.length === 0) {
|
||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||
}
|
||
} else if (!hasUpdatableFields && !headers) {
|
||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||
}
|
||
|
||
try {
|
||
// Get existing hostnames from DB
|
||
const { rows: existingRows } = await pool.query(
|
||
`SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active'`
|
||
);
|
||
const existingHostnames = new Set(existingRows.map(r => r.hostname));
|
||
|
||
// Match by hostname
|
||
const { matched, unmatched } = matchByHostname(rows, existingHostnames);
|
||
|
||
// Validate fields on matched rows
|
||
const validRows = [];
|
||
const invalidRows = [];
|
||
|
||
for (const row of matched) {
|
||
const errors = [];
|
||
|
||
if (row.resolution_date !== undefined && row.resolution_date !== null && row.resolution_date !== '') {
|
||
if (!isValidDateString(row.resolution_date)) {
|
||
errors.push('resolution_date: invalid date format');
|
||
}
|
||
}
|
||
|
||
if (row.remediation_plan !== undefined && row.remediation_plan !== null) {
|
||
const planCheck = validateRemediationPlan(row.remediation_plan);
|
||
if (!planCheck.valid) {
|
||
errors.push('remediation_plan: ' + planCheck.error);
|
||
}
|
||
}
|
||
|
||
if (errors.length > 0) {
|
||
invalidRows.push({ hostname: row.hostname, errors });
|
||
} else {
|
||
validRows.push(row);
|
||
}
|
||
}
|
||
|
||
// Get current data for diff computation
|
||
const { rows: currentRows } = await pool.query(
|
||
`SELECT DISTINCT ON (hostname) hostname, resolution_date, remediation_plan
|
||
FROM compliance_items WHERE status = 'active' AND hostname = ANY($1)
|
||
ORDER BY hostname, id DESC`,
|
||
[validRows.map(r => r.hostname)]
|
||
);
|
||
const currentData = new Map();
|
||
for (const row of currentRows) {
|
||
currentData.set(row.hostname, {
|
||
resolution_date: row.resolution_date ? row.resolution_date.toISOString?.().slice(0, 10) || String(row.resolution_date).slice(0, 10) : null,
|
||
remediation_plan: row.remediation_plan || null,
|
||
notes: null,
|
||
});
|
||
}
|
||
|
||
// Compute diff
|
||
const diffResults = computeBulkDiff(validRows, currentData);
|
||
const changedRows = diffResults.filter(r => r.status === 'changed');
|
||
|
||
res.json({
|
||
matched: matched.length,
|
||
unmatched: unmatched.length,
|
||
changes: changedRows.length,
|
||
invalid: invalidRows.length,
|
||
details: diffResults,
|
||
unmatched_rows: unmatched.map(r => r.hostname),
|
||
invalid_rows: invalidRows,
|
||
});
|
||
} catch (err) {
|
||
console.error('[Compliance] POST /vcl/bulk-preview error:', err.message);
|
||
res.status(500).json({ error: 'Failed to process bulk preview' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /vcl/bulk-commit
|
||
* Commits validated bulk changes to compliance items in a single transaction.
|
||
* Updates resolution_date and/or remediation_plan for each hostname provided.
|
||
*
|
||
* @body { changes: Array<{ hostname, resolution_date?, remediation_plan?, notes? }> }
|
||
* @response 200 { committed: number }
|
||
* @response 400 { error } — missing or empty changes array
|
||
* @response 500 { error } — transaction failure (full rollback)
|
||
*/
|
||
router.post('/vcl/bulk-commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { changes } = req.body;
|
||
|
||
if (!changes || !Array.isArray(changes) || changes.length === 0) {
|
||
return res.status(400).json({ error: 'changes array is required' });
|
||
}
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
let committedCount = 0;
|
||
for (const change of changes) {
|
||
const setClauses = [];
|
||
const values = [];
|
||
let paramIdx = 1;
|
||
|
||
if (change.resolution_date !== undefined) {
|
||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||
values.push(change.resolution_date);
|
||
}
|
||
if (change.remediation_plan !== undefined) {
|
||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||
values.push(change.remediation_plan);
|
||
}
|
||
if (change.notes !== undefined) {
|
||
// Notes are stored separately in compliance_notes, but we can update a field if it exists
|
||
// For now, skip notes in the direct update
|
||
}
|
||
|
||
if (setClauses.length === 0) continue;
|
||
|
||
values.push(change.hostname);
|
||
const result = await client.query(
|
||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||
values
|
||
);
|
||
if (result.rowCount > 0) committedCount++;
|
||
}
|
||
|
||
await client.query('COMMIT');
|
||
|
||
logAudit({
|
||
userId: req.user.id,
|
||
username: req.user.username,
|
||
action: 'compliance_bulk_update',
|
||
entityType: 'compliance_items',
|
||
entityId: null,
|
||
details: { rows_updated: committedCount, total_changes: changes.length },
|
||
ipAddress: req.ip,
|
||
});
|
||
|
||
res.json({ committed: committedCount });
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
console.error('[Compliance] POST /vcl/bulk-commit error:', err.message);
|
||
res.status(500).json({ error: 'Failed to commit changes' });
|
||
} finally {
|
||
client.release();
|
||
}
|
||
});
|
||
|
||
// -----------------------------------------------------------------------
|
||
// VCL Vertical Metadata endpoints
|
||
// -----------------------------------------------------------------------
|
||
|
||
/**
|
||
* GET /vcl/vertical-metadata
|
||
* Returns all rows from vcl_vertical_metadata.
|
||
*
|
||
* @response 200 { metadata: Array<{ id, team, notes, risk_acceptances, compliance_date, updated_at }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/vcl/vertical-metadata', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT id, team, notes, risk_acceptances, compliance_date, updated_at FROM vcl_vertical_metadata ORDER BY team`
|
||
);
|
||
res.json({ metadata: rows });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /vcl/vertical-metadata error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* PATCH /vcl/vertical-metadata/:team
|
||
* Upserts notes, risk_acceptances, and/or compliance_date for a team.
|
||
*
|
||
* @param team — the team/vertical name
|
||
* @body { notes?: string, risk_acceptances?: number, compliance_date?: string|null }
|
||
* @response 200 { success: true }
|
||
* @response 400 { error } — no fields provided or invalid values
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.patch('/vcl/vertical-metadata/:team', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const team = req.params.team;
|
||
if (!team || team.length > 100) return res.status(400).json({ error: 'Invalid team' });
|
||
|
||
const { notes, risk_acceptances, compliance_date } = req.body;
|
||
|
||
if (notes === undefined && risk_acceptances === undefined && compliance_date === undefined) {
|
||
return res.status(400).json({ error: 'No fields to update' });
|
||
}
|
||
|
||
if (risk_acceptances !== undefined && risk_acceptances !== null) {
|
||
if (typeof risk_acceptances !== 'number' || risk_acceptances < 0 || !Number.isInteger(risk_acceptances)) {
|
||
return res.status(400).json({ error: 'risk_acceptances must be a non-negative integer' });
|
||
}
|
||
}
|
||
|
||
if (compliance_date !== undefined && compliance_date !== null && compliance_date !== '') {
|
||
if (typeof compliance_date !== 'string' || compliance_date.length > 50) {
|
||
return res.status(400).json({ error: 'compliance_date must be a string (max 50 chars)' });
|
||
}
|
||
}
|
||
|
||
try {
|
||
// Build the upsert dynamically
|
||
const upsertNotes = notes !== undefined ? notes : '';
|
||
const upsertRAs = risk_acceptances !== undefined ? risk_acceptances : 0;
|
||
const upsertDate = compliance_date !== undefined ? (compliance_date || null) : null;
|
||
|
||
// Use ON CONFLICT to insert or update only the provided fields
|
||
const updateParts = [];
|
||
if (notes !== undefined) updateParts.push('notes = EXCLUDED.notes');
|
||
if (risk_acceptances !== undefined) updateParts.push('risk_acceptances = EXCLUDED.risk_acceptances');
|
||
if (compliance_date !== undefined) updateParts.push('compliance_date = EXCLUDED.compliance_date');
|
||
updateParts.push('updated_at = NOW()');
|
||
|
||
await pool.query(
|
||
`INSERT INTO vcl_vertical_metadata (team, notes, risk_acceptances, compliance_date, updated_at)
|
||
VALUES ($1, $2, $3, $4, NOW())
|
||
ON CONFLICT (team) DO UPDATE SET ${updateParts.join(', ')}`,
|
||
[team, upsertNotes, upsertRAs, upsertDate]
|
||
);
|
||
|
||
res.json({ success: true });
|
||
} catch (err) {
|
||
console.error('[Compliance] PATCH /vcl/vertical-metadata error:', err.message);
|
||
res.status(500).json({ error: 'Failed to update vertical metadata' });
|
||
}
|
||
});
|
||
|
||
return router;
|
||
}
|
||
|
||
module.exports = { createComplianceRouter, bucketAgingItems, computeWaterfall };
|