d65411b0d7
Add ci.resolution_date and ci.remediation_plan to the GET /items endpoint SELECT clause and update groupByHostname() to aggregate them as first-non-null across each hostname's metric rows. The frontend already rendered these columns but the list endpoint never fetched the data from the database. Includes exploration and preservation property tests for groupByHostname().
1744 lines
85 KiB
JavaScript
1744 lines
85 KiB
JavaScript
// Compliance Routes — AEO metric tracking
|
||
// Handles xlsx upload/parse, non-compliant item history, and notes.
|
||
|
||
const express = require('express');
|
||
const path = require('path');
|
||
const fs = require('fs');
|
||
const crypto = require('crypto');
|
||
const { spawn } = require('child_process');
|
||
const pool = require('../db');
|
||
const { requireAuth, requireGroup } = require('../middleware/auth');
|
||
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
|
||
const { isValidDateString, validateRemediationPlan, computeVCLStats, categorizeNonCompliant, rankHeavyHitters, computeForecastBurndown, matchByHostname, computeBulkDiff, mapColumnHeaders } = require('../helpers/vclHelpers');
|
||
const logAudit = require('../helpers/auditLog');
|
||
|
||
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
|
||
const SCHEMA_SCRIPT = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
|
||
const CONFIG_PATH = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
|
||
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
|
||
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
|
||
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Run Python parser, return parsed object
|
||
// ---------------------------------------------------------------------------
|
||
function parseXlsx(filePath) {
|
||
return new Promise((resolve, reject) => {
|
||
const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
|
||
let out = '';
|
||
let err = '';
|
||
py.stdout.on('data', d => { out += d; });
|
||
py.stderr.on('data', d => { err += d; });
|
||
py.on('close', code => {
|
||
if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
|
||
try { resolve(JSON.parse(out)); }
|
||
catch (e) { reject(new Error('Parser returned invalid JSON')); }
|
||
});
|
||
py.on('error', reject);
|
||
});
|
||
}
|
||
|
||
function extractXlsxSchema(filePath) {
|
||
return new Promise((resolve, reject) => {
|
||
const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
|
||
let out = '';
|
||
let err = '';
|
||
py.stdout.on('data', d => { out += d; });
|
||
py.stderr.on('data', d => { err += d; });
|
||
py.on('close', code => {
|
||
if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
|
||
try { resolve(JSON.parse(out)); }
|
||
catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
|
||
});
|
||
py.on('error', reject);
|
||
});
|
||
}
|
||
|
||
function isSafeTempPath(filePath) {
|
||
const resolved = path.resolve(filePath);
|
||
return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Compute diff: new / recurring / resolved
|
||
// ---------------------------------------------------------------------------
|
||
async function computeDiff(incomingItems) {
|
||
const { rows: activeRows } = await pool.query(
|
||
`SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
|
||
);
|
||
const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
|
||
const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));
|
||
|
||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
|
||
for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; }
|
||
|
||
return { newCount, recurringCount, resolvedCount };
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Write a parsed upload to the DB (within a transaction)
|
||
//
|
||
// `vertical` defaults to null for legacy AEO uploads (the /commit route).
|
||
// When threaded through from a multi-vertical caller it filters the
|
||
// compliance_snapshots aggregation so the snapshot reflects only the
|
||
// snapshotted vertical's items — this prevents cross-vertical
|
||
// contamination on dates where multiple verticals share a report_date.
|
||
// ---------------------------------------------------------------------------
|
||
async function persistUpload({ items, summary, reportDate, filename, userId, vertical = null }) {
|
||
const { rows: activeRows } = await pool.query(
|
||
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
|
||
);
|
||
const activeMap = {};
|
||
activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });
|
||
|
||
const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
// 1. Insert the upload record
|
||
const uploadResult = await client.query(
|
||
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
|
||
VALUES ($1, $2, $3, NOW(), $4)
|
||
RETURNING id`,
|
||
[filename, reportDate || null, userId || null, JSON.stringify(summary)]
|
||
);
|
||
const uploadId = uploadResult.rows[0].id;
|
||
|
||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||
|
||
// 2. Upsert each incoming non-compliant item
|
||
for (const item of items) {
|
||
const key = `${item.hostname}|||${item.metric_id}`;
|
||
const existing = activeMap[key];
|
||
const extraStr = JSON.stringify(item.extra_json || {});
|
||
|
||
if (existing) {
|
||
await client.query(
|
||
`UPDATE compliance_items
|
||
SET upload_id = $1, seen_count = $2, ip_address = $3, device_type = $4, extra_json = $5
|
||
WHERE id = $6`,
|
||
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
|
||
);
|
||
recurringCount++;
|
||
} else {
|
||
await client.query(
|
||
`INSERT INTO compliance_items
|
||
(upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
|
||
category, extra_json, status, first_seen_upload_id, seen_count)
|
||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, 1)`,
|
||
[uploadId, item.hostname, item.ip_address, item.device_type, item.team,
|
||
item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
|
||
);
|
||
newCount++;
|
||
}
|
||
}
|
||
|
||
// 3. Mark items not present in this upload as resolved
|
||
for (const [key, row] of Object.entries(activeMap)) {
|
||
if (!newKeys.has(key)) {
|
||
await client.query(
|
||
`UPDATE compliance_items SET status = 'resolved', resolved_upload_id = $1 WHERE id = $2`,
|
||
[uploadId, row.id]
|
||
);
|
||
resolvedCount++;
|
||
}
|
||
}
|
||
|
||
// 4. Update upload with final counts
|
||
await client.query(
|
||
`UPDATE compliance_uploads SET new_count = $1, resolved_count = $2, recurring_count = $3 WHERE id = $4`,
|
||
[newCount, resolvedCount, recurringCount, uploadId]
|
||
);
|
||
|
||
await client.query('COMMIT');
|
||
|
||
// Task 7: Create/update compliance_snapshots for the current month
|
||
try {
|
||
const currentMonth = new Date().toISOString().slice(0, 7); // YYYY-MM
|
||
|
||
// Compute compliance percentages for the snapshotted vertical only.
|
||
// CTE classifies each hostname by its worst-case status (active wins
|
||
// over resolved via MIN) so a hostname with both active and resolved
|
||
// rows across verticals is counted in exactly one column.
|
||
const { rows: verticalStats } = await pool.query(
|
||
`WITH hostname_status AS (
|
||
SELECT team,
|
||
hostname,
|
||
MIN(status) AS status
|
||
FROM compliance_items
|
||
WHERE team IS NOT NULL AND vertical IS NOT DISTINCT FROM $1
|
||
GROUP BY team, hostname
|
||
)
|
||
SELECT team AS vertical,
|
||
COUNT(*)::int AS total_devices,
|
||
COUNT(*) FILTER (WHERE status = 'resolved')::int AS compliant,
|
||
COUNT(*) FILTER (WHERE status = 'active')::int AS non_compliant
|
||
FROM hostname_status
|
||
GROUP BY team`,
|
||
[vertical]
|
||
);
|
||
|
||
for (const vs of verticalStats) {
|
||
const total = vs.total_devices;
|
||
const compPct = total > 0 ? Math.round((vs.compliant / total) * 100 * 100) / 100 : 0;
|
||
// For non-null verticals (multi-vertical uploads), the snapshot
|
||
// row is keyed by the actual vertical so /vcl/stats consumers
|
||
// see the correct breakdown. For legacy AEO uploads (vertical
|
||
// is null), preserve the historical team-as-vertical key so
|
||
// existing single-vertical-month consumers are unchanged.
|
||
const snapshotVertical = vs.vertical || vs.team;
|
||
|
||
await pool.query(
|
||
`INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||
VALUES ($1, $2, $3, $4, $5, $6)
|
||
ON CONFLICT (snapshot_month, vertical)
|
||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||
[currentMonth, snapshotVertical, total, vs.compliant, vs.non_compliant, compPct]
|
||
);
|
||
}
|
||
} catch (snapshotErr) {
|
||
// Snapshot creation is non-critical — log but don't fail the upload
|
||
console.error('[Compliance] Snapshot creation error:', snapshotErr.message);
|
||
}
|
||
|
||
return { uploadId, newCount, recurringCount, resolvedCount };
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
throw err;
|
||
} finally {
|
||
client.release();
|
||
}
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Group flat compliance_items rows into per-device objects
|
||
// ---------------------------------------------------------------------------
|
||
function groupByHostname(rows, noteHostnames) {
|
||
const deviceMap = {};
|
||
for (const row of rows) {
|
||
if (!deviceMap[row.hostname]) {
|
||
deviceMap[row.hostname] = {
|
||
hostname: row.hostname, ip_address: row.ip_address || '', device_type: row.device_type || '',
|
||
team: row.team || '', status: row.status, failing_metrics: [],
|
||
_seenMetricIds: new Set(),
|
||
seen_count: row.seen_count || 1, first_seen: row.first_seen || null,
|
||
last_seen: row.last_seen || null, resolved_on: row.resolved_on || null,
|
||
has_notes: noteHostnames.has(row.hostname),
|
||
resolution_date: null, remediation_plan: null,
|
||
};
|
||
}
|
||
const dev = deviceMap[row.hostname];
|
||
if (!dev._seenMetricIds.has(row.metric_id)) {
|
||
dev._seenMetricIds.add(row.metric_id);
|
||
dev.failing_metrics.push({ metric_id: row.metric_id, metric_desc: row.metric_desc || '', category: row.category || '' });
|
||
}
|
||
if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count;
|
||
if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen)) dev.first_seen = row.first_seen;
|
||
if (row.last_seen && (!dev.last_seen || row.last_seen > dev.last_seen)) dev.last_seen = row.last_seen;
|
||
if (row.resolution_date && !dev.resolution_date) dev.resolution_date = row.resolution_date;
|
||
if (row.remediation_plan && !dev.remediation_plan) dev.remediation_plan = row.remediation_plan;
|
||
}
|
||
return Object.values(deviceMap).map(({ _seenMetricIds, ...dev }) => dev);
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Pure helpers
|
||
// ---------------------------------------------------------------------------
|
||
const BUCKET_ORDER = ['1 cycle', '2–3 cycles', '4–6 cycles', '7+ cycles'];
|
||
|
||
function bucketAgingItems(items) {
|
||
const teams = ['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV'];
|
||
const buckets = {};
|
||
for (const b of BUCKET_ORDER) {
|
||
buckets[b] = { bucket: b, total: 0 };
|
||
for (const t of teams) buckets[b][t] = 0;
|
||
}
|
||
for (const item of items) {
|
||
const sc = item.seen_count;
|
||
let label;
|
||
if (sc === 1) label = '1 cycle';
|
||
else if (sc >= 2 && sc <= 3) label = '2–3 cycles';
|
||
else if (sc >= 4 && sc <= 6) label = '4–6 cycles';
|
||
else label = '7+ cycles';
|
||
buckets[label].total += 1;
|
||
if (item.team in buckets[label]) buckets[label][item.team] += 1;
|
||
}
|
||
return BUCKET_ORDER.map(b => buckets[b]);
|
||
}
|
||
|
||
function computeWaterfall(uploads) {
|
||
let start = 0;
|
||
return uploads.map((row) => {
|
||
const end = start + row.new_count + row.recurring_count - row.resolved_count;
|
||
const entry = { date: row.report_date, start, new_count: row.new_count, recurring_count: row.recurring_count, resolved_count: row.resolved_count, end };
|
||
start = end;
|
||
return entry;
|
||
});
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Router factory
|
||
// ---------------------------------------------------------------------------
|
||
function createComplianceRouter(upload) {
|
||
const router = express.Router();
|
||
|
||
// All compliance routes require authentication
|
||
router.use(requireAuth());
|
||
|
||
/**
|
||
* POST /preview
|
||
* Uploads an xlsx file, parses it, computes a diff against current active items,
|
||
* performs schema drift detection, and stores parsed data in a temp file for later commit.
|
||
*
|
||
* @body multipart/form-data — field "file" (xlsx spreadsheet, max 10MB)
|
||
* @response 200 { drift, drift_error, schema, diff: { new_count, recurring_count, resolved_count }, tempFile, filename, report_date, total_items }
|
||
* @response 400 { error } — no file, wrong extension, or upload error
|
||
* @response 422 { error } — parser returned an error
|
||
* @response 500 { error } — config load failure or parse failure
|
||
*/
|
||
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
|
||
upload.single('file')(req, res, async (uploadErr) => {
|
||
if (uploadErr) return res.status(400).json({ error: uploadErr.message });
|
||
if (!req.file) return res.status(400).json({ error: 'No file uploaded' });
|
||
if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
|
||
fs.unlink(req.file.path, () => {});
|
||
return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
|
||
}
|
||
|
||
try {
|
||
let drift = null, drift_error = null;
|
||
let config;
|
||
try { config = loadConfig(CONFIG_PATH); } catch (configErr) {
|
||
fs.unlink(req.file.path, () => {});
|
||
return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
|
||
}
|
||
|
||
let xlsxSchema = null;
|
||
try {
|
||
xlsxSchema = await extractXlsxSchema(req.file.path);
|
||
if (xlsxSchema.error) throw new Error(xlsxSchema.error);
|
||
drift = compareSchemaToDrift(xlsxSchema, config);
|
||
} catch (driftErr) {
|
||
drift = null;
|
||
drift_error = driftErr.message || 'Drift check failed';
|
||
}
|
||
|
||
const parsed = await parseXlsx(req.file.path);
|
||
if (parsed.error) {
|
||
fs.unlink(req.file.path, () => {});
|
||
return res.status(422).json({ error: parsed.error });
|
||
}
|
||
|
||
const diff = await computeDiff(parsed.items);
|
||
|
||
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||
const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
|
||
const tempFilePath = path.join(TEMP_DIR, tempFilename);
|
||
|
||
fs.writeFileSync(tempFilePath, JSON.stringify({
|
||
items: parsed.items, summary: parsed.summary,
|
||
report_date: parsed.report_date,
|
||
filename: req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
|
||
}));
|
||
|
||
fs.unlink(req.file.path, () => {});
|
||
|
||
res.json({
|
||
drift, drift_error, schema: xlsxSchema,
|
||
diff: { new_count: diff.newCount, recurring_count: diff.recurringCount, resolved_count: diff.resolvedCount },
|
||
tempFile: tempFilePath, filename: req.file.originalname,
|
||
report_date: parsed.report_date, total_items: parsed.total,
|
||
});
|
||
} catch (err) {
|
||
fs.unlink(req.file.path, () => {});
|
||
console.error('[Compliance] Preview error:', err.message);
|
||
res.status(500).json({ error: 'Failed to parse file: ' + err.message });
|
||
}
|
||
});
|
||
});
|
||
|
||
/**
|
||
* POST /reconcile-config
|
||
* Applies schema drift reconciliation to the compliance config file based on detected drift findings.
|
||
*
|
||
* @body { drift: object, schema?: object }
|
||
* @response 200 { changes: Array<{ action, key, value, detail }>, message }
|
||
* @response 400 { error } — missing drift or no findings to reconcile
|
||
* @response 500 { error } — reconciliation failure
|
||
*/
|
||
router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
|
||
const { drift, schema } = req.body;
|
||
if (!drift || typeof drift !== 'object') return res.status(400).json({ error: 'drift report is required in request body' });
|
||
const hasFindings = (drift.breaking && drift.breaking.length > 0) || (drift.silent_miss && drift.silent_miss.length > 0);
|
||
if (!hasFindings) return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });
|
||
|
||
try {
|
||
const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);
|
||
if (changes.length === 0) return res.json({ changes: [], message: 'No changes needed' });
|
||
|
||
for (const change of changes) {
|
||
logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_config_reconcile', entityType: 'compliance_config', entityId: change.value, details: { action: change.action, key: change.key, detail: change.detail }, ipAddress: req.ip });
|
||
}
|
||
res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });
|
||
} catch (err) {
|
||
console.error('[Compliance] Reconcile config error:', err.message);
|
||
res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /commit
|
||
* Commits a previously previewed compliance upload to the database. Resolves items no longer
|
||
* present, upserts recurring/new items, and creates a compliance snapshot for the current month.
|
||
*
|
||
* @body { tempFile: string, filename?: string, report_date?: string }
|
||
* @response 200 { upload: { id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count } }
|
||
* @response 400 { error } — missing/invalid tempFile or expired preview session
|
||
* @response 500 { error } — commit failure
|
||
*/
|
||
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { tempFile, filename, report_date } = req.body;
|
||
if (!tempFile || typeof tempFile !== 'string') return res.status(400).json({ error: 'tempFile is required' });
|
||
if (!isSafeTempPath(tempFile)) return res.status(400).json({ error: 'Invalid tempFile path' });
|
||
if (!fs.existsSync(tempFile)) return res.status(400).json({ error: 'Preview session expired — please upload again' });
|
||
|
||
let parsed;
|
||
try { parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8')); }
|
||
catch { return res.status(400).json({ error: 'Could not read preview data — please upload again' }); }
|
||
|
||
try {
|
||
const result = await persistUpload({
|
||
items: parsed.items, summary: parsed.summary,
|
||
reportDate: report_date || parsed.report_date,
|
||
filename: filename || parsed.filename,
|
||
userId: req.user?.id || null,
|
||
});
|
||
fs.unlink(tempFile, () => {});
|
||
|
||
const { rows } = await pool.query(
|
||
`SELECT id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count
|
||
FROM compliance_uploads WHERE id = $1`, [result.uploadId]
|
||
);
|
||
res.json({ upload: rows[0] });
|
||
} catch (err) {
|
||
console.error('[Compliance] Commit error:', err.message);
|
||
res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /uploads
|
||
* Returns all compliance upload records ordered by most recent first.
|
||
*
|
||
* @response 200 { uploads: Array<{ id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/uploads', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count
|
||
FROM compliance_uploads ORDER BY id DESC`
|
||
);
|
||
res.json({ uploads: rows });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /uploads error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /rollback/:uploadId
|
||
* Rolls back the most recent compliance upload — deletes new items introduced by that upload,
|
||
* reactivates items it resolved, and removes the upload record.
|
||
*
|
||
* @param uploadId — numeric ID of the upload to roll back (must be the latest)
|
||
* @response 200 { message, rolled_back: { upload_id, filename, report_date, items_deleted, items_reactivated } }
|
||
* @response 400 { error } — invalid ID or not the latest upload
|
||
* @response 404 { error } — upload not found
|
||
* @response 500 { error } — rollback failure
|
||
*/
|
||
router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
|
||
const uploadId = parseInt(req.params.uploadId, 10);
|
||
if (isNaN(uploadId)) return res.status(400).json({ error: 'Invalid upload ID' });
|
||
|
||
try {
|
||
const { rows: uploadRows } = await pool.query(
|
||
`SELECT id, filename, report_date, new_count, resolved_count, recurring_count FROM compliance_uploads WHERE id = $1`, [uploadId]
|
||
);
|
||
const upload = uploadRows[0];
|
||
if (!upload) return res.status(404).json({ error: 'Upload not found' });
|
||
|
||
const { rows: latestRows } = await pool.query(`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`);
|
||
if (latestRows[0].id !== uploadId) {
|
||
return res.status(400).json({ error: 'Only the most recent upload can be rolled back', latest_upload_id: latestRows[0].id });
|
||
}
|
||
|
||
const { rows: prevRows } = await pool.query(`SELECT id FROM compliance_uploads WHERE id < $1 ORDER BY id DESC LIMIT 1`, [uploadId]);
|
||
const previousUpload = prevRows[0];
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
const deleteNew = await client.query(
|
||
`DELETE FROM compliance_items WHERE first_seen_upload_id = $1 AND upload_id = $1`, [uploadId]
|
||
);
|
||
const reactivate = await client.query(
|
||
`UPDATE compliance_items SET status = 'active', resolved_upload_id = NULL WHERE resolved_upload_id = $1`, [uploadId]
|
||
);
|
||
if (previousUpload) {
|
||
await client.query(
|
||
`UPDATE compliance_items SET upload_id = $1, seen_count = GREATEST(seen_count - 1, 1) WHERE upload_id = $2 AND first_seen_upload_id != $2`,
|
||
[previousUpload.id, uploadId]
|
||
);
|
||
}
|
||
await client.query(`DELETE FROM compliance_uploads WHERE id = $1`, [uploadId]);
|
||
await client.query('COMMIT');
|
||
|
||
logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_upload_rollback', entityType: 'compliance_upload', entityId: String(uploadId), details: { filename: upload.filename, report_date: upload.report_date, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount }, ipAddress: req.ip });
|
||
|
||
res.json({ message: `Rolled back upload "${upload.filename}"`, rolled_back: { upload_id: uploadId, filename: upload.filename, report_date: upload.report_date, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount } });
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
throw err;
|
||
} finally {
|
||
client.release();
|
||
}
|
||
} catch (err) {
|
||
console.error('[Compliance] Rollback error:', err.message);
|
||
res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /summary
|
||
* Returns the summary data from the most recent compliance upload, optionally filtered by team.
|
||
*
|
||
* @query team — optional, one of STEAM | ACCESS-ENG | ACCESS-OPS | INTELDEV
|
||
* @response 200 { entries: Array, overall_scores: object, upload: { id, report_date, uploaded_at } | null, multi_vertical_uploads?: Array<{ id, vertical, uploaded_at }> }
|
||
* @response 400 { error } — invalid team
|
||
* @response 500 { error } — database error
|
||
*
|
||
* When two or more uploads share the latest `report_date` (multi-vertical
|
||
* upload day), the `multi_vertical_uploads` field discloses the sibling
|
||
* uploads (id/vertical/uploaded_at) so callers know the response is
|
||
* partial. The field is omitted on single-upload-per-date dates to
|
||
* preserve the legacy response shape.
|
||
*/
|
||
router.get('/summary', async (req, res) => {
|
||
const team = req.query.team;
|
||
if (team && !ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
||
|
||
try {
|
||
// Try AEO uploads first (vertical IS NULL), fall back to NTS_AEO multi-vertical upload
|
||
let { rows: latestRows } = await pool.query(
|
||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical IS NULL ORDER BY id DESC LIMIT 1`
|
||
);
|
||
if (latestRows.length === 0 || !latestRows[0].summary_json) {
|
||
({ rows: latestRows } = await pool.query(
|
||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical = 'NTS_AEO' ORDER BY id DESC LIMIT 1`
|
||
));
|
||
}
|
||
const latestUpload = latestRows[0];
|
||
if (!latestUpload || !latestUpload.summary_json) return res.json({ entries: [], overall_scores: {}, upload: null });
|
||
|
||
let summary;
|
||
try { summary = JSON.parse(latestUpload.summary_json); } catch { return res.json({ entries: [], overall_scores: {}, upload: null }); }
|
||
|
||
let entries = summary.entries || [];
|
||
if (team) entries = entries.filter(e => e.team === team);
|
||
|
||
// Disclose sibling uploads sharing the same report_date so callers
|
||
// know the response is a single vertical's view of a multi-vertical
|
||
// day. Field is omitted when no siblings exist (preserves legacy
|
||
// single-upload-per-date response shape).
|
||
const { rows: siblingRows } = await pool.query(
|
||
`SELECT id, vertical, uploaded_at FROM compliance_uploads WHERE report_date = $1 AND id != $2 ORDER BY id ASC`,
|
||
[latestUpload.report_date, latestUpload.id]
|
||
);
|
||
|
||
const response = {
|
||
entries,
|
||
overall_scores: summary.overall_scores || {},
|
||
upload: { id: latestUpload.id, report_date: latestUpload.report_date, uploaded_at: latestUpload.uploaded_at },
|
||
};
|
||
if (siblingRows.length > 0) {
|
||
response.multi_vertical_uploads = siblingRows.map(s => ({
|
||
id: s.id,
|
||
vertical: s.vertical,
|
||
uploaded_at: s.uploaded_at,
|
||
}));
|
||
}
|
||
res.json(response);
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /summary error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /items
|
||
* Returns compliance items grouped by hostname for a given team and status.
|
||
*
|
||
* @query team — required, one of STEAM | ACCESS-ENG | ACCESS-OPS | INTELDEV
|
||
* @query status — optional, "active" (default) or "resolved"
|
||
* @response 200 { devices: Array<{ hostname, ip_address, device_type, team, status, failing_metrics, seen_count, first_seen, last_seen, resolved_on, has_notes }>, team, status }
|
||
* @response 400 { error } — missing/invalid team or invalid status
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/items', async (req, res) => {
|
||
const { team, status = 'active' } = req.query;
|
||
if (!team) return res.status(400).json({ error: 'team is required' });
|
||
if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
||
if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' });
|
||
|
||
try {
|
||
// Include items from both AEO uploads (vertical IS NULL) and NTS_AEO multi-vertical uploads
|
||
// DISTINCT ON deduplicates cross-vertical (hostname, metric_id) pairs, keeping the representative row
|
||
const { rows } = await pool.query(
|
||
`SELECT DISTINCT ON (ci.hostname, ci.metric_id)
|
||
ci.hostname, ci.ip_address, ci.device_type, ci.team, ci.metric_id, ci.metric_desc, ci.category, ci.status, ci.seen_count,
|
||
ci.resolution_date, ci.remediation_plan,
|
||
fu.report_date AS first_seen, lu.report_date AS last_seen, ru.report_date AS resolved_on
|
||
FROM compliance_items ci
|
||
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
||
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
||
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
||
WHERE ci.team = $1 AND ci.status = $2 AND (ci.vertical IS NULL OR ci.vertical = 'NTS_AEO')
|
||
ORDER BY ci.hostname, ci.metric_id, ci.seen_count DESC, ci.upload_id DESC`,
|
||
[team, status]
|
||
);
|
||
|
||
const { rows: noteRows } = await pool.query(`SELECT DISTINCT hostname FROM compliance_notes`);
|
||
const noteHostnames = new Set(noteRows.map(r => r.hostname));
|
||
const devices = groupByHostname(rows, noteHostnames);
|
||
res.json({ devices, team, status });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /items error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /items/:hostname
|
||
* Returns detailed information for a single device including all metrics and notes.
|
||
*
|
||
* @param hostname — the device hostname
|
||
* @response 200 { hostname, ip_address, device_type, team, metrics: Array<{ metric_id, metric_desc, category, status, ... }>, notes: Array<{ id, metric_id, note, group_id, created_at, created_by }> }
|
||
* @response 400 { error } — invalid hostname
|
||
* @response 404 { error } — device not found
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/items/:hostname', async (req, res) => {
|
||
const hostname = req.params.hostname;
|
||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||
|
||
try {
|
||
const { rows: metricRows } = await pool.query(
|
||
`SELECT DISTINCT ON (ci.metric_id, ci.status)
|
||
ci.metric_id, ci.metric_desc, ci.category, ci.status, ci.ip_address, ci.device_type, ci.team, ci.seen_count, ci.extra_json,
|
||
ci.resolution_date, ci.remediation_plan,
|
||
fu.report_date AS first_seen, fu.uploaded_at AS first_seen_at, lu.report_date AS last_seen, lu.uploaded_at AS last_seen_at, ru.report_date AS resolved_on
|
||
FROM compliance_items ci
|
||
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
||
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
||
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
||
WHERE ci.hostname = $1
|
||
ORDER BY ci.metric_id, ci.status, ci.seen_count DESC, ci.upload_id DESC`, [hostname]
|
||
);
|
||
if (metricRows.length === 0) return res.status(404).json({ error: 'Device not found' });
|
||
|
||
// Reproduce original ORDER BY ci.status DESC, ci.metric_id on the deduped rows
|
||
metricRows.sort((a, b) => {
|
||
if (a.status !== b.status) return b.status.localeCompare(a.status);
|
||
return a.metric_id.localeCompare(b.metric_id);
|
||
});
|
||
|
||
const metrics = metricRows.map(r => ({ ...r, extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(), extra_json: undefined }));
|
||
|
||
const { rows: notes } = await pool.query(
|
||
`SELECT cn.id, cn.metric_id, cn.note, cn.group_id, cn.created_at, u.username AS created_by
|
||
FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
|
||
WHERE cn.hostname = $1 ORDER BY cn.created_at DESC`, [hostname]
|
||
);
|
||
|
||
// Fetch remediation history
|
||
let history = [];
|
||
try {
|
||
const { rows: historyRows } = await pool.query(
|
||
`SELECT id, metric_id, field_name, old_value, new_value, change_reason, changed_by, changed_at
|
||
FROM compliance_item_history WHERE hostname = $1 ORDER BY changed_at DESC LIMIT 10`,
|
||
[hostname]
|
||
);
|
||
history = historyRows;
|
||
} catch (histErr) {
|
||
console.error('[Compliance] History fetch error:', histErr.message);
|
||
}
|
||
|
||
const identity = metricRows.find(r => r.status === 'active') || metricRows[0];
|
||
// Return resolution_date and remediation_plan from the first active item (or any item)
|
||
const resDate = identity.resolution_date ? (typeof identity.resolution_date === 'string' ? identity.resolution_date : identity.resolution_date.toISOString().slice(0, 10)) : null;
|
||
res.json({ hostname, ip_address: identity.ip_address || '', device_type: identity.device_type || '', team: identity.team || '', resolution_date: resDate, remediation_plan: identity.remediation_plan || '', metrics, notes, history });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /items/:hostname error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /notes
|
||
* Creates one or more compliance notes for a device, linked to specific metric IDs.
|
||
* All notes in a single call share a group_id for batch operations.
|
||
*
|
||
* @body { hostname: string, metric_id?: string, metric_ids?: string[], note: string }
|
||
* @response 201 { notes: Array<{ id, hostname, metric_id, note, group_id, created_at, created_by }> }
|
||
* @response 400 { error } — invalid hostname, missing/invalid metric_id(s), or empty note
|
||
* @response 500 { error } — save failure
|
||
*/
|
||
router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { hostname, metric_id, metric_ids, note } = req.body;
|
||
if (!hostname || typeof hostname !== 'string' || hostname.length > 300 || !/^[a-zA-Z0-9._-]+$/.test(hostname)) return res.status(400).json({ error: 'Invalid hostname format' });
|
||
|
||
let resolvedIds;
|
||
if (metric_ids !== undefined) {
|
||
if (!Array.isArray(metric_ids)) return res.status(400).json({ error: 'metric_ids must be an array' });
|
||
resolvedIds = metric_ids;
|
||
} else if (metric_id !== undefined && metric_id !== null && metric_id !== '') {
|
||
if (typeof metric_id !== 'string' || metric_id.length > 50) return res.status(400).json({ error: 'Invalid metric_id' });
|
||
resolvedIds = [metric_id];
|
||
} else {
|
||
return res.status(400).json({ error: 'metric_id or metric_ids is required' });
|
||
}
|
||
|
||
if (resolvedIds.length === 0) return res.status(400).json({ error: 'At least one metric ID is required' });
|
||
for (let i = 0; i < resolvedIds.length; i++) {
|
||
const mid = resolvedIds[i];
|
||
if (!mid || typeof mid !== 'string' || mid.length === 0 || mid.length > 50) return res.status(400).json({ error: `Invalid metric_id at index ${i}` });
|
||
}
|
||
|
||
const noteText = String(note || '').trim().slice(0, 1000);
|
||
if (!noteText) return res.status(400).json({ error: 'Note cannot be empty' });
|
||
|
||
const groupId = crypto.randomUUID();
|
||
const userId = req.user?.id || null;
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
const insertedIds = [];
|
||
for (const mid of resolvedIds) {
|
||
const { rows } = await client.query(
|
||
`INSERT INTO compliance_notes (hostname, metric_id, note, group_id, created_by, created_at) VALUES ($1, $2, $3, $4, $5, NOW()) RETURNING id`,
|
||
[hostname, mid, noteText, groupId, userId]
|
||
);
|
||
insertedIds.push(rows[0].id);
|
||
}
|
||
await client.query('COMMIT');
|
||
|
||
const { rows: notes } = await pool.query(
|
||
`SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.group_id, cn.created_at, u.username AS created_by
|
||
FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
|
||
WHERE cn.id = ANY($1) ORDER BY cn.id ASC`, [insertedIds]
|
||
);
|
||
res.status(201).json({ notes });
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
console.error('[Compliance] POST /notes error:', err.message);
|
||
res.status(500).json({ error: 'Failed to save note' });
|
||
} finally {
|
||
client.release();
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /notes/:hostname/:metricId
|
||
* Returns all notes for a specific device and metric combination.
|
||
*
|
||
* @param hostname — the device hostname
|
||
* @param metricId — the metric identifier
|
||
* @response 200 { notes: Array<{ id, note, created_at, created_by }> }
|
||
* @response 400 { error } — invalid hostname or metricId
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/notes/:hostname/:metricId', async (req, res) => {
|
||
const { hostname, metricId } = req.params;
|
||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||
if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metricId' });
|
||
|
||
try {
|
||
const { rows: notes } = await pool.query(
|
||
`SELECT cn.id, cn.note, cn.created_at, u.username AS created_by
|
||
FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
|
||
WHERE cn.hostname = $1 AND cn.metric_id = $2 ORDER BY cn.created_at DESC`, [hostname, metricId]
|
||
);
|
||
res.json({ notes });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /notes error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* DELETE /notes/:id
|
||
* Deletes a compliance note by ID. Only the note author or an Admin can delete.
|
||
* Optionally deletes all notes in the same group when ?group=true.
|
||
*
|
||
* @param id — numeric note ID
|
||
* @query group — optional, "true" to delete all notes sharing the same group_id
|
||
* @response 200 { deleted: number }
|
||
* @response 400 { error } — invalid note ID
|
||
* @response 403 { error } — not the author and not Admin
|
||
* @response 404 { error } — note not found
|
||
* @response 500 { error } — delete failure
|
||
*/
|
||
router.delete('/notes/:id', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const noteId = parseInt(req.params.id, 10);
|
||
if (isNaN(noteId)) return res.status(400).json({ error: 'Invalid note ID' });
|
||
const deleteGroup = req.query.group === 'true';
|
||
|
||
try {
|
||
const { rows } = await pool.query(`SELECT id, hostname, metric_id, note, group_id, created_by FROM compliance_notes WHERE id = $1`, [noteId]);
|
||
const noteRow = rows[0];
|
||
if (!noteRow) return res.status(404).json({ error: 'Note not found' });
|
||
|
||
const isAuthor = req.user && String(req.user.id) === String(noteRow.created_by);
|
||
const isAdminUser = req.user && req.user.group === 'Admin';
|
||
if (!isAuthor && !isAdminUser) return res.status(403).json({ error: 'You can only delete your own notes' });
|
||
|
||
let deleted = 0;
|
||
if (deleteGroup && noteRow.group_id) {
|
||
const result = await pool.query(`DELETE FROM compliance_notes WHERE group_id = $1`, [noteRow.group_id]);
|
||
deleted = result.rowCount;
|
||
} else {
|
||
const result = await pool.query(`DELETE FROM compliance_notes WHERE id = $1`, [noteId]);
|
||
deleted = result.rowCount;
|
||
}
|
||
|
||
logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_note_delete', entityType: 'compliance_note', entityId: String(noteId), details: JSON.stringify({ hostname: noteRow.hostname, group_id: noteRow.group_id, deleted_count: deleted }), ipAddress: req.ip });
|
||
res.json({ deleted });
|
||
} catch (err) {
|
||
console.error('[Compliance] DELETE /notes error:', err.message);
|
||
res.status(500).json({ error: 'Failed to delete note' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /trends
|
||
* Returns historical compliance upload trends with per-team breakdowns for charting.
|
||
*
|
||
* @response 200 { trends: Array<{ report_date, new_count, recurring_count, resolved_count, total_active, STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/trends', async (req, res) => {
|
||
try {
|
||
const { rows: uploads } = await pool.query(
|
||
`SELECT report_date,
|
||
SUM(COALESCE(new_count, 0))::int AS new_count,
|
||
SUM(COALESCE(recurring_count, 0))::int AS recurring_count,
|
||
SUM(COALESCE(resolved_count, 0))::int AS resolved_count,
|
||
SUM(COALESCE(new_count, 0) + COALESCE(recurring_count, 0))::int AS total_active
|
||
FROM compliance_uploads
|
||
WHERE report_date IS NOT NULL
|
||
GROUP BY report_date
|
||
ORDER BY report_date ASC`
|
||
);
|
||
if (uploads.length === 0) return res.json({ trends: [] });
|
||
|
||
const { rows: teamRows } = await pool.query(
|
||
`SELECT cu.report_date, ci.team, COUNT(ci.id)::int AS count
|
||
FROM compliance_items ci
|
||
JOIN compliance_uploads cu ON ci.upload_id = cu.id
|
||
WHERE ci.team IS NOT NULL AND cu.report_date IS NOT NULL
|
||
GROUP BY cu.report_date, ci.team`
|
||
);
|
||
const teamMap = {};
|
||
teamRows.forEach(r => { if (!teamMap[r.report_date]) teamMap[r.report_date] = {}; teamMap[r.report_date][r.team] = r.count; });
|
||
|
||
const trends = uploads.map(u => ({
|
||
report_date: u.report_date, new_count: u.new_count, recurring_count: u.recurring_count, resolved_count: u.resolved_count, total_active: u.total_active,
|
||
STEAM: teamMap[u.report_date]?.STEAM || 0, 'ACCESS-ENG': teamMap[u.report_date]?.['ACCESS-ENG'] || 0, 'ACCESS-OPS': teamMap[u.report_date]?.['ACCESS-OPS'] || 0, INTELDEV: teamMap[u.report_date]?.INTELDEV || 0,
|
||
}));
|
||
res.json({ trends });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /trends error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /mttr
|
||
* Returns aging bucket distribution of active compliance items (1 cycle, 2–3, 4–6, 7+) with per-team counts.
|
||
*
|
||
* @response 200 { aging: Array<{ bucket, total, STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/mttr', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT DISTINCT ON (hostname, metric_id)
|
||
COALESCE(seen_count, 1) AS seen_count, team
|
||
FROM compliance_items
|
||
WHERE status = 'active'
|
||
ORDER BY hostname, metric_id, seen_count DESC, upload_id DESC`
|
||
);
|
||
if (rows.length === 0) return res.json({ aging: [] });
|
||
const aging = bucketAgingItems(rows);
|
||
res.json({ aging });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /mttr error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /top-recurring
|
||
* Returns waterfall chart data computed from upload history (start, new, recurring, resolved, end per upload).
|
||
*
|
||
* @response 200 { waterfall: Array<{ date, start, new_count, recurring_count, resolved_count, end }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/top-recurring', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT report_date,
|
||
SUM(COALESCE(new_count, 0))::int AS new_count,
|
||
SUM(COALESCE(recurring_count, 0))::int AS recurring_count,
|
||
SUM(COALESCE(resolved_count, 0))::int AS resolved_count
|
||
FROM compliance_uploads
|
||
WHERE report_date IS NOT NULL
|
||
GROUP BY report_date
|
||
ORDER BY report_date ASC`
|
||
);
|
||
const waterfall = computeWaterfall(rows);
|
||
res.json({ waterfall });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /top-recurring error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /category-trend
|
||
* Returns per-upload category breakdown counts for trend charting.
|
||
*
|
||
* @response 200 { categoryTrend: Array<{ report_date, category, count }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/category-trend', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT cu.report_date,
|
||
COALESCE(ci.category, 'Unknown') AS category,
|
||
COUNT(ci.id)::int AS count
|
||
FROM compliance_uploads cu JOIN compliance_items ci ON ci.upload_id = cu.id
|
||
WHERE cu.report_date IS NOT NULL
|
||
GROUP BY cu.report_date, COALESCE(ci.category, 'Unknown')
|
||
ORDER BY cu.report_date ASC, category ASC`
|
||
);
|
||
res.json({ categoryTrend: rows });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /category-trend error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* PATCH /items/:hostname/metadata
|
||
* Updates resolution_date and/or remediation_plan for active compliance items matching a hostname.
|
||
* Supports optional per-metric scoping via metric_id (single) or metric_ids (array).
|
||
* Records field-level change history in compliance_item_history for each modified field.
|
||
*
|
||
* @param hostname — the device hostname
|
||
* @body { resolution_date?: string|null, remediation_plan?: string|null, change_reason?: string|null, metric_id?: string, metric_ids?: string[] }
|
||
* @response 200 { updated: number }
|
||
* @response 400 { error } — invalid hostname, invalid date format, plan exceeds 2000 chars, change_reason exceeds 500 chars, no fields provided, or invalid metric_id
|
||
* @response 404 { error } — device not found
|
||
* @response 500 { error } — update failure
|
||
*/
|
||
router.patch('/items/:hostname/metadata', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const hostname = req.params.hostname;
|
||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||
|
||
const { resolution_date, remediation_plan, change_reason, metric_id, metric_ids } = req.body;
|
||
|
||
// Validate resolution_date: must be a valid ISO date string or null
|
||
if (resolution_date !== undefined && resolution_date !== null) {
|
||
if (!isValidDateString(resolution_date)) {
|
||
return res.status(400).json({ error: 'Invalid resolution_date format' });
|
||
}
|
||
}
|
||
|
||
// Validate remediation_plan: must be <= 2000 chars or null
|
||
if (remediation_plan !== undefined && remediation_plan !== null) {
|
||
const planValidation = validateRemediationPlan(remediation_plan);
|
||
if (!planValidation.valid) {
|
||
return res.status(400).json({ error: planValidation.error });
|
||
}
|
||
}
|
||
|
||
// Validate change_reason: optional, max 500 chars
|
||
if (change_reason !== undefined && change_reason !== null && change_reason.length > 500) {
|
||
return res.status(400).json({ error: 'Change reason exceeds 500 characters' });
|
||
}
|
||
|
||
// Resolve metric scoping: metric_ids takes precedence over metric_id
|
||
let resolvedMetricIds = null; // null means hostname-level (no metric scoping)
|
||
if (metric_ids !== undefined) {
|
||
if (!Array.isArray(metric_ids)) return res.status(400).json({ error: 'metric_ids must be an array' });
|
||
if (metric_ids.length === 0) return res.status(400).json({ error: 'metric_ids must contain at least one entry' });
|
||
for (let i = 0; i < metric_ids.length; i++) {
|
||
const mid = metric_ids[i];
|
||
if (!mid || typeof mid !== 'string' || mid.length === 0) {
|
||
return res.status(400).json({ error: 'metric_id cannot be empty' });
|
||
}
|
||
if (mid.length > 100) {
|
||
return res.status(400).json({ error: 'metric_id exceeds 100 characters' });
|
||
}
|
||
}
|
||
resolvedMetricIds = metric_ids;
|
||
} else if (metric_id !== undefined && metric_id !== null) {
|
||
if (typeof metric_id !== 'string' || metric_id.length === 0) {
|
||
return res.status(400).json({ error: 'metric_id cannot be empty' });
|
||
}
|
||
if (metric_id.length > 100) {
|
||
return res.status(400).json({ error: 'metric_id exceeds 100 characters' });
|
||
}
|
||
resolvedMetricIds = [metric_id];
|
||
}
|
||
|
||
const setClauses = [];
|
||
const values = [];
|
||
let paramIdx = 1;
|
||
|
||
if (resolution_date !== undefined) {
|
||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||
values.push(resolution_date);
|
||
}
|
||
if (remediation_plan !== undefined) {
|
||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||
values.push(remediation_plan);
|
||
}
|
||
|
||
if (setClauses.length === 0) {
|
||
return res.status(400).json({ error: 'No fields to update' });
|
||
}
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
const reasonText = change_reason && change_reason.trim() ? change_reason.trim() : null;
|
||
|
||
if (resolvedMetricIds !== null) {
|
||
// --- Per-metric scoping path ---
|
||
// Validate that each metric_id corresponds to an active compliance_item for this hostname
|
||
const { rows: activeMetricRows } = await client.query(
|
||
`SELECT metric_id, resolution_date, remediation_plan
|
||
FROM compliance_items
|
||
WHERE hostname = $1 AND metric_id = ANY($2) AND status = 'active'`,
|
||
[hostname, resolvedMetricIds]
|
||
);
|
||
|
||
const activeMetricMap = new Map();
|
||
for (const row of activeMetricRows) {
|
||
activeMetricMap.set(row.metric_id, row);
|
||
}
|
||
|
||
// Check for invalid metric_ids
|
||
for (const mid of resolvedMetricIds) {
|
||
if (!activeMetricMap.has(mid)) {
|
||
await client.query('ROLLBACK');
|
||
client.release();
|
||
return res.status(400).json({ error: `Invalid metric_id: ${mid} — no active compliance item found` });
|
||
}
|
||
}
|
||
|
||
// Insert history per metric per changed field
|
||
for (const mid of resolvedMetricIds) {
|
||
const current = activeMetricMap.get(mid);
|
||
const currentResDate = current.resolution_date
|
||
? (typeof current.resolution_date === 'string' ? current.resolution_date : current.resolution_date.toISOString().slice(0, 10))
|
||
: null;
|
||
const currentPlan = current.remediation_plan || null;
|
||
|
||
if (resolution_date !== undefined) {
|
||
const newVal = resolution_date || null;
|
||
if (currentResDate !== newVal) {
|
||
await client.query(
|
||
`INSERT INTO compliance_item_history (hostname, metric_id, field_name, old_value, new_value, change_reason, changed_by)
|
||
VALUES ($1, $2, 'resolution_date', $3, $4, $5, $6)`,
|
||
[hostname, mid, currentResDate, newVal, reasonText, req.user.username]
|
||
);
|
||
}
|
||
}
|
||
if (remediation_plan !== undefined) {
|
||
const newVal = remediation_plan || null;
|
||
if (currentPlan !== newVal) {
|
||
await client.query(
|
||
`INSERT INTO compliance_item_history (hostname, metric_id, field_name, old_value, new_value, change_reason, changed_by)
|
||
VALUES ($1, $2, 'remediation_plan', $3, $4, $5, $6)`,
|
||
[hostname, mid, currentPlan, newVal, reasonText, req.user.username]
|
||
);
|
||
}
|
||
}
|
||
}
|
||
|
||
// Update only matching rows
|
||
values.push(hostname);
|
||
values.push(resolvedMetricIds);
|
||
const result = await client.query(
|
||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND metric_id = ANY($${paramIdx + 1}) AND status = 'active'`,
|
||
values
|
||
);
|
||
|
||
await client.query('COMMIT');
|
||
|
||
logAudit({
|
||
userId: req.user.id,
|
||
username: req.user.username,
|
||
action: 'compliance_metadata_update',
|
||
entityType: 'compliance_item',
|
||
entityId: hostname,
|
||
details: { resolution_date, remediation_plan, change_reason: reasonText, metric_ids: resolvedMetricIds },
|
||
ipAddress: req.ip,
|
||
});
|
||
|
||
res.json({ updated: result.rowCount });
|
||
} else {
|
||
// --- Hostname-level path (backward compatible, NULL metric_id in history) ---
|
||
// Get current values before updating (pick one representative row)
|
||
const { rows: currentRows } = await client.query(
|
||
`SELECT DISTINCT ON (hostname) resolution_date, remediation_plan
|
||
FROM compliance_items WHERE hostname = $1 AND status = 'active'
|
||
ORDER BY hostname, id DESC LIMIT 1`,
|
||
[hostname]
|
||
);
|
||
|
||
if (currentRows.length === 0) {
|
||
await client.query('ROLLBACK');
|
||
client.release();
|
||
return res.status(404).json({ error: 'Device not found' });
|
||
}
|
||
|
||
const current = currentRows[0];
|
||
const currentResDate = current.resolution_date
|
||
? (typeof current.resolution_date === 'string' ? current.resolution_date : current.resolution_date.toISOString().slice(0, 10))
|
||
: null;
|
||
const currentPlan = current.remediation_plan || null;
|
||
|
||
// Insert history for each changed field with NULL metric_id
|
||
if (resolution_date !== undefined) {
|
||
const newVal = resolution_date || null;
|
||
if (currentResDate !== newVal) {
|
||
await client.query(
|
||
`INSERT INTO compliance_item_history (hostname, metric_id, field_name, old_value, new_value, change_reason, changed_by)
|
||
VALUES ($1, NULL, 'resolution_date', $2, $3, $4, $5)`,
|
||
[hostname, currentResDate, newVal, reasonText, req.user.username]
|
||
);
|
||
}
|
||
}
|
||
if (remediation_plan !== undefined) {
|
||
const newVal = remediation_plan || null;
|
||
if (currentPlan !== newVal) {
|
||
await client.query(
|
||
`INSERT INTO compliance_item_history (hostname, metric_id, field_name, old_value, new_value, change_reason, changed_by)
|
||
VALUES ($1, NULL, 'remediation_plan', $2, $3, $4, $5)`,
|
||
[hostname, currentPlan, newVal, reasonText, req.user.username]
|
||
);
|
||
}
|
||
}
|
||
|
||
// Update all active items for hostname
|
||
values.push(hostname);
|
||
const result = await client.query(
|
||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||
values
|
||
);
|
||
|
||
await client.query('COMMIT');
|
||
|
||
logAudit({
|
||
userId: req.user.id,
|
||
username: req.user.username,
|
||
action: 'compliance_metadata_update',
|
||
entityType: 'compliance_item',
|
||
entityId: hostname,
|
||
details: { resolution_date, remediation_plan, change_reason: reasonText },
|
||
ipAddress: req.ip,
|
||
});
|
||
|
||
res.json({ updated: result.rowCount });
|
||
}
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
console.error('[Compliance] PATCH /items/:hostname/metadata error:', err.message);
|
||
res.status(500).json({ error: 'Failed to update device metadata' });
|
||
} finally {
|
||
client.release();
|
||
}
|
||
});
|
||
|
||
const VCL_TARGET_PCT = parseInt(process.env.VCL_TARGET_PCT, 10) || 95;
|
||
|
||
/**
|
||
* GET /vcl/stats
|
||
* Returns VCL executive summary statistics including device counts, compliance percentage,
|
||
* non-compliant asset categorization (donut), heavy hitters by team, and vertical breakdown with burndown.
|
||
*
|
||
* @response 200 { stats: { total_devices, in_scope, compliant, non_compliant, remediations_required, compliance_pct, target_pct }, donut: { blocked: { count, pct }, in_progress: { count, pct } }, heavy_hitters: Array<{ vertical, team, non_compliant, compliance_date, notes }>, vertical_breakdown: Array<{ vertical, compliance_pct, team, non_compliant, actual_burndown, forecast_burndown, blockers, risk_acceptances, notes }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/vcl/stats', async (req, res) => {
|
||
try {
|
||
// Compute device-level stats using DISTINCT hostname
|
||
// A device is "compliant" if it has NO active findings
|
||
const { rows: statsRows } = await pool.query(`
|
||
SELECT
|
||
COUNT(DISTINCT hostname) AS total_devices,
|
||
COUNT(DISTINCT hostname) AS in_scope,
|
||
COUNT(DISTINCT CASE
|
||
WHEN hostname NOT IN (SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active')
|
||
THEN hostname END) AS compliant,
|
||
COUNT(DISTINCT CASE
|
||
WHEN hostname IN (SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active')
|
||
THEN hostname END) AS non_compliant
|
||
FROM compliance_items
|
||
`);
|
||
|
||
const raw = statsRows[0] || {};
|
||
const total_devices = parseInt(raw.total_devices) || 0;
|
||
const in_scope = parseInt(raw.in_scope) || 0;
|
||
const compliant = parseInt(raw.compliant) || 0;
|
||
const non_compliant = parseInt(raw.non_compliant) || 0;
|
||
const compliance_pct = in_scope > 0 ? Math.round((compliant / in_scope) * 100) : 0;
|
||
|
||
const stats = {
|
||
total_devices,
|
||
in_scope,
|
||
compliant,
|
||
non_compliant,
|
||
remediations_required: non_compliant,
|
||
compliance_pct,
|
||
target_pct: VCL_TARGET_PCT,
|
||
};
|
||
|
||
// Donut: categorize non-compliant DEVICES by resolution_date presence
|
||
// A device is "blocked" if it has no resolution_date on any of its active findings
|
||
// A device is "in_progress" if at least one active finding has a resolution_date
|
||
const { rows: donutRows } = await pool.query(`
|
||
SELECT
|
||
hostname,
|
||
MAX(resolution_date) AS resolution_date
|
||
FROM compliance_items
|
||
WHERE status = 'active'
|
||
GROUP BY hostname
|
||
`);
|
||
const donut = categorizeNonCompliant(donutRows);
|
||
|
||
// Heavy hitters: group by team, count non-compliant DEVICES per team
|
||
// CTE deduplicates hostnames to one team via representative row (highest seen_count, most recent upload_id)
|
||
const { rows: teamRows } = await pool.query(`
|
||
WITH device_team AS (
|
||
SELECT DISTINCT ON (hostname)
|
||
hostname,
|
||
COALESCE(team, 'Unknown') AS team,
|
||
resolution_date
|
||
FROM compliance_items
|
||
WHERE status = 'active'
|
||
ORDER BY hostname, seen_count DESC, upload_id DESC
|
||
)
|
||
SELECT team,
|
||
COUNT(DISTINCT hostname)::int AS non_compliant,
|
||
MAX(resolution_date) AS compliance_date
|
||
FROM device_team
|
||
GROUP BY team
|
||
ORDER BY COUNT(DISTINCT hostname) DESC
|
||
`);
|
||
const heavy_hitters = teamRows.map(r => ({
|
||
vertical: r.team,
|
||
team: r.team,
|
||
non_compliant: parseInt(r.non_compliant),
|
||
compliance_date: r.compliance_date ? r.compliance_date.toISOString().slice(0, 10) : null,
|
||
notes: '',
|
||
}));
|
||
|
||
// Vertical breakdown with burndown
|
||
const verticalBreakdown = [];
|
||
for (const teamRow of teamRows) {
|
||
const team = teamRow.team;
|
||
const teamNonCompliant = parseInt(teamRow.non_compliant);
|
||
|
||
// Get total devices for this team (all statuses) — CTE deduplicates hostnames to one team
|
||
const { rows: teamTotalRows } = await pool.query(
|
||
`WITH device_team AS (
|
||
SELECT DISTINCT ON (hostname)
|
||
hostname,
|
||
COALESCE(team, 'Unknown') AS team
|
||
FROM compliance_items
|
||
ORDER BY hostname, seen_count DESC, upload_id DESC
|
||
)
|
||
SELECT COUNT(*)::int AS total FROM device_team WHERE team = $1`,
|
||
[team]
|
||
);
|
||
const teamTotal = parseInt(teamTotalRows[0]?.total) || 0;
|
||
const teamCompliant = teamTotal - teamNonCompliant;
|
||
const compliance_pct_team = teamTotal > 0 ? Math.round((teamCompliant / teamTotal) * 100) : 0;
|
||
|
||
// Forecast burndown from resolution_dates — DISTINCT ON deduplicates cross-vertical (hostname, metric_id) pairs
|
||
const { rows: forecastItems } = await pool.query(
|
||
`SELECT DISTINCT ON (hostname, metric_id) resolution_date
|
||
FROM compliance_items
|
||
WHERE status = 'active' AND COALESCE(team, 'Unknown') = $1 AND resolution_date IS NOT NULL
|
||
ORDER BY hostname, metric_id, seen_count DESC, upload_id DESC`,
|
||
[team]
|
||
);
|
||
const forecast_burndown = computeForecastBurndown(forecastItems);
|
||
const blockers = teamNonCompliant - forecastItems.length;
|
||
|
||
verticalBreakdown.push({
|
||
vertical: team,
|
||
compliance_pct: compliance_pct_team,
|
||
team: team,
|
||
non_compliant: teamNonCompliant,
|
||
actual_burndown: {},
|
||
forecast_burndown,
|
||
blockers: blockers > 0 ? blockers : 0,
|
||
risk_acceptances: 0,
|
||
notes: '',
|
||
});
|
||
}
|
||
|
||
// Merge vertical metadata (notes, risk_acceptances, compliance_date)
|
||
try {
|
||
const { rows: metaRows } = await pool.query(`SELECT team, notes, risk_acceptances, compliance_date FROM vcl_vertical_metadata`);
|
||
const metaMap = {};
|
||
metaRows.forEach(r => { metaMap[r.team] = r; });
|
||
|
||
for (const hh of heavy_hitters) {
|
||
const meta = metaMap[hh.vertical] || metaMap[hh.team];
|
||
if (meta) {
|
||
hh.notes = meta.notes || '';
|
||
hh.compliance_date = meta.compliance_date || hh.compliance_date;
|
||
}
|
||
}
|
||
for (const vb of verticalBreakdown) {
|
||
const meta = metaMap[vb.vertical] || metaMap[vb.team];
|
||
if (meta) {
|
||
vb.notes = meta.notes || '';
|
||
vb.risk_acceptances = meta.risk_acceptances || 0;
|
||
vb.compliance_date = meta.compliance_date || null;
|
||
}
|
||
}
|
||
} catch (metaErr) {
|
||
// Non-critical — continue without metadata
|
||
console.error('[Compliance] VCL metadata merge error:', metaErr.message);
|
||
}
|
||
|
||
res.json({ stats, donut, heavy_hitters, vertical_breakdown: verticalBreakdown });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /vcl/stats error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /vcl/trend
|
||
* Returns monthly compliance trend data with actual percentages and linear regression forecast.
|
||
* Forecast is computed when 3+ months of historical data exist, projecting 3 months forward.
|
||
*
|
||
* @response 200 { months: Array<{ month, compliant_count, compliance_pct, forecast_pct, target_pct }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/vcl/trend', async (req, res) => {
|
||
try {
|
||
const { rows: snapshots } = await pool.query(
|
||
`SELECT snapshot_month, SUM(compliant)::int AS compliant_count,
|
||
CASE WHEN SUM(total_devices) > 0
|
||
THEN ROUND((SUM(compliant)::numeric / SUM(total_devices)::numeric) * 100, 1)
|
||
ELSE 0 END AS compliance_pct
|
||
FROM compliance_snapshots
|
||
GROUP BY snapshot_month
|
||
ORDER BY snapshot_month ASC`
|
||
);
|
||
|
||
// Build months array with actuals
|
||
const months = snapshots.map(s => ({
|
||
month: s.snapshot_month,
|
||
compliant_count: s.compliant_count,
|
||
compliance_pct: parseFloat(s.compliance_pct),
|
||
forecast_pct: null,
|
||
target_pct: VCL_TARGET_PCT,
|
||
}));
|
||
|
||
// Compute forecast using linear regression if we have 3+ months
|
||
if (months.length >= 3) {
|
||
const n = months.length;
|
||
// Use last data points for regression
|
||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
|
||
for (let i = 0; i < n; i++) {
|
||
sumX += i;
|
||
sumY += months[i].compliance_pct;
|
||
sumXY += i * months[i].compliance_pct;
|
||
sumX2 += i * i;
|
||
}
|
||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
|
||
const intercept = (sumY - slope * sumX) / n;
|
||
|
||
// Project forward 3 months
|
||
for (let i = 0; i < 3; i++) {
|
||
const futureIdx = n + i;
|
||
const forecastPct = Math.min(100, Math.max(0, Math.round((slope * futureIdx + intercept) * 10) / 10));
|
||
|
||
// Compute the future month string
|
||
const lastMonth = months[months.length - 1].month;
|
||
const [year, mon] = lastMonth.split('-').map(Number);
|
||
const futureDate = new Date(year, mon - 1 + i + 1, 1);
|
||
const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`;
|
||
|
||
months.push({
|
||
month: futureMonth,
|
||
compliant_count: null,
|
||
compliance_pct: null,
|
||
forecast_pct: forecastPct,
|
||
target_pct: VCL_TARGET_PCT,
|
||
});
|
||
}
|
||
|
||
// Also add forecast_pct to the last actual month as the starting point
|
||
if (months.length > 0 && n > 0) {
|
||
months[n - 1].forecast_pct = months[n - 1].compliance_pct;
|
||
}
|
||
}
|
||
|
||
res.json({ months });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /vcl/trend error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /vcl/bulk-preview
|
||
* Accepts parsed bulk upload rows, matches hostnames against active devices, validates fields,
|
||
* and returns a diff preview showing matched/unmatched/changed/invalid row counts.
|
||
*
|
||
* @body { rows: Array<{ hostname, resolution_date?, remediation_plan?, notes? }>, headers?: string[] }
|
||
* @response 200 { matched, unmatched, changes, invalid, details: Array<{ hostname, status, fields? }>, unmatched_rows: string[], invalid_rows: Array<{ hostname, errors }> }
|
||
* @response 400 { error } — missing rows, exceeds 2000 rows, no Hostname column, or no updatable fields
|
||
* @response 500 { error } — processing failure
|
||
*/
|
||
router.post('/vcl/bulk-preview', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { rows, headers } = req.body;
|
||
|
||
// Validate: require rows array
|
||
if (!rows || !Array.isArray(rows)) {
|
||
return res.status(400).json({ error: 'rows array is required' });
|
||
}
|
||
|
||
// Enforce 2000 row limit
|
||
if (rows.length === 0) {
|
||
return res.status(400).json({ error: 'File contains no data rows' });
|
||
}
|
||
if (rows.length > 2000) {
|
||
return res.status(400).json({ error: 'File exceeds maximum of 2000 rows' });
|
||
}
|
||
|
||
// Map column headers if provided
|
||
let columnMapping = {};
|
||
if (headers && Array.isArray(headers)) {
|
||
columnMapping = mapColumnHeaders(headers);
|
||
}
|
||
|
||
// Require hostname field
|
||
const hasHostname = rows.every(r => r.hostname != null && r.hostname !== '');
|
||
if (!hasHostname) {
|
||
return res.status(400).json({ error: 'File must contain a Hostname column' });
|
||
}
|
||
|
||
// Check for updatable fields (resolution_date, remediation_plan, or notes)
|
||
const sampleRow = rows[0] || {};
|
||
const updatableFields = ['resolution_date', 'remediation_plan', 'notes'];
|
||
const hasUpdatableFields = updatableFields.some(f => f in sampleRow);
|
||
if (!hasUpdatableFields && headers) {
|
||
// Check via column mapping
|
||
const mappedFields = Object.keys(columnMapping).filter(k => k !== 'hostname');
|
||
if (mappedFields.length === 0) {
|
||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||
}
|
||
} else if (!hasUpdatableFields && !headers) {
|
||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||
}
|
||
|
||
try {
|
||
// Get existing hostnames from DB
|
||
const { rows: existingRows } = await pool.query(
|
||
`SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active'`
|
||
);
|
||
const existingHostnames = new Set(existingRows.map(r => r.hostname));
|
||
|
||
// Match by hostname
|
||
const { matched, unmatched } = matchByHostname(rows, existingHostnames);
|
||
|
||
// Validate fields on matched rows
|
||
const validRows = [];
|
||
const invalidRows = [];
|
||
|
||
for (const row of matched) {
|
||
const errors = [];
|
||
|
||
if (row.resolution_date !== undefined && row.resolution_date !== null && row.resolution_date !== '') {
|
||
if (!isValidDateString(row.resolution_date)) {
|
||
errors.push('resolution_date: invalid date format');
|
||
}
|
||
}
|
||
|
||
if (row.remediation_plan !== undefined && row.remediation_plan !== null) {
|
||
const planCheck = validateRemediationPlan(row.remediation_plan);
|
||
if (!planCheck.valid) {
|
||
errors.push('remediation_plan: ' + planCheck.error);
|
||
}
|
||
}
|
||
|
||
if (errors.length > 0) {
|
||
invalidRows.push({ hostname: row.hostname, errors });
|
||
} else {
|
||
validRows.push(row);
|
||
}
|
||
}
|
||
|
||
// Get current data for diff computation
|
||
const { rows: currentRows } = await pool.query(
|
||
`SELECT DISTINCT ON (hostname) hostname, resolution_date, remediation_plan
|
||
FROM compliance_items WHERE status = 'active' AND hostname = ANY($1)
|
||
ORDER BY hostname, id DESC`,
|
||
[validRows.map(r => r.hostname)]
|
||
);
|
||
const currentData = new Map();
|
||
for (const row of currentRows) {
|
||
currentData.set(row.hostname, {
|
||
resolution_date: row.resolution_date ? row.resolution_date.toISOString?.().slice(0, 10) || String(row.resolution_date).slice(0, 10) : null,
|
||
remediation_plan: row.remediation_plan || null,
|
||
notes: null,
|
||
});
|
||
}
|
||
|
||
// Compute diff
|
||
const diffResults = computeBulkDiff(validRows, currentData);
|
||
const changedRows = diffResults.filter(r => r.status === 'changed');
|
||
|
||
res.json({
|
||
matched: matched.length,
|
||
unmatched: unmatched.length,
|
||
changes: changedRows.length,
|
||
invalid: invalidRows.length,
|
||
details: diffResults,
|
||
unmatched_rows: unmatched.map(r => r.hostname),
|
||
invalid_rows: invalidRows,
|
||
});
|
||
} catch (err) {
|
||
console.error('[Compliance] POST /vcl/bulk-preview error:', err.message);
|
||
res.status(500).json({ error: 'Failed to process bulk preview' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* POST /vcl/bulk-commit
|
||
* Commits validated bulk changes to compliance items in a single transaction.
|
||
* Updates resolution_date and/or remediation_plan for each hostname provided.
|
||
*
|
||
* @body { changes: Array<{ hostname, resolution_date?, remediation_plan?, notes? }> }
|
||
* @response 200 { committed: number }
|
||
* @response 400 { error } — missing or empty changes array
|
||
* @response 500 { error } — transaction failure (full rollback)
|
||
*/
|
||
router.post('/vcl/bulk-commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { changes } = req.body;
|
||
|
||
if (!changes || !Array.isArray(changes) || changes.length === 0) {
|
||
return res.status(400).json({ error: 'changes array is required' });
|
||
}
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
// Pre-fetch current values for all hostnames in the batch
|
||
const hostnames = changes.map(c => c.hostname);
|
||
const { rows: currentRows } = await client.query(
|
||
`SELECT DISTINCT ON (hostname) hostname, resolution_date, remediation_plan
|
||
FROM compliance_items WHERE status = 'active' AND hostname = ANY($1)
|
||
ORDER BY hostname, id DESC`,
|
||
[hostnames]
|
||
);
|
||
const currentData = new Map();
|
||
for (const row of currentRows) {
|
||
currentData.set(row.hostname, {
|
||
resolution_date: row.resolution_date ? (typeof row.resolution_date === 'string' ? row.resolution_date : row.resolution_date.toISOString().slice(0, 10)) : null,
|
||
remediation_plan: row.remediation_plan || null,
|
||
});
|
||
}
|
||
|
||
let committedCount = 0;
|
||
for (const change of changes) {
|
||
const setClauses = [];
|
||
const values = [];
|
||
let paramIdx = 1;
|
||
|
||
if (change.resolution_date !== undefined) {
|
||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||
values.push(change.resolution_date);
|
||
}
|
||
if (change.remediation_plan !== undefined) {
|
||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||
values.push(change.remediation_plan);
|
||
}
|
||
if (change.notes !== undefined) {
|
||
// Notes are stored separately in compliance_notes, but we can update a field if it exists
|
||
// For now, skip notes in the direct update
|
||
}
|
||
|
||
if (setClauses.length === 0) continue;
|
||
|
||
// Record history for changed fields
|
||
const current = currentData.get(change.hostname) || { resolution_date: null, remediation_plan: null };
|
||
if (change.resolution_date !== undefined) {
|
||
const newVal = change.resolution_date || null;
|
||
if (current.resolution_date !== newVal) {
|
||
await client.query(
|
||
`INSERT INTO compliance_item_history (hostname, field_name, old_value, new_value, change_reason, changed_by)
|
||
VALUES ($1, 'resolution_date', $2, $3, NULL, $4)`,
|
||
[change.hostname, current.resolution_date, newVal, req.user.username]
|
||
);
|
||
}
|
||
}
|
||
if (change.remediation_plan !== undefined) {
|
||
const newVal = change.remediation_plan || null;
|
||
if (current.remediation_plan !== newVal) {
|
||
await client.query(
|
||
`INSERT INTO compliance_item_history (hostname, field_name, old_value, new_value, change_reason, changed_by)
|
||
VALUES ($1, 'remediation_plan', $2, $3, NULL, $4)`,
|
||
[change.hostname, current.remediation_plan, newVal, req.user.username]
|
||
);
|
||
}
|
||
}
|
||
|
||
values.push(change.hostname);
|
||
const result = await client.query(
|
||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||
values
|
||
);
|
||
if (result.rowCount > 0) committedCount++;
|
||
}
|
||
|
||
await client.query('COMMIT');
|
||
|
||
logAudit({
|
||
userId: req.user.id,
|
||
username: req.user.username,
|
||
action: 'compliance_bulk_update',
|
||
entityType: 'compliance_items',
|
||
entityId: null,
|
||
details: { rows_updated: committedCount, total_changes: changes.length },
|
||
ipAddress: req.ip,
|
||
});
|
||
|
||
res.json({ committed: committedCount });
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
console.error('[Compliance] POST /vcl/bulk-commit error:', err.message);
|
||
res.status(500).json({ error: 'Failed to commit changes' });
|
||
} finally {
|
||
client.release();
|
||
}
|
||
});
|
||
|
||
// -----------------------------------------------------------------------
|
||
// VCL Vertical Metadata endpoints
|
||
// -----------------------------------------------------------------------
|
||
|
||
/**
|
||
* GET /vcl/vertical-metadata
|
||
* Returns all rows from vcl_vertical_metadata.
|
||
*
|
||
* @response 200 { metadata: Array<{ id, team, notes, risk_acceptances, compliance_date, updated_at }> }
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.get('/vcl/vertical-metadata', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT id, team, notes, risk_acceptances, compliance_date, updated_at FROM vcl_vertical_metadata ORDER BY team`
|
||
);
|
||
res.json({ metadata: rows });
|
||
} catch (err) {
|
||
console.error('[Compliance] GET /vcl/vertical-metadata error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* PATCH /vcl/vertical-metadata/:team
|
||
* Upserts notes, risk_acceptances, and/or compliance_date for a team.
|
||
*
|
||
* @param team — the team/vertical name
|
||
* @body { notes?: string, risk_acceptances?: number, compliance_date?: string|null }
|
||
* @response 200 { success: true }
|
||
* @response 400 { error } — no fields provided or invalid values
|
||
* @response 500 { error } — database error
|
||
*/
|
||
router.patch('/vcl/vertical-metadata/:team', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const team = req.params.team;
|
||
if (!team || team.length > 100) return res.status(400).json({ error: 'Invalid team' });
|
||
|
||
const { notes, risk_acceptances, compliance_date } = req.body;
|
||
|
||
if (notes === undefined && risk_acceptances === undefined && compliance_date === undefined) {
|
||
return res.status(400).json({ error: 'No fields to update' });
|
||
}
|
||
|
||
if (risk_acceptances !== undefined && risk_acceptances !== null) {
|
||
if (typeof risk_acceptances !== 'number' || risk_acceptances < 0 || !Number.isInteger(risk_acceptances)) {
|
||
return res.status(400).json({ error: 'risk_acceptances must be a non-negative integer' });
|
||
}
|
||
}
|
||
|
||
if (compliance_date !== undefined && compliance_date !== null && compliance_date !== '') {
|
||
if (typeof compliance_date !== 'string' || compliance_date.length > 50) {
|
||
return res.status(400).json({ error: 'compliance_date must be a string (max 50 chars)' });
|
||
}
|
||
}
|
||
|
||
try {
|
||
// Build the upsert dynamically
|
||
const upsertNotes = notes !== undefined ? notes : '';
|
||
const upsertRAs = risk_acceptances !== undefined ? risk_acceptances : 0;
|
||
const upsertDate = compliance_date !== undefined ? (compliance_date || null) : null;
|
||
|
||
// Use ON CONFLICT to insert or update only the provided fields
|
||
const updateParts = [];
|
||
if (notes !== undefined) updateParts.push('notes = EXCLUDED.notes');
|
||
if (risk_acceptances !== undefined) updateParts.push('risk_acceptances = EXCLUDED.risk_acceptances');
|
||
if (compliance_date !== undefined) updateParts.push('compliance_date = EXCLUDED.compliance_date');
|
||
updateParts.push('updated_at = NOW()');
|
||
|
||
await pool.query(
|
||
`INSERT INTO vcl_vertical_metadata (team, notes, risk_acceptances, compliance_date, updated_at)
|
||
VALUES ($1, $2, $3, $4, NOW())
|
||
ON CONFLICT (team) DO UPDATE SET ${updateParts.join(', ')}`,
|
||
[team, upsertNotes, upsertRAs, upsertDate]
|
||
);
|
||
|
||
res.json({ success: true });
|
||
} catch (err) {
|
||
console.error('[Compliance] PATCH /vcl/vertical-metadata error:', err.message);
|
||
res.status(500).json({ error: 'Failed to update vertical metadata' });
|
||
}
|
||
});
|
||
|
||
return router;
|
||
}
|
||
|
||
module.exports = { createComplianceRouter, bucketAgingItems, computeWaterfall, persistUpload, groupByHostname };
|