Files
cve-dashboard/backend/routes/compliance.js

1144 lines
49 KiB
JavaScript
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Compliance Routes — AEO metric tracking
// Handles xlsx upload/parse, non-compliant item history, and notes.
//
// Endpoints:
// POST /preview — parse xlsx, run drift check, compute diff (no DB write)
// POST /reconcile-config — patch compliance_config.json to resolve drift findings
// POST /commit — commit a previewed upload to DB
// GET /uploads — list all uploads
// POST /rollback/:uploadId — roll back the most recent upload (Admin only)
// GET /summary — metric health cards for a team (from latest upload)
// GET /items — non-compliant devices grouped by hostname (?team=X&status=active)
// GET /items/:hostname — detail panel: all metrics + notes + upload history for a device
// POST /notes — add a note to one or more (hostname, metric_id) pairs
// GET /notes/:hostname/:metricId — notes for a specific device+metric
// GET /trends — per-upload totals + per-team counts for time-series charts
// GET /mttr — aging findings distribution by seen_count bucket and team
// GET /top-recurring — net change waterfall (per-cycle start/new/recurring/resolved/end)
// GET /category-trend — active counts per category per upload for stacked area chart
const express = require('express');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
const { spawn } = require('child_process');
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
const logAudit = require('../helpers/auditLog');
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
const SCHEMA_SCRIPT = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
const CONFIG_PATH = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
// ---------------------------------------------------------------------------
// DB helpers
// ---------------------------------------------------------------------------
function dbRun(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.run(sql, params, function (err) {
if (err) reject(err);
else resolve({ lastID: this.lastID, changes: this.changes });
});
});
}
function dbGet(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.get(sql, params, (err, row) => { if (err) reject(err); else resolve(row || null); });
});
}
function dbAll(db, sql, params = []) {
return new Promise((resolve, reject) => {
db.all(sql, params, (err, rows) => { if (err) reject(err); else resolve(rows || []); });
});
}
// ---------------------------------------------------------------------------
// Run Python parser, return parsed object
// ---------------------------------------------------------------------------
function parseXlsx(filePath) {
return new Promise((resolve, reject) => {
const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
let out = '';
let err = '';
py.stdout.on('data', d => { out += d; });
py.stderr.on('data', d => { err += d; });
py.on('close', code => {
if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
try { resolve(JSON.parse(out)); }
catch (e) { reject(new Error('Parser returned invalid JSON')); }
});
py.on('error', reject);
});
}
// ---------------------------------------------------------------------------
// Run Python schema extractor, return xlsx schema object
// ---------------------------------------------------------------------------
function extractXlsxSchema(filePath) {
return new Promise((resolve, reject) => {
const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
let out = '';
let err = '';
py.stdout.on('data', d => { out += d; });
py.stderr.on('data', d => { err += d; });
py.on('close', code => {
if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
try { resolve(JSON.parse(out)); }
catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
});
py.on('error', reject);
});
}
// ---------------------------------------------------------------------------
// Validate that a temp file path is safely within uploads/temp/
// ---------------------------------------------------------------------------
function isSafeTempPath(filePath) {
const resolved = path.resolve(filePath);
return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
}
// ---------------------------------------------------------------------------
// Compute diff: new / recurring / resolved
// ---------------------------------------------------------------------------
async function computeDiff(db, incomingItems) {
const activeRows = await dbAll(db,
`SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
);
const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));
let newCount = 0, recurringCount = 0, resolvedCount = 0;
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; }
return { newCount, recurringCount, resolvedCount };
}
// ---------------------------------------------------------------------------
// Write a parsed upload to the DB (within a transaction)
// ---------------------------------------------------------------------------
async function persistUpload(db, { items, summary, reportDate, filename, userId }) {
// Pull current active items before we modify anything
const activeRows = await dbAll(db,
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
);
const activeMap = {};
activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });
const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));
await dbRun(db, 'BEGIN TRANSACTION');
try {
// 1. Insert the upload record
const { lastID: uploadId } = await dbRun(db,
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
VALUES (?, ?, ?, datetime('now'), ?)`,
[filename, reportDate || null, userId || null, JSON.stringify(summary)]
);
let newCount = 0, recurringCount = 0, resolvedCount = 0;
// 2. Upsert each incoming non-compliant item
for (const item of items) {
const key = `${item.hostname}|||${item.metric_id}`;
const existing = activeMap[key];
const extraStr = JSON.stringify(item.extra_json || {});
if (existing) {
// Recurring — bump seen_count, refresh snapshot fields
await dbRun(db,
`UPDATE compliance_items
SET upload_id = ?, seen_count = ?, ip_address = ?, device_type = ?, extra_json = ?
WHERE id = ?`,
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
);
recurringCount++;
} else {
// New item (or previously resolved and re-appearing)
await dbRun(db,
`INSERT INTO compliance_items
(upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
category, extra_json, status, first_seen_upload_id, seen_count)
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?, 1)`,
[uploadId, item.hostname, item.ip_address, item.device_type, item.team,
item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
);
newCount++;
}
}
// 3. Mark items not present in this upload as resolved
for (const [key, row] of Object.entries(activeMap)) {
if (!newKeys.has(key)) {
await dbRun(db,
`UPDATE compliance_items
SET status = 'resolved', resolved_upload_id = ?
WHERE id = ?`,
[uploadId, row.id]
);
resolvedCount++;
}
}
// 4. Update upload with final counts
await dbRun(db,
`UPDATE compliance_uploads
SET new_count = ?, resolved_count = ?, recurring_count = ?
WHERE id = ?`,
[newCount, resolvedCount, recurringCount, uploadId]
);
await dbRun(db, 'COMMIT');
return { uploadId, newCount, recurringCount, resolvedCount };
} catch (err) {
await dbRun(db, 'ROLLBACK').catch(() => {});
throw err;
}
}
// ---------------------------------------------------------------------------
// Group flat compliance_items rows into per-device objects
// ---------------------------------------------------------------------------
function groupByHostname(rows, noteHostnames) {
const deviceMap = {};
for (const row of rows) {
if (!deviceMap[row.hostname]) {
deviceMap[row.hostname] = {
hostname: row.hostname,
ip_address: row.ip_address || '',
device_type: row.device_type || '',
team: row.team || '',
status: row.status,
failing_metrics: [],
seen_count: row.seen_count || 1,
first_seen: row.first_seen || null,
last_seen: row.last_seen || null,
resolved_on: row.resolved_on || null,
has_notes: noteHostnames.has(row.hostname),
};
}
const dev = deviceMap[row.hostname];
dev.failing_metrics.push({
metric_id: row.metric_id,
metric_desc: row.metric_desc || '',
category: row.category || '',
});
// Use the highest seen_count and earliest first_seen across all metrics
if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count;
if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen))
dev.first_seen = row.first_seen;
if (row.last_seen && (!dev.last_seen || row.last_seen > dev.last_seen))
dev.last_seen = row.last_seen;
}
return Object.values(deviceMap);
}
// ---------------------------------------------------------------------------
// Pure function: bucket active items by age group and pivot per-team counts
// ---------------------------------------------------------------------------
const BUCKET_ORDER = ['1 cycle', '23 cycles', '46 cycles', '7+ cycles'];
function bucketAgingItems(items) {
// Initialise empty buckets with all teams at zero
const teams = ['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV'];
const buckets = {};
for (const b of BUCKET_ORDER) {
buckets[b] = { bucket: b, total: 0 };
for (const t of teams) buckets[b][t] = 0;
}
// Classify each item into a bucket
for (const item of items) {
const sc = item.seen_count;
let label;
if (sc === 1) label = '1 cycle';
else if (sc >= 2 && sc <= 3) label = '23 cycles';
else if (sc >= 4 && sc <= 6) label = '46 cycles';
else label = '7+ cycles';
const team = item.team;
buckets[label].total += 1;
if (team in buckets[label]) {
buckets[label][team] += 1;
}
}
// Return in ascending age order
return BUCKET_ORDER.map(b => buckets[b]);
}
// ---------------------------------------------------------------------------
// Pure function: compute waterfall chain from ordered upload records
// ---------------------------------------------------------------------------
function computeWaterfall(uploads) {
let start = 0;
return uploads.map((row) => {
const end = start + row.new_count + row.recurring_count - row.resolved_count;
const entry = {
date: row.report_date,
start,
new_count: row.new_count,
recurring_count: row.recurring_count,
resolved_count: row.resolved_count,
end,
};
start = end;
return entry;
});
}
// ---------------------------------------------------------------------------
// Router factory
// ---------------------------------------------------------------------------
function createComplianceRouter(db, upload, requireAuth, requireGroup) {
const router = express.Router();
// Idempotent column additions — errors mean column already exists, which is fine
db.run(`ALTER TABLE compliance_items ADD COLUMN seen_count INTEGER DEFAULT 1`, () => {});
db.run(`ALTER TABLE compliance_uploads ADD COLUMN summary_json TEXT`, () => {});
// All compliance routes require authentication
router.use(requireAuth(db));
// -----------------------------------------------------------------------
// POST /preview
// Parse the uploaded xlsx, compute diff, save parsed data to a temp JSON.
// Returns diff counts + tempFile path for the commit step.
//
// Body: multipart/form-data with `file` field (xlsx)
// Response: {
// drift: { breaking: [], silent_miss: [], cosmetic: [] } | null,
// drift_error: string | null,
// diff: { new_count, recurring_count, resolved_count },
// tempFile: string, filename: string,
// report_date: string, total_items: number
// }
// -----------------------------------------------------------------------
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
upload.single('file')(req, res, async (uploadErr) => {
if (uploadErr) {
return res.status(400).json({ error: uploadErr.message });
}
if (!req.file) {
return res.status(400).json({ error: 'No file uploaded' });
}
if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
fs.unlink(req.file.path, () => {});
return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
}
try {
// --- Drift check: load config, extract schema, compare ---
let drift = null;
let drift_error = null;
let config;
try {
config = loadConfig(CONFIG_PATH);
} catch (configErr) {
fs.unlink(req.file.path, () => {});
return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
}
let xlsxSchema = null;
try {
xlsxSchema = await extractXlsxSchema(req.file.path);
if (xlsxSchema.error) {
throw new Error(xlsxSchema.error);
}
drift = compareSchemaToDrift(xlsxSchema, config);
} catch (driftErr) {
drift = null;
drift_error = driftErr.message || 'Drift check failed';
}
// --- Existing parse flow ---
const parsed = await parseXlsx(req.file.path);
if (parsed.error) {
fs.unlink(req.file.path, () => {});
return res.status(422).json({ error: parsed.error });
}
const diff = await computeDiff(db, parsed.items);
// Save parsed data to temp JSON — the commit step reads this
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
const tempFilePath = path.join(TEMP_DIR, tempFilename);
fs.writeFileSync(tempFilePath, JSON.stringify({
items: parsed.items,
summary: parsed.summary,
report_date: parsed.report_date,
filename: req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
}));
// Delete the original xlsx from temp (we only need the JSON now)
fs.unlink(req.file.path, () => {});
res.json({
drift,
drift_error,
schema: xlsxSchema,
diff: {
new_count: diff.newCount,
recurring_count: diff.recurringCount,
resolved_count: diff.resolvedCount,
},
tempFile: tempFilePath,
filename: req.file.originalname,
report_date: parsed.report_date,
total_items: parsed.total,
});
} catch (err) {
fs.unlink(req.file.path, () => {});
console.error('[Compliance] Preview error:', err.message);
res.status(500).json({ error: 'Failed to parse file: ' + err.message });
}
});
});
// -----------------------------------------------------------------------
// POST /reconcile-config
// Admin-only. Patches compliance_config.json to resolve breaking and
// silent-miss drift findings, then re-runs the drift check and returns
// the updated report. Logs every change to the audit trail.
//
// Body: { drift: { breaking: [...], silent_miss: [...] } }
// Response: { changes: [{ action, key, value, detail }], message: string }
// -----------------------------------------------------------------------
router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
const { drift, schema } = req.body;
if (!drift || typeof drift !== 'object') {
return res.status(400).json({ error: 'drift report is required in request body' });
}
const hasFindings = (drift.breaking && drift.breaking.length > 0) ||
(drift.silent_miss && drift.silent_miss.length > 0);
if (!hasFindings) {
return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });
}
try {
const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);
if (changes.length === 0) {
return res.json({ changes: [], message: 'No changes needed' });
}
// Audit log each change
for (const change of changes) {
logAudit(db, {
userId: req.user.id,
username: req.user.username,
action: 'compliance_config_reconcile',
entityType: 'compliance_config',
entityId: change.value,
details: { action: change.action, key: change.key, detail: change.detail },
ipAddress: req.ip,
});
}
res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });
} catch (err) {
console.error('[Compliance] Reconcile config error:', err.message);
res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
}
});
// -----------------------------------------------------------------------
// POST /commit
// Commit a previewed upload to the DB.
//
// Body: { tempFile: string, filename: string, report_date: string }
// Response: { upload: { id, filename, report_date, uploaded_at,
// new_count, resolved_count, recurring_count } }
// -----------------------------------------------------------------------
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
const { tempFile, filename, report_date } = req.body;
if (!tempFile || typeof tempFile !== 'string') {
return res.status(400).json({ error: 'tempFile is required' });
}
if (!isSafeTempPath(tempFile)) {
return res.status(400).json({ error: 'Invalid tempFile path' });
}
if (!fs.existsSync(tempFile)) {
return res.status(400).json({ error: 'Preview session expired — please upload again' });
}
let parsed;
try {
parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8'));
} catch {
return res.status(400).json({ error: 'Could not read preview data — please upload again' });
}
try {
const result = await persistUpload(db, {
items: parsed.items,
summary: parsed.summary,
reportDate: report_date || parsed.report_date,
filename: filename || parsed.filename,
userId: req.user?.id || null,
});
fs.unlink(tempFile, () => {});
const upload = await dbGet(db,
`SELECT id, filename, report_date, uploaded_at,
new_count, resolved_count, recurring_count
FROM compliance_uploads WHERE id = ?`,
[result.uploadId]
);
res.json({ upload });
} catch (err) {
console.error('[Compliance] Commit error:', err.message);
res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
}
});
// -----------------------------------------------------------------------
// GET /uploads
// List all uploads, most recent first.
//
// Response: { uploads: [{ id, filename, report_date, uploaded_at,
// new_count, resolved_count, recurring_count }] }
// -----------------------------------------------------------------------
router.get('/uploads', async (req, res) => {
try {
const rows = await dbAll(db,
`SELECT id, filename, report_date, uploaded_at,
new_count, resolved_count, recurring_count
FROM compliance_uploads
ORDER BY id DESC`
);
res.json({ uploads: rows });
} catch (err) {
console.error('[Compliance] GET /uploads error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// POST /rollback/:uploadId
// Admin-only. Rolls back a specific upload. Only the most recent upload
// can be rolled back to avoid cascading data integrity issues.
//
// Params: uploadId — integer ID of the upload to roll back
// Response: { message: string, rolled_back: { upload_id, filename,
// report_date, items_deleted, items_reactivated } }
//
// Reversal logic:
// 1. Delete items first seen in this upload (new items)
// 2. Re-activate items resolved by this upload
// 3. Revert recurring items: decrement seen_count, point upload_id
// back to the previous upload
// 4. Delete the upload record
// -----------------------------------------------------------------------
router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
const uploadId = parseInt(req.params.uploadId, 10);
if (isNaN(uploadId)) {
return res.status(400).json({ error: 'Invalid upload ID' });
}
try {
// Verify the upload exists
const upload = await dbGet(db,
`SELECT id, filename, report_date, new_count, resolved_count, recurring_count
FROM compliance_uploads WHERE id = ?`,
[uploadId]
);
if (!upload) {
return res.status(404).json({ error: 'Upload not found' });
}
// Only allow rolling back the most recent upload
const latest = await dbGet(db,
`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
);
if (latest.id !== uploadId) {
return res.status(400).json({
error: 'Only the most recent upload can be rolled back',
latest_upload_id: latest.id
});
}
// Find the previous upload (to restore recurring items' upload_id)
const previousUpload = await dbGet(db,
`SELECT id FROM compliance_uploads WHERE id < ? ORDER BY id DESC LIMIT 1`,
[uploadId]
);
await dbRun(db, 'BEGIN TRANSACTION');
try {
// 1. Delete items that were NEW in this upload
const deleteNew = await dbRun(db,
`DELETE FROM compliance_items WHERE first_seen_upload_id = ? AND upload_id = ?`,
[uploadId, uploadId]
);
// 2. Re-activate items that were RESOLVED by this upload
const reactivate = await dbRun(db,
`UPDATE compliance_items
SET status = 'active', resolved_upload_id = NULL
WHERE resolved_upload_id = ?`,
[uploadId]
);
// 3. Revert RECURRING items: decrement seen_count, restore upload_id
if (previousUpload) {
await dbRun(db,
`UPDATE compliance_items
SET upload_id = ?, seen_count = MAX(seen_count - 1, 1)
WHERE upload_id = ? AND first_seen_upload_id != ?`,
[previousUpload.id, uploadId, uploadId]
);
}
// 4. Delete the upload record
await dbRun(db, `DELETE FROM compliance_uploads WHERE id = ?`, [uploadId]);
await dbRun(db, 'COMMIT');
// Audit log
logAudit(db, {
userId: req.user.id,
username: req.user.username,
action: 'compliance_upload_rollback',
entityType: 'compliance_upload',
entityId: String(uploadId),
details: {
filename: upload.filename,
report_date: upload.report_date,
items_deleted: deleteNew.changes,
items_reactivated: reactivate.changes,
},
ipAddress: req.ip,
});
res.json({
message: `Rolled back upload "${upload.filename}"`,
rolled_back: {
upload_id: uploadId,
filename: upload.filename,
report_date: upload.report_date,
items_deleted: deleteNew.changes,
items_reactivated: reactivate.changes,
},
});
} catch (err) {
await dbRun(db, 'ROLLBACK').catch(() => {});
throw err;
}
} catch (err) {
console.error('[Compliance] Rollback error:', err.message);
res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
}
});
// -----------------------------------------------------------------------
// GET /summary?team=STEAM
// Return metric health rows for a team from the latest upload's summary_json.
//
// Query: team — optional, one of ALLOWED_TEAMS
// Response: { entries: [...], overall_scores: {}, upload: { id,
// report_date, uploaded_at } | null }
// -----------------------------------------------------------------------
router.get('/summary', async (req, res) => {
const team = req.query.team;
if (team && !ALLOWED_TEAMS.has(team)) {
return res.status(400).json({ error: 'Invalid team' });
}
try {
const latestUpload = await dbGet(db,
`SELECT id, summary_json, report_date, uploaded_at
FROM compliance_uploads ORDER BY id DESC LIMIT 1`
);
if (!latestUpload || !latestUpload.summary_json) {
return res.json({ entries: [], overall_scores: {}, upload: null });
}
let summary;
try { summary = JSON.parse(latestUpload.summary_json); }
catch { return res.json({ entries: [], overall_scores: {}, upload: null }); }
let entries = summary.entries || [];
if (team) {
entries = entries.filter(e => e.team === team);
}
res.json({
entries,
overall_scores: summary.overall_scores || {},
upload: {
id: latestUpload.id,
report_date: latestUpload.report_date,
uploaded_at: latestUpload.uploaded_at,
},
});
} catch (err) {
console.error('[Compliance] GET /summary error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// GET /items?team=STEAM&status=active
// Return non-compliant devices grouped by hostname.
//
// Query: team — required, one of ALLOWED_TEAMS
// status — optional, 'active' (default) or 'resolved'
// Response: { devices: [{ hostname, ip_address, device_type, team,
// status, failing_metrics, seen_count, first_seen, last_seen,
// resolved_on, has_notes }], team, status }
// -----------------------------------------------------------------------
router.get('/items', async (req, res) => {
const { team, status = 'active' } = req.query;
if (!team) return res.status(400).json({ error: 'team is required' });
if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' });
try {
const rows = await dbAll(db,
`SELECT
ci.hostname, ci.ip_address, ci.device_type, ci.team,
ci.metric_id, ci.metric_desc, ci.category,
ci.status, ci.seen_count,
fu.report_date AS first_seen,
lu.report_date AS last_seen,
ru.report_date AS resolved_on
FROM compliance_items ci
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
WHERE ci.team = ? AND ci.status = ?
ORDER BY ci.hostname, ci.metric_id`,
[team, status]
);
// Fetch hostnames that have any notes (for the has_notes indicator)
const noteRows = await dbAll(db,
`SELECT DISTINCT hostname FROM compliance_notes`
);
const noteHostnames = new Set(noteRows.map(r => r.hostname));
const devices = groupByHostname(rows, noteHostnames);
res.json({ devices, team, status });
} catch (err) {
console.error('[Compliance] GET /items error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// GET /items/:hostname
// Detail panel: all metric rows for this hostname + notes + upload history.
//
// Params: hostname — device hostname string
// Response: { hostname, ip_address, device_type, team,
// metrics: [{ metric_id, metric_desc, category, status, seen_count,
// extra, first_seen, last_seen, resolved_on, ... }],
// notes: [{ id, metric_id, note, group_id, created_at, created_by }] }
// -----------------------------------------------------------------------
router.get('/items/:hostname', async (req, res) => {
const hostname = req.params.hostname;
if (!hostname || hostname.length > 300) {
return res.status(400).json({ error: 'Invalid hostname' });
}
try {
// All metric rows for this hostname
const metricRows = await dbAll(db,
`SELECT
ci.metric_id, ci.metric_desc, ci.category, ci.status,
ci.ip_address, ci.device_type, ci.team,
ci.seen_count, ci.extra_json,
fu.report_date AS first_seen,
fu.uploaded_at AS first_seen_at,
lu.report_date AS last_seen,
lu.uploaded_at AS last_seen_at,
ru.report_date AS resolved_on
FROM compliance_items ci
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
WHERE ci.hostname = ?
ORDER BY ci.status DESC, ci.metric_id`,
[hostname]
);
if (metricRows.length === 0) {
return res.status(404).json({ error: 'Device not found' });
}
// Parse extra_json on each row
const metrics = metricRows.map(r => ({
...r,
extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(),
extra_json: undefined,
}));
// Notes (all metrics for this hostname, sorted newest first)
const notes = await dbAll(db,
`SELECT cn.id, cn.metric_id, cn.note, cn.group_id, cn.created_at,
u.username AS created_by
FROM compliance_notes cn
LEFT JOIN users u ON cn.created_by = u.id
WHERE cn.hostname = ?
ORDER BY cn.created_at DESC`,
[hostname]
);
// Derive device identity from the first active row, else any row
const identity = metricRows.find(r => r.status === 'active') || metricRows[0];
res.json({
hostname,
ip_address: identity.ip_address || '',
device_type: identity.device_type || '',
team: identity.team || '',
metrics,
notes,
});
} catch (err) {
console.error('[Compliance] GET /items/:hostname error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// POST /notes
// Add a note to one or more (hostname, metric_id) pairs.
//
// Body: { hostname: string, metric_ids: string[], note: string }
// — or legacy: { hostname: string, metric_id: string, note: string }
// Response: { notes: [{ id, hostname, metric_id, note, group_id,
// created_at, created_by }] }
// -----------------------------------------------------------------------
router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
const { hostname, metric_id, metric_ids, note } = req.body;
if (!hostname || typeof hostname !== 'string' || hostname.length > 300 || !/^[a-zA-Z0-9._-]+$/.test(hostname)) {
return res.status(400).json({ error: 'Invalid hostname format' });
}
// --- Resolve metric IDs: metric_ids takes precedence over metric_id ---
let resolvedIds;
if (metric_ids !== undefined) {
if (!Array.isArray(metric_ids)) {
return res.status(400).json({ error: 'metric_ids must be an array' });
}
resolvedIds = metric_ids;
} else if (metric_id !== undefined && metric_id !== null && metric_id !== '') {
if (typeof metric_id !== 'string' || metric_id.length > 50) {
return res.status(400).json({ error: 'Invalid metric_id' });
}
resolvedIds = [metric_id];
} else {
return res.status(400).json({ error: 'metric_id or metric_ids is required' });
}
// --- Validate resolved metric IDs ---
if (resolvedIds.length === 0) {
return res.status(400).json({ error: 'At least one metric ID is required' });
}
for (let i = 0; i < resolvedIds.length; i++) {
const mid = resolvedIds[i];
if (!mid || typeof mid !== 'string' || mid.length === 0 || mid.length > 50) {
return res.status(400).json({ error: `Invalid metric_id at index ${i}` });
}
}
const noteText = String(note || '').trim().slice(0, 1000);
if (!noteText) {
return res.status(400).json({ error: 'Note cannot be empty' });
}
const groupId = crypto.randomUUID();
const userId = req.user?.id || null;
try {
await dbRun(db, 'BEGIN TRANSACTION');
const insertedIds = [];
for (const mid of resolvedIds) {
const { lastID } = await dbRun(db,
`INSERT INTO compliance_notes (hostname, metric_id, note, group_id, created_by, created_at)
VALUES (?, ?, ?, ?, ?, datetime('now'))`,
[hostname, mid, noteText, groupId, userId]
);
insertedIds.push(lastID);
}
await dbRun(db, 'COMMIT');
// Fetch all created rows with username
const placeholders = insertedIds.map(() => '?').join(', ');
const notes = await dbAll(db,
`SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.group_id, cn.created_at,
u.username AS created_by
FROM compliance_notes cn
LEFT JOIN users u ON cn.created_by = u.id
WHERE cn.id IN (${placeholders})
ORDER BY cn.id ASC`,
insertedIds
);
res.status(201).json({ notes });
} catch (err) {
await dbRun(db, 'ROLLBACK').catch(() => {});
console.error('[Compliance] POST /notes error:', err.message);
res.status(500).json({ error: 'Failed to save note' });
}
});
// -----------------------------------------------------------------------
// GET /notes/:hostname/:metricId
// Return all notes for a (hostname, metric_id) pair.
//
// Params: hostname — device hostname string
// metricId — metric identifier string
// Response: { notes: [{ id, note, created_at, created_by }] }
// -----------------------------------------------------------------------
router.get('/notes/:hostname/:metricId', async (req, res) => {
const { hostname, metricId } = req.params;
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metricId' });
try {
const notes = await dbAll(db,
`SELECT cn.id, cn.note, cn.created_at, u.username AS created_by
FROM compliance_notes cn
LEFT JOIN users u ON cn.created_by = u.id
WHERE cn.hostname = ? AND cn.metric_id = ?
ORDER BY cn.created_at DESC`,
[hostname, metricId]
);
res.json({ notes });
} catch (err) {
console.error('[Compliance] GET /notes error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// DELETE /notes/:id
// Delete a note (or all notes in the same group_id) by note ID.
// Only the note author or an Admin can delete.
//
// Params: id — note row ID
// Query: ?group=true — delete all notes sharing the same group_id
// Response: { deleted: number }
// -----------------------------------------------------------------------
router.delete('/notes/:id', requireGroup('Admin', 'Standard_User'), async (req, res) => {
const noteId = parseInt(req.params.id, 10);
if (isNaN(noteId)) return res.status(400).json({ error: 'Invalid note ID' });
const deleteGroup = req.query.group === 'true';
try {
// Fetch the note to verify ownership
const note = await dbGet(db,
`SELECT id, hostname, metric_id, note, group_id, created_by FROM compliance_notes WHERE id = ?`,
[noteId]
);
if (!note) return res.status(404).json({ error: 'Note not found' });
// Only the author or an Admin can delete
const isAuthor = req.user && String(req.user.id) === String(note.created_by);
const isAdminUser = req.user && req.user.group === 'Admin';
if (!isAuthor && !isAdminUser) {
return res.status(403).json({ error: 'You can only delete your own notes' });
}
let deleted = 0;
if (deleteGroup && note.group_id) {
const result = await dbRun(db,
`DELETE FROM compliance_notes WHERE group_id = ?`,
[note.group_id]
);
deleted = result.changes || 0;
} else {
const result = await dbRun(db,
`DELETE FROM compliance_notes WHERE id = ?`,
[noteId]
);
deleted = result.changes || 0;
}
logAudit(db, {
userId: req.user.id,
username: req.user.username,
action: 'compliance_note_delete',
entityType: 'compliance_note',
entityId: String(noteId),
details: JSON.stringify({ hostname: note.hostname, group_id: note.group_id, deleted_count: deleted }),
ipAddress: req.ip,
});
res.json({ deleted });
} catch (err) {
console.error('[Compliance] DELETE /notes error:', err.message);
res.status(500).json({ error: 'Failed to delete note' });
}
});
// -----------------------------------------------------------------------
// GET /trends
// Per-upload active totals + per-team counts for time-series charts.
// Returns rows ordered ascending by report_date.
//
// Response: { trends: [{ report_date, new_count, recurring_count,
// resolved_count, total_active, STEAM, ACCESS-ENG, ACCESS-OPS,
// INTELDEV }] }
// -----------------------------------------------------------------------
router.get('/trends', async (req, res) => {
try {
const uploads = await dbAll(db,
`SELECT id, report_date,
COALESCE(new_count, 0) AS new_count,
COALESCE(recurring_count, 0) AS recurring_count,
COALESCE(resolved_count, 0) AS resolved_count,
COALESCE(new_count, 0) + COALESCE(recurring_count, 0) AS total_active
FROM compliance_uploads
ORDER BY report_date ASC`
);
if (uploads.length === 0) return res.json({ trends: [] });
// Per-team active counts — items whose upload_id matches the upload
// (recurring items have upload_id bumped each cycle, so this is accurate)
const teamRows = await dbAll(db,
`SELECT ci.upload_id, ci.team, COUNT(ci.id) AS count
FROM compliance_items ci
WHERE ci.team IS NOT NULL
GROUP BY ci.upload_id, ci.team`
);
const teamMap = {};
teamRows.forEach(r => {
if (!teamMap[r.upload_id]) teamMap[r.upload_id] = {};
teamMap[r.upload_id][r.team] = r.count;
});
const trends = uploads.map(u => ({
report_date: u.report_date,
new_count: u.new_count,
recurring_count: u.recurring_count,
resolved_count: u.resolved_count,
total_active: u.total_active,
STEAM: teamMap[u.id]?.STEAM || 0,
'ACCESS-ENG': teamMap[u.id]?.['ACCESS-ENG'] || 0,
'ACCESS-OPS': teamMap[u.id]?.['ACCESS-OPS'] || 0,
INTELDEV: teamMap[u.id]?.INTELDEV || 0,
}));
res.json({ trends });
} catch (err) {
console.error('[Compliance] GET /trends error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// GET /mttr
// Aging Findings Distribution — active findings bucketed by seen_count
// with per-team breakdown for stacked bar chart.
//
// Response: { aging: [{ bucket, total, STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV }] }
// -----------------------------------------------------------------------
router.get('/mttr', async (req, res) => {
try {
const rows = await dbAll(db,
`SELECT COALESCE(seen_count, 1) AS seen_count, team
FROM compliance_items
WHERE status = 'active'`
);
if (rows.length === 0) {
return res.json({ aging: [] });
}
const aging = bucketAgingItems(rows);
res.json({ aging });
} catch (err) {
console.error('[Compliance] GET /mttr error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// GET /top-recurring
// Net Change Waterfall — per-cycle net movement (start → +new →
// +recurring → resolved → end) computed from compliance_uploads.
//
// Response: { waterfall: [{ date, start, new_count, recurring_count,
// resolved_count, end }] }
// -----------------------------------------------------------------------
router.get('/top-recurring', async (req, res) => {
try {
const rows = await dbAll(db,
`SELECT id, report_date,
COALESCE(new_count, 0) AS new_count,
COALESCE(recurring_count, 0) AS recurring_count,
COALESCE(resolved_count, 0) AS resolved_count
FROM compliance_uploads
ORDER BY report_date ASC`
);
const waterfall = computeWaterfall(rows);
res.json({ waterfall });
} catch (err) {
console.error('[Compliance] GET /top-recurring error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
// -----------------------------------------------------------------------
// GET /category-trend
// Active item counts per category per upload, for stacked area chart.
//
// Response: { categoryTrend: [{ report_date, category, count }] }
// -----------------------------------------------------------------------
router.get('/category-trend', async (req, res) => {
try {
const rows = await dbAll(db,
`SELECT cu.report_date, COALESCE(ci.category, 'Unknown') AS category, COUNT(ci.id) AS count
FROM compliance_uploads cu
JOIN compliance_items ci ON ci.upload_id = cu.id
GROUP BY cu.id, category
ORDER BY cu.report_date ASC`
);
res.json({ categoryTrend: rows });
} catch (err) {
console.error('[Compliance] GET /category-trend error:', err.message);
res.status(500).json({ error: 'Database error' });
}
});
return router;
}
module.exports = { createComplianceRouter, bucketAgingItems, computeWaterfall };