1031 lines
44 KiB
JavaScript
1031 lines
44 KiB
JavaScript
// Compliance Routes — AEO metric tracking
|
|
// Handles xlsx upload/parse, non-compliant item history, and notes.
|
|
//
|
|
// Endpoints:
|
|
// POST /preview — parse xlsx, run drift check, compute diff (no DB write)
|
|
// POST /reconcile-config — patch compliance_config.json to resolve drift findings
|
|
// POST /commit — commit a previewed upload to DB
|
|
// GET /uploads — list all uploads
|
|
// POST /rollback/:uploadId — roll back the most recent upload (Admin only)
|
|
// GET /summary — metric health cards for a team (from latest upload)
|
|
// GET /items — non-compliant devices grouped by hostname (?team=X&status=active)
|
|
// GET /items/:hostname — detail panel: all metrics + notes + upload history for a device
|
|
// POST /notes — add a note to one or more (hostname, metric_id) pairs
|
|
// GET /notes/:hostname/:metricId — notes for a specific device+metric
|
|
// GET /trends — per-upload totals + per-team counts for time-series charts
|
|
// GET /mttr — mean time to resolution per team
|
|
// GET /top-recurring — chronic compliance gaps sorted by seen_count
|
|
// GET /category-trend — active counts per category per upload for stacked area chart
|
|
|
|
const express = require('express');
|
|
const path = require('path');
|
|
const fs = require('fs');
|
|
const crypto = require('crypto');
|
|
const { spawn } = require('child_process');
|
|
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
|
|
const logAudit = require('../helpers/auditLog');
|
|
|
|
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
|
|
const SCHEMA_SCRIPT = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
|
|
const CONFIG_PATH = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
|
|
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
|
|
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
|
|
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// DB helpers
|
|
// ---------------------------------------------------------------------------
|
|
function dbRun(db, sql, params = []) {
|
|
return new Promise((resolve, reject) => {
|
|
db.run(sql, params, function (err) {
|
|
if (err) reject(err);
|
|
else resolve({ lastID: this.lastID, changes: this.changes });
|
|
});
|
|
});
|
|
}
|
|
function dbGet(db, sql, params = []) {
|
|
return new Promise((resolve, reject) => {
|
|
db.get(sql, params, (err, row) => { if (err) reject(err); else resolve(row || null); });
|
|
});
|
|
}
|
|
function dbAll(db, sql, params = []) {
|
|
return new Promise((resolve, reject) => {
|
|
db.all(sql, params, (err, rows) => { if (err) reject(err); else resolve(rows || []); });
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Run Python parser, return parsed object
|
|
// ---------------------------------------------------------------------------
|
|
function parseXlsx(filePath) {
|
|
return new Promise((resolve, reject) => {
|
|
const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
|
|
let out = '';
|
|
let err = '';
|
|
py.stdout.on('data', d => { out += d; });
|
|
py.stderr.on('data', d => { err += d; });
|
|
py.on('close', code => {
|
|
if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
|
|
try { resolve(JSON.parse(out)); }
|
|
catch (e) { reject(new Error('Parser returned invalid JSON')); }
|
|
});
|
|
py.on('error', reject);
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Run Python schema extractor, return xlsx schema object
|
|
// ---------------------------------------------------------------------------
|
|
function extractXlsxSchema(filePath) {
|
|
return new Promise((resolve, reject) => {
|
|
const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
|
|
let out = '';
|
|
let err = '';
|
|
py.stdout.on('data', d => { out += d; });
|
|
py.stderr.on('data', d => { err += d; });
|
|
py.on('close', code => {
|
|
if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
|
|
try { resolve(JSON.parse(out)); }
|
|
catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
|
|
});
|
|
py.on('error', reject);
|
|
});
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Validate that a temp file path is safely within uploads/temp/
|
|
// ---------------------------------------------------------------------------
|
|
function isSafeTempPath(filePath) {
|
|
const resolved = path.resolve(filePath);
|
|
return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Compute diff: new / recurring / resolved
|
|
// ---------------------------------------------------------------------------
|
|
async function computeDiff(db, incomingItems) {
|
|
const activeRows = await dbAll(db,
|
|
`SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
|
|
);
|
|
const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
|
|
const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));
|
|
|
|
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
|
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
|
|
for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; }
|
|
|
|
return { newCount, recurringCount, resolvedCount };
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Write a parsed upload to the DB (within a transaction)
|
|
// ---------------------------------------------------------------------------
|
|
async function persistUpload(db, { items, summary, reportDate, filename, userId }) {
|
|
// Pull current active items before we modify anything
|
|
const activeRows = await dbAll(db,
|
|
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
|
|
);
|
|
const activeMap = {};
|
|
activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });
|
|
|
|
const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));
|
|
|
|
await dbRun(db, 'BEGIN TRANSACTION');
|
|
try {
|
|
// 1. Insert the upload record
|
|
const { lastID: uploadId } = await dbRun(db,
|
|
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
|
|
VALUES (?, ?, ?, datetime('now'), ?)`,
|
|
[filename, reportDate || null, userId || null, JSON.stringify(summary)]
|
|
);
|
|
|
|
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
|
|
|
// 2. Upsert each incoming non-compliant item
|
|
for (const item of items) {
|
|
const key = `${item.hostname}|||${item.metric_id}`;
|
|
const existing = activeMap[key];
|
|
const extraStr = JSON.stringify(item.extra_json || {});
|
|
|
|
if (existing) {
|
|
// Recurring — bump seen_count, refresh snapshot fields
|
|
await dbRun(db,
|
|
`UPDATE compliance_items
|
|
SET upload_id = ?, seen_count = ?, ip_address = ?, device_type = ?, extra_json = ?
|
|
WHERE id = ?`,
|
|
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
|
|
);
|
|
recurringCount++;
|
|
} else {
|
|
// New item (or previously resolved and re-appearing)
|
|
await dbRun(db,
|
|
`INSERT INTO compliance_items
|
|
(upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
|
|
category, extra_json, status, first_seen_upload_id, seen_count)
|
|
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?, 1)`,
|
|
[uploadId, item.hostname, item.ip_address, item.device_type, item.team,
|
|
item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
|
|
);
|
|
newCount++;
|
|
}
|
|
}
|
|
|
|
// 3. Mark items not present in this upload as resolved
|
|
for (const [key, row] of Object.entries(activeMap)) {
|
|
if (!newKeys.has(key)) {
|
|
await dbRun(db,
|
|
`UPDATE compliance_items
|
|
SET status = 'resolved', resolved_upload_id = ?
|
|
WHERE id = ?`,
|
|
[uploadId, row.id]
|
|
);
|
|
resolvedCount++;
|
|
}
|
|
}
|
|
|
|
// 4. Update upload with final counts
|
|
await dbRun(db,
|
|
`UPDATE compliance_uploads
|
|
SET new_count = ?, resolved_count = ?, recurring_count = ?
|
|
WHERE id = ?`,
|
|
[newCount, resolvedCount, recurringCount, uploadId]
|
|
);
|
|
|
|
await dbRun(db, 'COMMIT');
|
|
return { uploadId, newCount, recurringCount, resolvedCount };
|
|
|
|
} catch (err) {
|
|
await dbRun(db, 'ROLLBACK').catch(() => {});
|
|
throw err;
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Group flat compliance_items rows into per-device objects
|
|
// ---------------------------------------------------------------------------
|
|
function groupByHostname(rows, noteHostnames) {
|
|
const deviceMap = {};
|
|
|
|
for (const row of rows) {
|
|
if (!deviceMap[row.hostname]) {
|
|
deviceMap[row.hostname] = {
|
|
hostname: row.hostname,
|
|
ip_address: row.ip_address || '',
|
|
device_type: row.device_type || '',
|
|
team: row.team || '',
|
|
status: row.status,
|
|
failing_metrics: [],
|
|
seen_count: row.seen_count || 1,
|
|
first_seen: row.first_seen || null,
|
|
last_seen: row.last_seen || null,
|
|
resolved_on: row.resolved_on || null,
|
|
has_notes: noteHostnames.has(row.hostname),
|
|
};
|
|
}
|
|
|
|
const dev = deviceMap[row.hostname];
|
|
dev.failing_metrics.push({
|
|
metric_id: row.metric_id,
|
|
metric_desc: row.metric_desc || '',
|
|
category: row.category || '',
|
|
});
|
|
// Use the highest seen_count and earliest first_seen across all metrics
|
|
if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count;
|
|
if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen))
|
|
dev.first_seen = row.first_seen;
|
|
if (row.last_seen && (!dev.last_seen || row.last_seen > dev.last_seen))
|
|
dev.last_seen = row.last_seen;
|
|
}
|
|
|
|
return Object.values(deviceMap);
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Router factory
|
|
// ---------------------------------------------------------------------------
|
|
function createComplianceRouter(db, upload, requireAuth, requireGroup) {
|
|
const router = express.Router();
|
|
|
|
// Idempotent column additions — errors mean column already exists, which is fine
|
|
db.run(`ALTER TABLE compliance_items ADD COLUMN seen_count INTEGER DEFAULT 1`, () => {});
|
|
db.run(`ALTER TABLE compliance_uploads ADD COLUMN summary_json TEXT`, () => {});
|
|
|
|
// All compliance routes require authentication
|
|
router.use(requireAuth(db));
|
|
|
|
// -----------------------------------------------------------------------
|
|
// POST /preview
|
|
// Parse the uploaded xlsx, compute diff, save parsed data to a temp JSON.
|
|
// Returns diff counts + tempFile path for the commit step.
|
|
//
|
|
// Body: multipart/form-data with `file` field (xlsx)
|
|
// Response: {
|
|
// drift: { breaking: [], silent_miss: [], cosmetic: [] } | null,
|
|
// drift_error: string | null,
|
|
// diff: { new_count, recurring_count, resolved_count },
|
|
// tempFile: string, filename: string,
|
|
// report_date: string, total_items: number
|
|
// }
|
|
// -----------------------------------------------------------------------
|
|
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
|
|
upload.single('file')(req, res, async (uploadErr) => {
|
|
if (uploadErr) {
|
|
return res.status(400).json({ error: uploadErr.message });
|
|
}
|
|
if (!req.file) {
|
|
return res.status(400).json({ error: 'No file uploaded' });
|
|
}
|
|
if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
|
|
fs.unlink(req.file.path, () => {});
|
|
return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
|
|
}
|
|
|
|
try {
|
|
// --- Drift check: load config, extract schema, compare ---
|
|
let drift = null;
|
|
let drift_error = null;
|
|
|
|
let config;
|
|
try {
|
|
config = loadConfig(CONFIG_PATH);
|
|
} catch (configErr) {
|
|
fs.unlink(req.file.path, () => {});
|
|
return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
|
|
}
|
|
|
|
let xlsxSchema = null;
|
|
try {
|
|
xlsxSchema = await extractXlsxSchema(req.file.path);
|
|
if (xlsxSchema.error) {
|
|
throw new Error(xlsxSchema.error);
|
|
}
|
|
drift = compareSchemaToDrift(xlsxSchema, config);
|
|
} catch (driftErr) {
|
|
drift = null;
|
|
drift_error = driftErr.message || 'Drift check failed';
|
|
}
|
|
|
|
// --- Existing parse flow ---
|
|
const parsed = await parseXlsx(req.file.path);
|
|
|
|
if (parsed.error) {
|
|
fs.unlink(req.file.path, () => {});
|
|
return res.status(422).json({ error: parsed.error });
|
|
}
|
|
|
|
const diff = await computeDiff(db, parsed.items);
|
|
|
|
// Save parsed data to temp JSON — the commit step reads this
|
|
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
|
|
const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
|
|
const tempFilePath = path.join(TEMP_DIR, tempFilename);
|
|
|
|
fs.writeFileSync(tempFilePath, JSON.stringify({
|
|
items: parsed.items,
|
|
summary: parsed.summary,
|
|
report_date: parsed.report_date,
|
|
filename: req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
|
|
}));
|
|
|
|
// Delete the original xlsx from temp (we only need the JSON now)
|
|
fs.unlink(req.file.path, () => {});
|
|
|
|
res.json({
|
|
drift,
|
|
drift_error,
|
|
schema: xlsxSchema,
|
|
diff: {
|
|
new_count: diff.newCount,
|
|
recurring_count: diff.recurringCount,
|
|
resolved_count: diff.resolvedCount,
|
|
},
|
|
tempFile: tempFilePath,
|
|
filename: req.file.originalname,
|
|
report_date: parsed.report_date,
|
|
total_items: parsed.total,
|
|
});
|
|
|
|
} catch (err) {
|
|
fs.unlink(req.file.path, () => {});
|
|
console.error('[Compliance] Preview error:', err.message);
|
|
res.status(500).json({ error: 'Failed to parse file: ' + err.message });
|
|
}
|
|
});
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// POST /reconcile-config
|
|
// Admin-only. Patches compliance_config.json to resolve breaking and
|
|
// silent-miss drift findings, then re-runs the drift check and returns
|
|
// the updated report. Logs every change to the audit trail.
|
|
//
|
|
// Body: { drift: { breaking: [...], silent_miss: [...] } }
|
|
// Response: { changes: [{ action, key, value, detail }], message: string }
|
|
// -----------------------------------------------------------------------
|
|
router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
|
|
const { drift, schema } = req.body;
|
|
|
|
if (!drift || typeof drift !== 'object') {
|
|
return res.status(400).json({ error: 'drift report is required in request body' });
|
|
}
|
|
|
|
const hasFindings = (drift.breaking && drift.breaking.length > 0) ||
|
|
(drift.silent_miss && drift.silent_miss.length > 0);
|
|
if (!hasFindings) {
|
|
return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });
|
|
}
|
|
|
|
try {
|
|
const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);
|
|
|
|
if (changes.length === 0) {
|
|
return res.json({ changes: [], message: 'No changes needed' });
|
|
}
|
|
|
|
// Audit log each change
|
|
for (const change of changes) {
|
|
logAudit(db, {
|
|
userId: req.user.id,
|
|
username: req.user.username,
|
|
action: 'compliance_config_reconcile',
|
|
entityType: 'compliance_config',
|
|
entityId: change.value,
|
|
details: { action: change.action, key: change.key, detail: change.detail },
|
|
ipAddress: req.ip,
|
|
});
|
|
}
|
|
|
|
res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });
|
|
|
|
} catch (err) {
|
|
console.error('[Compliance] Reconcile config error:', err.message);
|
|
res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// POST /commit
|
|
// Commit a previewed upload to the DB.
|
|
//
|
|
// Body: { tempFile: string, filename: string, report_date: string }
|
|
// Response: { upload: { id, filename, report_date, uploaded_at,
|
|
// new_count, resolved_count, recurring_count } }
|
|
// -----------------------------------------------------------------------
|
|
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
|
const { tempFile, filename, report_date } = req.body;
|
|
|
|
if (!tempFile || typeof tempFile !== 'string') {
|
|
return res.status(400).json({ error: 'tempFile is required' });
|
|
}
|
|
if (!isSafeTempPath(tempFile)) {
|
|
return res.status(400).json({ error: 'Invalid tempFile path' });
|
|
}
|
|
if (!fs.existsSync(tempFile)) {
|
|
return res.status(400).json({ error: 'Preview session expired — please upload again' });
|
|
}
|
|
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8'));
|
|
} catch {
|
|
return res.status(400).json({ error: 'Could not read preview data — please upload again' });
|
|
}
|
|
|
|
try {
|
|
const result = await persistUpload(db, {
|
|
items: parsed.items,
|
|
summary: parsed.summary,
|
|
reportDate: report_date || parsed.report_date,
|
|
filename: filename || parsed.filename,
|
|
userId: req.user?.id || null,
|
|
});
|
|
|
|
fs.unlink(tempFile, () => {});
|
|
|
|
const upload = await dbGet(db,
|
|
`SELECT id, filename, report_date, uploaded_at,
|
|
new_count, resolved_count, recurring_count
|
|
FROM compliance_uploads WHERE id = ?`,
|
|
[result.uploadId]
|
|
);
|
|
|
|
res.json({ upload });
|
|
|
|
} catch (err) {
|
|
console.error('[Compliance] Commit error:', err.message);
|
|
res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /uploads
|
|
// List all uploads, most recent first.
|
|
//
|
|
// Response: { uploads: [{ id, filename, report_date, uploaded_at,
|
|
// new_count, resolved_count, recurring_count }] }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/uploads', async (req, res) => {
|
|
try {
|
|
const rows = await dbAll(db,
|
|
`SELECT id, filename, report_date, uploaded_at,
|
|
new_count, resolved_count, recurring_count
|
|
FROM compliance_uploads
|
|
ORDER BY id DESC`
|
|
);
|
|
res.json({ uploads: rows });
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /uploads error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// POST /rollback/:uploadId
|
|
// Admin-only. Rolls back a specific upload. Only the most recent upload
|
|
// can be rolled back to avoid cascading data integrity issues.
|
|
//
|
|
// Params: uploadId — integer ID of the upload to roll back
|
|
// Response: { message: string, rolled_back: { upload_id, filename,
|
|
// report_date, items_deleted, items_reactivated } }
|
|
//
|
|
// Reversal logic:
|
|
// 1. Delete items first seen in this upload (new items)
|
|
// 2. Re-activate items resolved by this upload
|
|
// 3. Revert recurring items: decrement seen_count, point upload_id
|
|
// back to the previous upload
|
|
// 4. Delete the upload record
|
|
// -----------------------------------------------------------------------
|
|
router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
|
|
const uploadId = parseInt(req.params.uploadId, 10);
|
|
if (isNaN(uploadId)) {
|
|
return res.status(400).json({ error: 'Invalid upload ID' });
|
|
}
|
|
|
|
try {
|
|
// Verify the upload exists
|
|
const upload = await dbGet(db,
|
|
`SELECT id, filename, report_date, new_count, resolved_count, recurring_count
|
|
FROM compliance_uploads WHERE id = ?`,
|
|
[uploadId]
|
|
);
|
|
if (!upload) {
|
|
return res.status(404).json({ error: 'Upload not found' });
|
|
}
|
|
|
|
// Only allow rolling back the most recent upload
|
|
const latest = await dbGet(db,
|
|
`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
|
|
);
|
|
if (latest.id !== uploadId) {
|
|
return res.status(400).json({
|
|
error: 'Only the most recent upload can be rolled back',
|
|
latest_upload_id: latest.id
|
|
});
|
|
}
|
|
|
|
// Find the previous upload (to restore recurring items' upload_id)
|
|
const previousUpload = await dbGet(db,
|
|
`SELECT id FROM compliance_uploads WHERE id < ? ORDER BY id DESC LIMIT 1`,
|
|
[uploadId]
|
|
);
|
|
|
|
await dbRun(db, 'BEGIN TRANSACTION');
|
|
|
|
try {
|
|
// 1. Delete items that were NEW in this upload
|
|
const deleteNew = await dbRun(db,
|
|
`DELETE FROM compliance_items WHERE first_seen_upload_id = ? AND upload_id = ?`,
|
|
[uploadId, uploadId]
|
|
);
|
|
|
|
// 2. Re-activate items that were RESOLVED by this upload
|
|
const reactivate = await dbRun(db,
|
|
`UPDATE compliance_items
|
|
SET status = 'active', resolved_upload_id = NULL
|
|
WHERE resolved_upload_id = ?`,
|
|
[uploadId]
|
|
);
|
|
|
|
// 3. Revert RECURRING items: decrement seen_count, restore upload_id
|
|
if (previousUpload) {
|
|
await dbRun(db,
|
|
`UPDATE compliance_items
|
|
SET upload_id = ?, seen_count = MAX(seen_count - 1, 1)
|
|
WHERE upload_id = ? AND first_seen_upload_id != ?`,
|
|
[previousUpload.id, uploadId, uploadId]
|
|
);
|
|
}
|
|
|
|
// 4. Delete the upload record
|
|
await dbRun(db, `DELETE FROM compliance_uploads WHERE id = ?`, [uploadId]);
|
|
|
|
await dbRun(db, 'COMMIT');
|
|
|
|
// Audit log
|
|
logAudit(db, {
|
|
userId: req.user.id,
|
|
username: req.user.username,
|
|
action: 'compliance_upload_rollback',
|
|
entityType: 'compliance_upload',
|
|
entityId: String(uploadId),
|
|
details: {
|
|
filename: upload.filename,
|
|
report_date: upload.report_date,
|
|
items_deleted: deleteNew.changes,
|
|
items_reactivated: reactivate.changes,
|
|
},
|
|
ipAddress: req.ip,
|
|
});
|
|
|
|
res.json({
|
|
message: `Rolled back upload "${upload.filename}"`,
|
|
rolled_back: {
|
|
upload_id: uploadId,
|
|
filename: upload.filename,
|
|
report_date: upload.report_date,
|
|
items_deleted: deleteNew.changes,
|
|
items_reactivated: reactivate.changes,
|
|
},
|
|
});
|
|
|
|
} catch (err) {
|
|
await dbRun(db, 'ROLLBACK').catch(() => {});
|
|
throw err;
|
|
}
|
|
|
|
} catch (err) {
|
|
console.error('[Compliance] Rollback error:', err.message);
|
|
res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /summary?team=STEAM
|
|
// Return metric health rows for a team from the latest upload's summary_json.
|
|
//
|
|
// Query: team — optional, one of ALLOWED_TEAMS
|
|
// Response: { entries: [...], overall_scores: {}, upload: { id,
|
|
// report_date, uploaded_at } | null }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/summary', async (req, res) => {
|
|
const team = req.query.team;
|
|
if (team && !ALLOWED_TEAMS.has(team)) {
|
|
return res.status(400).json({ error: 'Invalid team' });
|
|
}
|
|
|
|
try {
|
|
const latestUpload = await dbGet(db,
|
|
`SELECT id, summary_json, report_date, uploaded_at
|
|
FROM compliance_uploads ORDER BY id DESC LIMIT 1`
|
|
);
|
|
if (!latestUpload || !latestUpload.summary_json) {
|
|
return res.json({ entries: [], overall_scores: {}, upload: null });
|
|
}
|
|
|
|
let summary;
|
|
try { summary = JSON.parse(latestUpload.summary_json); }
|
|
catch { return res.json({ entries: [], overall_scores: {}, upload: null }); }
|
|
|
|
let entries = summary.entries || [];
|
|
if (team) {
|
|
entries = entries.filter(e => e.team === team);
|
|
}
|
|
|
|
res.json({
|
|
entries,
|
|
overall_scores: summary.overall_scores || {},
|
|
upload: {
|
|
id: latestUpload.id,
|
|
report_date: latestUpload.report_date,
|
|
uploaded_at: latestUpload.uploaded_at,
|
|
},
|
|
});
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /summary error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /items?team=STEAM&status=active
|
|
// Return non-compliant devices grouped by hostname.
|
|
//
|
|
// Query: team — required, one of ALLOWED_TEAMS
|
|
// status — optional, 'active' (default) or 'resolved'
|
|
// Response: { devices: [{ hostname, ip_address, device_type, team,
|
|
// status, failing_metrics, seen_count, first_seen, last_seen,
|
|
// resolved_on, has_notes }], team, status }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/items', async (req, res) => {
|
|
const { team, status = 'active' } = req.query;
|
|
|
|
if (!team) return res.status(400).json({ error: 'team is required' });
|
|
if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
|
if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' });
|
|
|
|
try {
|
|
const rows = await dbAll(db,
|
|
`SELECT
|
|
ci.hostname, ci.ip_address, ci.device_type, ci.team,
|
|
ci.metric_id, ci.metric_desc, ci.category,
|
|
ci.status, ci.seen_count,
|
|
fu.report_date AS first_seen,
|
|
lu.report_date AS last_seen,
|
|
ru.report_date AS resolved_on
|
|
FROM compliance_items ci
|
|
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
|
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
|
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
|
WHERE ci.team = ? AND ci.status = ?
|
|
ORDER BY ci.hostname, ci.metric_id`,
|
|
[team, status]
|
|
);
|
|
|
|
// Fetch hostnames that have any notes (for the has_notes indicator)
|
|
const noteRows = await dbAll(db,
|
|
`SELECT DISTINCT hostname FROM compliance_notes`
|
|
);
|
|
const noteHostnames = new Set(noteRows.map(r => r.hostname));
|
|
|
|
const devices = groupByHostname(rows, noteHostnames);
|
|
|
|
res.json({ devices, team, status });
|
|
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /items error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /items/:hostname
|
|
// Detail panel: all metric rows for this hostname + notes + upload history.
|
|
//
|
|
// Params: hostname — device hostname string
|
|
// Response: { hostname, ip_address, device_type, team,
|
|
// metrics: [{ metric_id, metric_desc, category, status, seen_count,
|
|
// extra, first_seen, last_seen, resolved_on, ... }],
|
|
// notes: [{ id, metric_id, note, group_id, created_at, created_by }] }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/items/:hostname', async (req, res) => {
|
|
const hostname = req.params.hostname;
|
|
if (!hostname || hostname.length > 300) {
|
|
return res.status(400).json({ error: 'Invalid hostname' });
|
|
}
|
|
|
|
try {
|
|
// All metric rows for this hostname
|
|
const metricRows = await dbAll(db,
|
|
`SELECT
|
|
ci.metric_id, ci.metric_desc, ci.category, ci.status,
|
|
ci.ip_address, ci.device_type, ci.team,
|
|
ci.seen_count, ci.extra_json,
|
|
fu.report_date AS first_seen,
|
|
fu.uploaded_at AS first_seen_at,
|
|
lu.report_date AS last_seen,
|
|
lu.uploaded_at AS last_seen_at,
|
|
ru.report_date AS resolved_on
|
|
FROM compliance_items ci
|
|
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
|
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
|
LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
|
WHERE ci.hostname = ?
|
|
ORDER BY ci.status DESC, ci.metric_id`,
|
|
[hostname]
|
|
);
|
|
|
|
if (metricRows.length === 0) {
|
|
return res.status(404).json({ error: 'Device not found' });
|
|
}
|
|
|
|
// Parse extra_json on each row
|
|
const metrics = metricRows.map(r => ({
|
|
...r,
|
|
extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(),
|
|
extra_json: undefined,
|
|
}));
|
|
|
|
// Notes (all metrics for this hostname, sorted newest first)
|
|
const notes = await dbAll(db,
|
|
`SELECT cn.id, cn.metric_id, cn.note, cn.group_id, cn.created_at,
|
|
u.username AS created_by
|
|
FROM compliance_notes cn
|
|
LEFT JOIN users u ON cn.created_by = u.id
|
|
WHERE cn.hostname = ?
|
|
ORDER BY cn.created_at DESC`,
|
|
[hostname]
|
|
);
|
|
|
|
// Derive device identity from the first active row, else any row
|
|
const identity = metricRows.find(r => r.status === 'active') || metricRows[0];
|
|
|
|
res.json({
|
|
hostname,
|
|
ip_address: identity.ip_address || '',
|
|
device_type: identity.device_type || '',
|
|
team: identity.team || '',
|
|
metrics,
|
|
notes,
|
|
});
|
|
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /items/:hostname error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// POST /notes
|
|
// Add a note to one or more (hostname, metric_id) pairs.
|
|
//
|
|
// Body: { hostname: string, metric_ids: string[], note: string }
|
|
// — or legacy: { hostname: string, metric_id: string, note: string }
|
|
// Response: { notes: [{ id, hostname, metric_id, note, group_id,
|
|
// created_at, created_by }] }
|
|
// -----------------------------------------------------------------------
|
|
router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
|
const { hostname, metric_id, metric_ids, note } = req.body;
|
|
|
|
if (!hostname || typeof hostname !== 'string' || hostname.length > 300 || !/^[a-zA-Z0-9._-]+$/.test(hostname)) {
|
|
return res.status(400).json({ error: 'Invalid hostname format' });
|
|
}
|
|
|
|
// --- Resolve metric IDs: metric_ids takes precedence over metric_id ---
|
|
let resolvedIds;
|
|
if (metric_ids !== undefined) {
|
|
if (!Array.isArray(metric_ids)) {
|
|
return res.status(400).json({ error: 'metric_ids must be an array' });
|
|
}
|
|
resolvedIds = metric_ids;
|
|
} else if (metric_id !== undefined && metric_id !== null && metric_id !== '') {
|
|
if (typeof metric_id !== 'string' || metric_id.length > 50) {
|
|
return res.status(400).json({ error: 'Invalid metric_id' });
|
|
}
|
|
resolvedIds = [metric_id];
|
|
} else {
|
|
return res.status(400).json({ error: 'metric_id or metric_ids is required' });
|
|
}
|
|
|
|
// --- Validate resolved metric IDs ---
|
|
if (resolvedIds.length === 0) {
|
|
return res.status(400).json({ error: 'At least one metric ID is required' });
|
|
}
|
|
for (let i = 0; i < resolvedIds.length; i++) {
|
|
const mid = resolvedIds[i];
|
|
if (!mid || typeof mid !== 'string' || mid.length === 0 || mid.length > 50) {
|
|
return res.status(400).json({ error: `Invalid metric_id at index ${i}` });
|
|
}
|
|
}
|
|
|
|
const noteText = String(note || '').trim().slice(0, 1000);
|
|
if (!noteText) {
|
|
return res.status(400).json({ error: 'Note cannot be empty' });
|
|
}
|
|
|
|
const groupId = crypto.randomUUID();
|
|
const userId = req.user?.id || null;
|
|
|
|
try {
|
|
await dbRun(db, 'BEGIN TRANSACTION');
|
|
|
|
const insertedIds = [];
|
|
for (const mid of resolvedIds) {
|
|
const { lastID } = await dbRun(db,
|
|
`INSERT INTO compliance_notes (hostname, metric_id, note, group_id, created_by, created_at)
|
|
VALUES (?, ?, ?, ?, ?, datetime('now'))`,
|
|
[hostname, mid, noteText, groupId, userId]
|
|
);
|
|
insertedIds.push(lastID);
|
|
}
|
|
|
|
await dbRun(db, 'COMMIT');
|
|
|
|
// Fetch all created rows with username
|
|
const placeholders = insertedIds.map(() => '?').join(', ');
|
|
const notes = await dbAll(db,
|
|
`SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.group_id, cn.created_at,
|
|
u.username AS created_by
|
|
FROM compliance_notes cn
|
|
LEFT JOIN users u ON cn.created_by = u.id
|
|
WHERE cn.id IN (${placeholders})
|
|
ORDER BY cn.id ASC`,
|
|
insertedIds
|
|
);
|
|
|
|
res.status(201).json({ notes });
|
|
|
|
} catch (err) {
|
|
await dbRun(db, 'ROLLBACK').catch(() => {});
|
|
console.error('[Compliance] POST /notes error:', err.message);
|
|
res.status(500).json({ error: 'Failed to save note' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /notes/:hostname/:metricId
|
|
// Return all notes for a (hostname, metric_id) pair.
|
|
//
|
|
// Params: hostname — device hostname string
|
|
// metricId — metric identifier string
|
|
// Response: { notes: [{ id, note, created_at, created_by }] }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/notes/:hostname/:metricId', async (req, res) => {
|
|
const { hostname, metricId } = req.params;
|
|
|
|
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
|
if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metricId' });
|
|
|
|
try {
|
|
const notes = await dbAll(db,
|
|
`SELECT cn.id, cn.note, cn.created_at, u.username AS created_by
|
|
FROM compliance_notes cn
|
|
LEFT JOIN users u ON cn.created_by = u.id
|
|
WHERE cn.hostname = ? AND cn.metric_id = ?
|
|
ORDER BY cn.created_at DESC`,
|
|
[hostname, metricId]
|
|
);
|
|
res.json({ notes });
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /notes error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /trends
|
|
// Per-upload active totals + per-team counts for time-series charts.
|
|
// Returns rows ordered ascending by report_date.
|
|
//
|
|
// Response: { trends: [{ report_date, new_count, recurring_count,
|
|
// resolved_count, total_active, STEAM, ACCESS-ENG, ACCESS-OPS,
|
|
// INTELDEV }] }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/trends', async (req, res) => {
|
|
try {
|
|
const uploads = await dbAll(db,
|
|
`SELECT id, report_date,
|
|
COALESCE(new_count, 0) AS new_count,
|
|
COALESCE(recurring_count, 0) AS recurring_count,
|
|
COALESCE(resolved_count, 0) AS resolved_count,
|
|
COALESCE(new_count, 0) + COALESCE(recurring_count, 0) AS total_active
|
|
FROM compliance_uploads
|
|
ORDER BY report_date ASC`
|
|
);
|
|
|
|
if (uploads.length === 0) return res.json({ trends: [] });
|
|
|
|
// Per-team active counts — items whose upload_id matches the upload
|
|
// (recurring items have upload_id bumped each cycle, so this is accurate)
|
|
const teamRows = await dbAll(db,
|
|
`SELECT ci.upload_id, ci.team, COUNT(ci.id) AS count
|
|
FROM compliance_items ci
|
|
WHERE ci.team IS NOT NULL
|
|
GROUP BY ci.upload_id, ci.team`
|
|
);
|
|
|
|
const teamMap = {};
|
|
teamRows.forEach(r => {
|
|
if (!teamMap[r.upload_id]) teamMap[r.upload_id] = {};
|
|
teamMap[r.upload_id][r.team] = r.count;
|
|
});
|
|
|
|
const trends = uploads.map(u => ({
|
|
report_date: u.report_date,
|
|
new_count: u.new_count,
|
|
recurring_count: u.recurring_count,
|
|
resolved_count: u.resolved_count,
|
|
total_active: u.total_active,
|
|
STEAM: teamMap[u.id]?.STEAM || 0,
|
|
'ACCESS-ENG': teamMap[u.id]?.['ACCESS-ENG'] || 0,
|
|
'ACCESS-OPS': teamMap[u.id]?.['ACCESS-OPS'] || 0,
|
|
INTELDEV: teamMap[u.id]?.INTELDEV || 0,
|
|
}));
|
|
|
|
res.json({ trends });
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /trends error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /mttr
|
|
// Mean time to resolution (calendar days) per team, for resolved items.
|
|
//
|
|
// Response: { mttr: [{ team, avg_days, resolved_count }] }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/mttr', async (req, res) => {
|
|
try {
|
|
const rows = await dbAll(db,
|
|
`SELECT
|
|
ci.team,
|
|
ROUND(AVG(JULIANDAY(ru.report_date) - JULIANDAY(fu.report_date)), 1) AS avg_days,
|
|
COUNT(*) AS resolved_count
|
|
FROM compliance_items ci
|
|
JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
|
JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
|
|
WHERE ci.resolved_upload_id IS NOT NULL
|
|
AND fu.report_date IS NOT NULL
|
|
AND ru.report_date IS NOT NULL
|
|
GROUP BY ci.team
|
|
ORDER BY avg_days DESC`
|
|
);
|
|
res.json({ mttr: rows });
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /mttr error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /top-recurring
|
|
// Active findings grouped by team + metric_id, sorted by seen_count desc.
|
|
// Identifies chronic compliance gaps that keep reappearing.
|
|
//
|
|
// Response: { items: [{ team, metric_id, metric_desc, seen_count,
|
|
// host_count }] } — limited to top 20
|
|
// -----------------------------------------------------------------------
|
|
router.get('/top-recurring', async (req, res) => {
|
|
try {
|
|
const rows = await dbAll(db,
|
|
`SELECT team, metric_id, metric_desc, seen_count, COUNT(*) AS host_count
|
|
FROM compliance_items
|
|
WHERE status = 'active'
|
|
GROUP BY team, metric_id
|
|
ORDER BY seen_count DESC, host_count DESC
|
|
LIMIT 20`
|
|
);
|
|
res.json({ items: rows });
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /top-recurring error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
// -----------------------------------------------------------------------
|
|
// GET /category-trend
|
|
// Active item counts per category per upload, for stacked area chart.
|
|
//
|
|
// Response: { categoryTrend: [{ report_date, category, count }] }
|
|
// -----------------------------------------------------------------------
|
|
router.get('/category-trend', async (req, res) => {
|
|
try {
|
|
const rows = await dbAll(db,
|
|
`SELECT cu.report_date, COALESCE(ci.category, 'Unknown') AS category, COUNT(ci.id) AS count
|
|
FROM compliance_uploads cu
|
|
JOIN compliance_items ci ON ci.upload_id = cu.id
|
|
GROUP BY cu.id, category
|
|
ORDER BY cu.report_date ASC`
|
|
);
|
|
res.json({ categoryTrend: rows });
|
|
} catch (err) {
|
|
console.error('[Compliance] GET /category-trend error:', err.message);
|
|
res.status(500).json({ error: 'Database error' });
|
|
}
|
|
});
|
|
|
|
return router;
|
|
}
|
|
|
|
module.exports = createComplianceRouter;
|