jramos/cve-dashboard
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add admin page overhaul and compliance schema drift check specs, compliance upload improvements, drift checker helper

Browse Source
This commit is contained in:
root
2026-04-20 20:12:12 +00:00
parent 6082721452
commit 043c85cc69
20 changed files with 56814 additions and 59 deletions
Download Patch File Download Diff File Expand all files Collapse all files

303
backend/routes/compliance.js
View File

@@ -2,25 +2,35 @@
// Handles xlsx upload/parse, non-compliant item history, and notes.
//
// Endpoints:
// POST /preview — parse xlsx, compute diff vs DB, return summary (no DB write)
// POST /commit — commit a previewed upload to DB
// GET /uploads — list all uploads
// GET /summary — metric health cards for a team (from latest upload)
// GET /items — non-compliant devices grouped by hostname (?team=X&status=active)
// GET /items/:hostname — detail panel: all metrics + notes + upload history for a device
// POST /notes — add a note to one or more (hostname, metric_id) pairs
// POST /preview — parse xlsx, run drift check, compute diff (no DB write)
// POST /reconcile-config — patch compliance_config.json to resolve drift findings
// POST /commit — commit a previewed upload to DB
// GET /uploads — list all uploads
// POST /rollback/:uploadId — roll back the most recent upload (Admin only)
// GET /summary — metric health cards for a team (from latest upload)
// GET /items — non-compliant devices grouped by hostname (?team=X&status=active)
// GET /items/:hostname — detail panel: all metrics + notes + upload history for a device
// POST /notes — add a note to one or more (hostname, metric_id) pairs
// GET /notes/:hostname/:metricId — notes for a specific device+metric
// GET /trends — per-upload totals + per-team counts for time-series charts
// GET /mttr — mean time to resolution per team
// GET /top-recurring — chronic compliance gaps sorted by seen_count
// GET /category-trend — active counts per category per upload for stacked area chart
const express = require('express');
const path = require('path');
const fs = require('fs');
const crypto = require('crypto');
const { spawn } = require('child_process');
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
const logAudit = require('../helpers/auditLog');
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
const SCHEMA_SCRIPT = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
const CONFIG_PATH = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
// ---------------------------------------------------------------------------
// DB helpers
@@ -63,6 +73,25 @@ function parseXlsx(filePath) {
});
}
// ---------------------------------------------------------------------------
// Run Python schema extractor, return xlsx schema object
// ---------------------------------------------------------------------------
function extractXlsxSchema(filePath) {
return new Promise((resolve, reject) => {
const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
let out = '';
let err = '';
py.stdout.on('data', d => { out += d; });
py.stderr.on('data', d => { err += d; });
py.on('close', code => {
if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
try { resolve(JSON.parse(out)); }
catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
});
py.on('error', reject);
});
}
// ---------------------------------------------------------------------------
// Validate that a temp file path is safely within uploads/temp/
// ---------------------------------------------------------------------------
@@ -228,6 +257,15 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// POST /preview
// Parse the uploaded xlsx, compute diff, save parsed data to a temp JSON.
// Returns diff counts + tempFile path for the commit step.
//
// Body: multipart/form-data with `file` field (xlsx)
// Response: {
// drift: { breaking: [], silent_miss: [], cosmetic: [] } | null,
// drift_error: string | null,
// diff: { new_count, recurring_count, resolved_count },
// tempFile: string, filename: string,
// report_date: string, total_items: number
// }
// -----------------------------------------------------------------------
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
upload.single('file')(req, res, async (uploadErr) => {
@@ -243,6 +281,31 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
}
try {
// --- Drift check: load config, extract schema, compare ---
let drift = null;
let drift_error = null;
let config;
try {
config = loadConfig(CONFIG_PATH);
} catch (configErr) {
fs.unlink(req.file.path, () => {});
return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
}
let xlsxSchema = null;
try {
xlsxSchema = await extractXlsxSchema(req.file.path);
if (xlsxSchema.error) {
throw new Error(xlsxSchema.error);
}
drift = compareSchemaToDrift(xlsxSchema, config);
} catch (driftErr) {
drift = null;
drift_error = driftErr.message || 'Drift check failed';
}
// --- Existing parse flow ---
const parsed = await parseXlsx(req.file.path);
if (parsed.error) {
@@ -268,6 +331,9 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
fs.unlink(req.file.path, () => {});
res.json({
drift,
drift_error,
schema: xlsxSchema,
diff: {
new_count: diff.newCount,
recurring_count: diff.recurringCount,
@@ -287,10 +353,63 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
});
});
// -----------------------------------------------------------------------
// POST /reconcile-config
// Admin-only. Patches compliance_config.json to resolve breaking and
// silent-miss drift findings, then re-runs the drift check and returns
// the updated report. Logs every change to the audit trail.
//
// Body: { drift: { breaking: [...], silent_miss: [...] } }
// Response: { changes: [{ action, key, value, detail }], message: string }
// -----------------------------------------------------------------------
router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
const { drift, schema } = req.body;
if (!drift || typeof drift !== 'object') {
return res.status(400).json({ error: 'drift report is required in request body' });
}
const hasFindings = (drift.breaking && drift.breaking.length > 0) ||
(drift.silent_miss && drift.silent_miss.length > 0);
if (!hasFindings) {
return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });
}
try {
const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);
if (changes.length === 0) {
return res.json({ changes: [], message: 'No changes needed' });
}
// Audit log each change
for (const change of changes) {
logAudit(db, {
userId: req.user.id,
username: req.user.username,
action: 'compliance_config_reconcile',
entityType: 'compliance_config',
entityId: change.value,
details: { action: change.action, key: change.key, detail: change.detail },
ipAddress: req.ip,
});
}
res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });
} catch (err) {
console.error('[Compliance] Reconcile config error:', err.message);
res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
}
});
// -----------------------------------------------------------------------
// POST /commit
// Commit a previewed upload to the DB.
// Body: { tempFile, filename, report_date }
//
// Body: { tempFile: string, filename: string, report_date: string }
// Response: { upload: { id, filename, report_date, uploaded_at,
// new_count, resolved_count, recurring_count } }
// -----------------------------------------------------------------------
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
const { tempFile, filename, report_date } = req.body;
@@ -341,6 +460,9 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// GET /uploads
// List all uploads, most recent first.
//
// Response: { uploads: [{ id, filename, report_date, uploaded_at,
// new_count, resolved_count, recurring_count }] }
// -----------------------------------------------------------------------
router.get('/uploads', async (req, res) => {
try {
@@ -357,9 +479,133 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
}
});
// -----------------------------------------------------------------------
// POST /rollback/:uploadId
// Admin-only. Rolls back a specific upload. Only the most recent upload
// can be rolled back to avoid cascading data integrity issues.
//
// Params: uploadId — integer ID of the upload to roll back
// Response: { message: string, rolled_back: { upload_id, filename,
// report_date, items_deleted, items_reactivated } }
//
// Reversal logic:
// 1. Delete items first seen in this upload (new items)
// 2. Re-activate items resolved by this upload
// 3. Revert recurring items: decrement seen_count, point upload_id
// back to the previous upload
// 4. Delete the upload record
// -----------------------------------------------------------------------
router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
const uploadId = parseInt(req.params.uploadId, 10);
if (isNaN(uploadId)) {
return res.status(400).json({ error: 'Invalid upload ID' });
}
try {
// Verify the upload exists
const upload = await dbGet(db,
`SELECT id, filename, report_date, new_count, resolved_count, recurring_count
FROM compliance_uploads WHERE id = ?`,
[uploadId]
);
if (!upload) {
return res.status(404).json({ error: 'Upload not found' });
}
// Only allow rolling back the most recent upload
const latest = await dbGet(db,
`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
);
if (latest.id !== uploadId) {
return res.status(400).json({
error: 'Only the most recent upload can be rolled back',
latest_upload_id: latest.id
});
}
// Find the previous upload (to restore recurring items' upload_id)
const previousUpload = await dbGet(db,
`SELECT id FROM compliance_uploads WHERE id < ? ORDER BY id DESC LIMIT 1`,
[uploadId]
);
await dbRun(db, 'BEGIN TRANSACTION');
try {
// 1. Delete items that were NEW in this upload
const deleteNew = await dbRun(db,
`DELETE FROM compliance_items WHERE first_seen_upload_id = ? AND upload_id = ?`,
[uploadId, uploadId]
);
// 2. Re-activate items that were RESOLVED by this upload
const reactivate = await dbRun(db,
`UPDATE compliance_items
SET status = 'active', resolved_upload_id = NULL
WHERE resolved_upload_id = ?`,
[uploadId]
);
// 3. Revert RECURRING items: decrement seen_count, restore upload_id
if (previousUpload) {
await dbRun(db,
`UPDATE compliance_items
SET upload_id = ?, seen_count = MAX(seen_count - 1, 1)
WHERE upload_id = ? AND first_seen_upload_id != ?`,
[previousUpload.id, uploadId, uploadId]
);
}
// 4. Delete the upload record
await dbRun(db, `DELETE FROM compliance_uploads WHERE id = ?`, [uploadId]);
await dbRun(db, 'COMMIT');
// Audit log
logAudit(db, {
userId: req.user.id,
username: req.user.username,
action: 'compliance_upload_rollback',
entityType: 'compliance_upload',
entityId: String(uploadId),
details: {
filename: upload.filename,
report_date: upload.report_date,
items_deleted: deleteNew.changes,
items_reactivated: reactivate.changes,
},
ipAddress: req.ip,
});
res.json({
message: `Rolled back upload "${upload.filename}"`,
rolled_back: {
upload_id: uploadId,
filename: upload.filename,
report_date: upload.report_date,
items_deleted: deleteNew.changes,
items_reactivated: reactivate.changes,
},
});
} catch (err) {
await dbRun(db, 'ROLLBACK').catch(() => {});
throw err;
}
} catch (err) {
console.error('[Compliance] Rollback error:', err.message);
res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
}
});
// -----------------------------------------------------------------------
// GET /summary?team=STEAM
// Return metric health rows for a team from the latest upload's summary_json.
//
// Query: team — optional, one of ALLOWED_TEAMS
// Response: { entries: [...], overall_scores: {}, upload: { id,
// report_date, uploaded_at } | null }
// -----------------------------------------------------------------------
router.get('/summary', async (req, res) => {
const team = req.query.team;
@@ -403,6 +649,12 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// GET /items?team=STEAM&status=active
// Return non-compliant devices grouped by hostname.
//
// Query: team — required, one of ALLOWED_TEAMS
// status — optional, 'active' (default) or 'resolved'
// Response: { devices: [{ hostname, ip_address, device_type, team,
// status, failing_metrics, seen_count, first_seen, last_seen,
// resolved_on, has_notes }], team, status }
// -----------------------------------------------------------------------
router.get('/items', async (req, res) => {
const { team, status = 'active' } = req.query;
@@ -448,6 +700,12 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// GET /items/:hostname
// Detail panel: all metric rows for this hostname + notes + upload history.
//
// Params: hostname — device hostname string
// Response: { hostname, ip_address, device_type, team,
// metrics: [{ metric_id, metric_desc, category, status, seen_count,
// extra, first_seen, last_seen, resolved_on, ... }],
// notes: [{ id, metric_id, note, group_id, created_at, created_by }] }
// -----------------------------------------------------------------------
router.get('/items/:hostname', async (req, res) => {
const hostname = req.params.hostname;
@@ -519,7 +777,11 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// POST /notes
// Add a note to one or more (hostname, metric_id) pairs.
// Body: { hostname, metric_ids: [...], note } — or legacy { hostname, metric_id, note }
//
// Body: { hostname: string, metric_ids: string[], note: string }
// — or legacy: { hostname: string, metric_id: string, note: string }
// Response: { notes: [{ id, hostname, metric_id, note, group_id,
// created_at, created_by }] }
// -----------------------------------------------------------------------
router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
const { hostname, metric_id, metric_ids, note } = req.body;
@@ -602,6 +864,10 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// GET /notes/:hostname/:metricId
// Return all notes for a (hostname, metric_id) pair.
//
// Params: hostname — device hostname string
// metricId — metric identifier string
// Response: { notes: [{ id, note, created_at, created_by }] }
// -----------------------------------------------------------------------
router.get('/notes/:hostname/:metricId', async (req, res) => {
const { hostname, metricId } = req.params;
@@ -629,6 +895,10 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// GET /trends
// Per-upload active totals + per-team counts for time-series charts.
// Returns rows ordered ascending by report_date.
//
// Response: { trends: [{ report_date, new_count, recurring_count,
// resolved_count, total_active, STEAM, ACCESS-ENG, ACCESS-OPS,
// INTELDEV }] }
// -----------------------------------------------------------------------
router.get('/trends', async (req, res) => {
try {
@@ -681,6 +951,8 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// GET /mttr
// Mean time to resolution (calendar days) per team, for resolved items.
//
// Response: { mttr: [{ team, avg_days, resolved_count }] }
// -----------------------------------------------------------------------
router.get('/mttr', async (req, res) => {
try {
@@ -709,6 +981,9 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// GET /top-recurring
// Active findings grouped by team + metric_id, sorted by seen_count desc.
// Identifies chronic compliance gaps that keep reappearing.
//
// Response: { items: [{ team, metric_id, metric_desc, seen_count,
// host_count }] } — limited to top 20
// -----------------------------------------------------------------------
router.get('/top-recurring', async (req, res) => {
try {
@@ -730,6 +1005,8 @@ function createComplianceRouter(db, upload, requireAuth, requireGroup) {
// -----------------------------------------------------------------------
// GET /category-trend
// Active item counts per category per upload, for stacked area chart.
//
// Response: { categoryTrend: [{ report_date, category, count }] }
// -----------------------------------------------------------------------
router.get('/category-trend', async (req, res) => {
try {