diff --git a/.gitignore b/.gitignore index 3b7f34b..002823f 100644 --- a/.gitignore +++ b/.gitignore @@ -39,6 +39,10 @@ frontend.pid backend/uploads/temp/ feature_request*.md +# Planning docs +docs/aeo-compliance-ui-plan.md +docs/aeo-compliance-wireframe.md + # AI tooling config .claude/ ai_notes.md diff --git a/backend/migrations/add_compliance_tables.js b/backend/migrations/add_compliance_tables.js new file mode 100644 index 0000000..04c8156 --- /dev/null +++ b/backend/migrations/add_compliance_tables.js @@ -0,0 +1,108 @@ +// Migration: Add compliance_uploads, compliance_items, compliance_notes tables +const sqlite3 = require('sqlite3').verbose(); +const path = require('path'); + +const dbPath = path.join(__dirname, '..', 'cve_database.db'); +const db = new sqlite3.Database(dbPath); + +console.log('Starting add_compliance_tables migration...'); + +db.serialize(() => { + // Each xlsx upload — one row per file ingested + db.run(` + CREATE TABLE IF NOT EXISTS compliance_uploads ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + filename TEXT NOT NULL, + report_date TEXT, + uploaded_by INTEGER, + uploaded_at DATETIME DEFAULT CURRENT_TIMESTAMP, + new_count INTEGER DEFAULT 0, + resolved_count INTEGER DEFAULT 0, + recurring_count INTEGER DEFAULT 0, + FOREIGN KEY (uploaded_by) REFERENCES users(id) ON DELETE SET NULL + ) + `, (err) => { + if (err) console.error('Error creating compliance_uploads:', err); + else console.log('✓ compliance_uploads created'); + }); + + // One row per non-compliant asset per metric per upload. + // hostname + metric_id is the stable identity key used to link history and notes. + db.run(` + CREATE TABLE IF NOT EXISTS compliance_items ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + upload_id INTEGER NOT NULL, + hostname TEXT NOT NULL, + ip_address TEXT, + device_type TEXT, + team TEXT, + metric_id TEXT NOT NULL, + metric_desc TEXT, + category TEXT, + extra_json TEXT, + status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active', 'resolved')), + first_seen_upload_id INTEGER, + resolved_upload_id INTEGER, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (upload_id) REFERENCES compliance_uploads(id) ON DELETE CASCADE, + FOREIGN KEY (first_seen_upload_id) REFERENCES compliance_uploads(id) ON DELETE SET NULL, + FOREIGN KEY (resolved_upload_id) REFERENCES compliance_uploads(id) ON DELETE SET NULL + ) + `, (err) => { + if (err) console.error('Error creating compliance_items:', err); + else console.log('✓ compliance_items created'); + }); + + db.run(` + CREATE INDEX IF NOT EXISTS idx_compliance_items_upload + ON compliance_items(upload_id) + `, (err) => { + if (err) console.error('Error creating upload index:', err); + else console.log('✓ idx_compliance_items_upload created'); + }); + + db.run(` + CREATE INDEX IF NOT EXISTS idx_compliance_items_identity + ON compliance_items(hostname, metric_id) + `, (err) => { + if (err) console.error('Error creating identity index:', err); + else console.log('✓ idx_compliance_items_identity created'); + }); + + db.run(` + CREATE INDEX IF NOT EXISTS idx_compliance_items_team_status + ON compliance_items(team, status) + `, (err) => { + if (err) console.error('Error creating team/status index:', err); + else console.log('✓ idx_compliance_items_team_status created'); + }); + + // Notes keyed on (hostname, metric_id) — persists across uploads. + // Each note is its own row so history is preserved. + db.run(` + CREATE TABLE IF NOT EXISTS compliance_notes ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT NOT NULL, + metric_id TEXT NOT NULL, + note TEXT NOT NULL, + created_by INTEGER, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP, + FOREIGN KEY (created_by) REFERENCES users(id) ON DELETE SET NULL + ) + `, (err) => { + if (err) console.error('Error creating compliance_notes:', err); + else console.log('✓ compliance_notes created'); + }); + + db.run(` + CREATE INDEX IF NOT EXISTS idx_compliance_notes_identity + ON compliance_notes(hostname, metric_id) + `, (err) => { + if (err) console.error('Error creating notes identity index:', err); + else console.log('✓ idx_compliance_notes_identity created'); + }); +}); + +db.close(() => { + console.log('Migration complete!'); +}); diff --git a/backend/routes/compliance.js b/backend/routes/compliance.js new file mode 100644 index 0000000..8f02d0c --- /dev/null +++ b/backend/routes/compliance.js @@ -0,0 +1,588 @@ +// Compliance Routes — AEO metric tracking +// Handles xlsx upload/parse, non-compliant item history, and notes. +// +// Endpoints: +// POST /preview — parse xlsx, compute diff vs DB, return summary (no DB write) +// POST /commit — commit a previewed upload to DB +// GET /uploads — list all uploads +// GET /summary — metric health cards for a team (from latest upload) +// GET /items — non-compliant devices grouped by hostname (?team=X&status=active) +// GET /items/:hostname — detail panel: all metrics + notes + upload history for a device +// POST /notes — add a note to a (hostname, metric_id) pair +// GET /notes/:hostname/:metricId — notes for a specific device+metric + +const express = require('express'); +const path = require('path'); +const fs = require('fs'); +const { spawn } = require('child_process'); + +const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py'); +const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp'); +const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']); + +// --------------------------------------------------------------------------- +// DB helpers +// --------------------------------------------------------------------------- +function dbRun(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.run(sql, params, function (err) { + if (err) reject(err); + else resolve({ lastID: this.lastID, changes: this.changes }); + }); + }); +} +function dbGet(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { if (err) reject(err); else resolve(row || null); }); + }); +} +function dbAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { if (err) reject(err); else resolve(rows || []); }); + }); +} + +// --------------------------------------------------------------------------- +// Run Python parser, return parsed object +// --------------------------------------------------------------------------- +function parseXlsx(filePath) { + return new Promise((resolve, reject) => { + const py = spawn('python3', [PARSER_SCRIPT, filePath]); + let out = ''; + let err = ''; + py.stdout.on('data', d => { out += d; }); + py.stderr.on('data', d => { err += d; }); + py.on('close', code => { + if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`)); + try { resolve(JSON.parse(out)); } + catch (e) { reject(new Error('Parser returned invalid JSON')); } + }); + py.on('error', reject); + }); +} + +// --------------------------------------------------------------------------- +// Validate that a temp file path is safely within uploads/temp/ +// --------------------------------------------------------------------------- +function isSafeTempPath(filePath) { + const resolved = path.resolve(filePath); + return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json'; +} + +// --------------------------------------------------------------------------- +// Compute diff: new / recurring / resolved +// --------------------------------------------------------------------------- +async function computeDiff(db, incomingItems) { + const activeRows = await dbAll(db, + `SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'` + ); + const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`)); + const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`)); + + let newCount = 0, recurringCount = 0, resolvedCount = 0; + for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; } + for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; } + + return { newCount, recurringCount, resolvedCount }; +} + +// --------------------------------------------------------------------------- +// Write a parsed upload to the DB (within a transaction) +// --------------------------------------------------------------------------- +async function persistUpload(db, { items, summary, reportDate, filename, userId }) { + // Pull current active items before we modify anything + const activeRows = await dbAll(db, + `SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'` + ); + const activeMap = {}; + activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; }); + + const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`)); + + await dbRun(db, 'BEGIN TRANSACTION'); + try { + // 1. Insert the upload record + const { lastID: uploadId } = await dbRun(db, + `INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json) + VALUES (?, ?, ?, datetime('now'), ?)`, + [filename, reportDate || null, userId || null, JSON.stringify(summary)] + ); + + let newCount = 0, recurringCount = 0, resolvedCount = 0; + + // 2. Upsert each incoming non-compliant item + for (const item of items) { + const key = `${item.hostname}|||${item.metric_id}`; + const existing = activeMap[key]; + const extraStr = JSON.stringify(item.extra_json || {}); + + if (existing) { + // Recurring — bump seen_count, refresh snapshot fields + await dbRun(db, + `UPDATE compliance_items + SET upload_id = ?, seen_count = ?, ip_address = ?, device_type = ?, extra_json = ? + WHERE id = ?`, + [uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id] + ); + recurringCount++; + } else { + // New item (or previously resolved and re-appearing) + await dbRun(db, + `INSERT INTO compliance_items + (upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc, + category, extra_json, status, first_seen_upload_id, seen_count) + VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?, 1)`, + [uploadId, item.hostname, item.ip_address, item.device_type, item.team, + item.metric_id, item.metric_desc, item.category, extraStr, uploadId] + ); + newCount++; + } + } + + // 3. Mark items not present in this upload as resolved + for (const [key, row] of Object.entries(activeMap)) { + if (!newKeys.has(key)) { + await dbRun(db, + `UPDATE compliance_items + SET status = 'resolved', resolved_upload_id = ? + WHERE id = ?`, + [uploadId, row.id] + ); + resolvedCount++; + } + } + + // 4. Update upload with final counts + await dbRun(db, + `UPDATE compliance_uploads + SET new_count = ?, resolved_count = ?, recurring_count = ? + WHERE id = ?`, + [newCount, resolvedCount, recurringCount, uploadId] + ); + + await dbRun(db, 'COMMIT'); + return { uploadId, newCount, recurringCount, resolvedCount }; + + } catch (err) { + await dbRun(db, 'ROLLBACK').catch(() => {}); + throw err; + } +} + +// --------------------------------------------------------------------------- +// Group flat compliance_items rows into per-device objects +// --------------------------------------------------------------------------- +function groupByHostname(rows, noteHostnames) { + const deviceMap = {}; + + for (const row of rows) { + if (!deviceMap[row.hostname]) { + deviceMap[row.hostname] = { + hostname: row.hostname, + ip_address: row.ip_address || '', + device_type: row.device_type || '', + team: row.team || '', + status: row.status, + failing_metrics: [], + seen_count: row.seen_count || 1, + first_seen: row.first_seen || null, + last_seen: row.last_seen || null, + resolved_on: row.resolved_on || null, + has_notes: noteHostnames.has(row.hostname), + }; + } + + const dev = deviceMap[row.hostname]; + dev.failing_metrics.push({ + metric_id: row.metric_id, + metric_desc: row.metric_desc || '', + category: row.category || '', + }); + // Use the highest seen_count and earliest first_seen across all metrics + if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count; + if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen)) { + dev.first_seen = row.first_seen; + } + } + + return Object.values(deviceMap); +} + +// --------------------------------------------------------------------------- +// Router factory +// --------------------------------------------------------------------------- +function createComplianceRouter(db, upload, requireAuth, requireRole) { + const router = express.Router(); + + // Idempotent column additions — errors mean column already exists, which is fine + db.run(`ALTER TABLE compliance_items ADD COLUMN seen_count INTEGER DEFAULT 1`, () => {}); + db.run(`ALTER TABLE compliance_uploads ADD COLUMN summary_json TEXT`, () => {}); + + // All compliance routes require authentication + router.use(requireAuth(db)); + + // ----------------------------------------------------------------------- + // POST /preview + // Parse the uploaded xlsx, compute diff, save parsed data to a temp JSON. + // Returns diff counts + tempFile path for the commit step. + // ----------------------------------------------------------------------- + router.post('/preview', requireRole('editor', 'admin'), (req, res) => { + upload.single('file')(req, res, async (uploadErr) => { + if (uploadErr) { + return res.status(400).json({ error: uploadErr.message }); + } + if (!req.file) { + return res.status(400).json({ error: 'No file uploaded' }); + } + if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') { + fs.unlink(req.file.path, () => {}); + return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' }); + } + + try { + const parsed = await parseXlsx(req.file.path); + + if (parsed.error) { + fs.unlink(req.file.path, () => {}); + return res.status(422).json({ error: parsed.error }); + } + + const diff = await computeDiff(db, parsed.items); + + // Save parsed data to temp JSON — the commit step reads this + if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true }); + const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`; + const tempFilePath = path.join(TEMP_DIR, tempFilename); + + fs.writeFileSync(tempFilePath, JSON.stringify({ + items: parsed.items, + summary: parsed.summary, + report_date: parsed.report_date, + filename: req.file.originalname, + })); + + // Delete the original xlsx from temp (we only need the JSON now) + fs.unlink(req.file.path, () => {}); + + res.json({ + diff: { + new_count: diff.newCount, + recurring_count: diff.recurringCount, + resolved_count: diff.resolvedCount, + }, + tempFile: tempFilePath, + filename: req.file.originalname, + report_date: parsed.report_date, + total_items: parsed.total, + }); + + } catch (err) { + fs.unlink(req.file.path, () => {}); + console.error('[Compliance] Preview error:', err.message); + res.status(500).json({ error: 'Failed to parse file: ' + err.message }); + } + }); + }); + + // ----------------------------------------------------------------------- + // POST /commit + // Commit a previewed upload to the DB. + // Body: { tempFile, filename, report_date } + // ----------------------------------------------------------------------- + router.post('/commit', requireRole('editor', 'admin'), async (req, res) => { + const { tempFile, filename, report_date } = req.body; + + if (!tempFile || typeof tempFile !== 'string') { + return res.status(400).json({ error: 'tempFile is required' }); + } + if (!isSafeTempPath(tempFile)) { + return res.status(400).json({ error: 'Invalid tempFile path' }); + } + if (!fs.existsSync(tempFile)) { + return res.status(400).json({ error: 'Preview session expired — please upload again' }); + } + + let parsed; + try { + parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8')); + } catch { + return res.status(400).json({ error: 'Could not read preview data — please upload again' }); + } + + try { + const result = await persistUpload(db, { + items: parsed.items, + summary: parsed.summary, + reportDate: report_date || parsed.report_date, + filename: filename || parsed.filename, + userId: req.user?.id || null, + }); + + fs.unlink(tempFile, () => {}); + + const upload = await dbGet(db, + `SELECT id, filename, report_date, uploaded_at, + new_count, resolved_count, recurring_count + FROM compliance_uploads WHERE id = ?`, + [result.uploadId] + ); + + res.json({ upload }); + + } catch (err) { + console.error('[Compliance] Commit error:', err.message); + res.status(500).json({ error: 'Failed to commit upload: ' + err.message }); + } + }); + + // ----------------------------------------------------------------------- + // GET /uploads + // List all uploads, most recent first. + // ----------------------------------------------------------------------- + router.get('/uploads', async (req, res) => { + try { + const rows = await dbAll(db, + `SELECT id, filename, report_date, uploaded_at, + new_count, resolved_count, recurring_count + FROM compliance_uploads + ORDER BY id DESC` + ); + res.json({ uploads: rows }); + } catch (err) { + console.error('[Compliance] GET /uploads error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /summary?team=STEAM + // Return metric health rows for a team from the latest upload's summary_json. + // ----------------------------------------------------------------------- + router.get('/summary', async (req, res) => { + const team = req.query.team; + if (team && !ALLOWED_TEAMS.has(team)) { + return res.status(400).json({ error: 'Invalid team' }); + } + + try { + const latestUpload = await dbGet(db, + `SELECT id, summary_json, report_date, uploaded_at + FROM compliance_uploads ORDER BY id DESC LIMIT 1` + ); + if (!latestUpload || !latestUpload.summary_json) { + return res.json({ entries: [], overall_scores: {}, upload: null }); + } + + let summary; + try { summary = JSON.parse(latestUpload.summary_json); } + catch { return res.json({ entries: [], overall_scores: {}, upload: null }); } + + let entries = summary.entries || []; + if (team) { + entries = entries.filter(e => e.team === team); + } + + res.json({ + entries, + overall_scores: summary.overall_scores || {}, + upload: { + id: latestUpload.id, + report_date: latestUpload.report_date, + uploaded_at: latestUpload.uploaded_at, + }, + }); + } catch (err) { + console.error('[Compliance] GET /summary error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /items?team=STEAM&status=active + // Return non-compliant devices grouped by hostname. + // ----------------------------------------------------------------------- + router.get('/items', async (req, res) => { + const { team, status = 'active' } = req.query; + + if (!team) return res.status(400).json({ error: 'team is required' }); + if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' }); + if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' }); + + try { + const rows = await dbAll(db, + `SELECT + ci.hostname, ci.ip_address, ci.device_type, ci.team, + ci.metric_id, ci.metric_desc, ci.category, + ci.status, ci.seen_count, + fu.report_date AS first_seen, + lu.report_date AS last_seen, + ru.report_date AS resolved_on + FROM compliance_items ci + LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id + LEFT JOIN compliance_uploads lu ON ci.upload_id = fu.id + LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id + WHERE ci.team = ? AND ci.status = ? + ORDER BY ci.hostname, ci.metric_id`, + [team, status] + ); + + // Fetch hostnames that have any notes (for the has_notes indicator) + const noteRows = await dbAll(db, + `SELECT DISTINCT hostname FROM compliance_notes` + ); + const noteHostnames = new Set(noteRows.map(r => r.hostname)); + + const devices = groupByHostname(rows, noteHostnames); + + res.json({ devices, team, status }); + + } catch (err) { + console.error('[Compliance] GET /items error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /items/:hostname + // Detail panel: all metric rows for this hostname + notes + upload history. + // ----------------------------------------------------------------------- + router.get('/items/:hostname', async (req, res) => { + const hostname = req.params.hostname; + if (!hostname || hostname.length > 300) { + return res.status(400).json({ error: 'Invalid hostname' }); + } + + try { + // All metric rows for this hostname + const metricRows = await dbAll(db, + `SELECT + ci.metric_id, ci.metric_desc, ci.category, ci.status, + ci.ip_address, ci.device_type, ci.team, + ci.seen_count, ci.extra_json, + fu.report_date AS first_seen, + fu.uploaded_at AS first_seen_at, + lu.report_date AS last_seen, + lu.uploaded_at AS last_seen_at, + ru.report_date AS resolved_on + FROM compliance_items ci + LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id + LEFT JOIN compliance_uploads lu ON ci.upload_id = fu.id + LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id + WHERE ci.hostname = ? + ORDER BY ci.status DESC, ci.metric_id`, + [hostname] + ); + + if (metricRows.length === 0) { + return res.status(404).json({ error: 'Device not found' }); + } + + // Parse extra_json on each row + const metrics = metricRows.map(r => ({ + ...r, + extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(), + extra_json: undefined, + })); + + // Notes (all metrics for this hostname, sorted newest first) + const notes = await dbAll(db, + `SELECT cn.id, cn.metric_id, cn.note, cn.created_at, + u.username AS created_by + FROM compliance_notes cn + LEFT JOIN users u ON cn.created_by = u.id + WHERE cn.hostname = ? + ORDER BY cn.created_at DESC`, + [hostname] + ); + + // Derive device identity from the first active row, else any row + const identity = metricRows.find(r => r.status === 'active') || metricRows[0]; + + res.json({ + hostname, + ip_address: identity.ip_address || '', + device_type: identity.device_type || '', + team: identity.team || '', + metrics, + notes, + }); + + } catch (err) { + console.error('[Compliance] GET /items/:hostname error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + // ----------------------------------------------------------------------- + // POST /notes + // Add a note to a (hostname, metric_id) pair. + // Body: { hostname, metric_id, note } + // ----------------------------------------------------------------------- + router.post('/notes', async (req, res) => { + const { hostname, metric_id, note } = req.body; + + if (!hostname || typeof hostname !== 'string' || hostname.length > 300) { + return res.status(400).json({ error: 'Invalid hostname' }); + } + if (!metric_id || typeof metric_id !== 'string' || metric_id.length > 50) { + return res.status(400).json({ error: 'Invalid metric_id' }); + } + const noteText = String(note || '').trim().slice(0, 1000); + if (!noteText) { + return res.status(400).json({ error: 'Note cannot be empty' }); + } + + try { + const { lastID } = await dbRun(db, + `INSERT INTO compliance_notes (hostname, metric_id, note, created_by, created_at) + VALUES (?, ?, ?, ?, datetime('now'))`, + [hostname, metric_id, noteText, req.user?.id || null] + ); + + const created = await dbGet(db, + `SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.created_at, + u.username AS created_by + FROM compliance_notes cn + LEFT JOIN users u ON cn.created_by = u.id + WHERE cn.id = ?`, + [lastID] + ); + + res.status(201).json(created); + + } catch (err) { + console.error('[Compliance] POST /notes error:', err.message); + res.status(500).json({ error: 'Failed to save note' }); + } + }); + + // ----------------------------------------------------------------------- + // GET /notes/:hostname/:metricId + // Return all notes for a (hostname, metric_id) pair. + // ----------------------------------------------------------------------- + router.get('/notes/:hostname/:metricId', async (req, res) => { + const { hostname, metricId } = req.params; + + if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' }); + if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metricId' }); + + try { + const notes = await dbAll(db, + `SELECT cn.id, cn.note, cn.created_at, u.username AS created_by + FROM compliance_notes cn + LEFT JOIN users u ON cn.created_by = u.id + WHERE cn.hostname = ? AND cn.metric_id = ? + ORDER BY cn.created_at DESC`, + [hostname, metricId] + ); + res.json({ notes }); + } catch (err) { + console.error('[Compliance] GET /notes error:', err.message); + res.status(500).json({ error: 'Database error' }); + } + }); + + return router; +} + +module.exports = createComplianceRouter; diff --git a/backend/scripts/parse_compliance_xlsx.py b/backend/scripts/parse_compliance_xlsx.py new file mode 100644 index 0000000..eb7f7dd --- /dev/null +++ b/backend/scripts/parse_compliance_xlsx.py @@ -0,0 +1,212 @@ +#!/usr/bin/env python3 +""" +Parse NTS_AEO compliance xlsx file and write JSON to stdout. +Usage: python3 parse_compliance_xlsx.py + +Output: +{ + "items": [...], # non-compliant asset rows + "summary": { ... }, # metric health data from Summary sheet + "report_date": "YYYY-MM-DD" | null, + "total": int +} +""" +import sys +import json +import re +import pandas as pd +from pathlib import Path + +METRIC_CATEGORIES = { + '2.3.4i': 'Vulnerability Management', + '2.3.6i': 'Vulnerability Management', + '2.3.8i': 'Vulnerability Management', + '5.2.4': 'Access & MFA', + '5.2.5': 'Access & MFA', + '5.2.6': 'Access & MFA', + '5.3.4': 'Endpoint Protection', + '5.5.2': 'End-of-Life OS', + '5.5.4i': 'Vulnerability Management', + '5.5.5': 'Decommissioned Assets', + '5.8.1': 'Application Security', + '7.1.1': 'Logging & Monitoring', + '7.6.13': 'Disaster Recovery', + '7.6.16': 'Disaster Recovery', + 'Missing_AppID': 'Asset Data Quality', + 'Missing_DF': 'Asset Data Quality', + 'Missing_OS': 'Asset Data Quality', +} + +# Columns that go into the main item fields — everything else becomes extra_json +CORE_COLS = { + 'Preferred - Hostname', 'GRANITE - IPv4_Address', 'GRANITE - Type', + 'Team', 'Compliant', 'Source_Network', 'Vertical', + 'GRANITE - Equip_Inst_ID', 'GRANITE - RESPONSIBLE_TEAM', +} + +SKIP_SHEETS = {'Summary', 'CMDB_9box'} + + +def safe_str(val): + s = str(val).strip() + return '' if s == 'nan' else s + + +def parse_summary(xl): + """Return { entries: [...], overall_scores: { customer_network, vertical } }""" + df_raw = pd.read_excel(xl, sheet_name='Summary', header=None) + + overall_scores = { + 'customer_network': float(df_raw.iloc[0, 4]) if pd.notna(df_raw.iloc[0, 4]) else None, + 'vertical': float(df_raw.iloc[1, 4]) if pd.notna(df_raw.iloc[1, 4]) else None, + } + + df = pd.read_excel(xl, sheet_name='Summary', header=3) + # Flatten any newlines in column names + df.columns = [str(c).replace('\n', ' ').strip() for c in df.columns] + + # Locate the sub-vertical/team column robustly + team_col = next((c for c in df.columns if 'Sub-Vertical' in c or 'Purchase Group' in c), None) + + entries = [] + for _, row in df.iterrows(): + metric_id = safe_str(row.get('Metric', '')) + if not metric_id or metric_id in ('Metric',): + continue + + team = safe_str(row.get(team_col, '')) if team_col else '' + + try: + non_compliant = int(row.get('Non-Compliant', 0) or 0) + compliant = int(row.get('Compliant', 0) or 0) + total = int(row.get('Total', 0) or 0) + compliance_pct = float(row.get('Current Compliance', 0) or 0) + target = float(row.get('Metric Target', 0) or 0) + except (ValueError, TypeError): + continue + + entries.append({ + 'metric_id': metric_id, + 'team': team, + 'priority': safe_str(row.get('Priority / Non-Priority / IR', '')), + 'non_compliant': non_compliant, + 'compliant': compliant, + 'total': total, + 'compliance_pct': compliance_pct, + 'target': target, + 'status': safe_str(row.get('Status', '')), + 'description': safe_str(row.get('Metric Description', '')), + 'category': METRIC_CATEGORIES.get(metric_id, 'Other'), + }) + + return {'entries': entries, 'overall_scores': overall_scores} + + +def parse_sheet(xl, sheet_name, summary_entries): + """Return list of non-compliant item dicts for a detail sheet.""" + try: + df = pd.read_excel(xl, sheet_name=sheet_name, header=0) + except Exception: + return [] + + if df.empty: + return [] + + df.columns = [str(c).strip() for c in df.columns] + + # Filter to non-compliant rows when the Compliant column exists + if 'Compliant' in df.columns: + df = df[df['Compliant'] == False] + + if df.empty: + return [] + + # Look up description from summary + metric_desc = '' + for e in summary_entries: + if e['metric_id'] == sheet_name and e['description']: + metric_desc = e['description'] + break + + category = METRIC_CATEGORIES.get(sheet_name, 'Other') + + items = [] + for _, row in df.iterrows(): + hostname = safe_str(row.get('Preferred - Hostname', '')) + if not hostname: + continue + + ip = safe_str(row.get('GRANITE - IPv4_Address', '')) + device_type = safe_str(row.get('GRANITE - Type', '')) + team = safe_str(row.get('Team', '')) + + # Everything non-core goes into extra_json + extra = {} + for col in df.columns: + if col in CORE_COLS: + continue + val = row.get(col) + if pd.isna(val) if not isinstance(val, str) else False: + continue + s = safe_str(val) + if s: + extra[col] = val.isoformat() if hasattr(val, 'isoformat') else s + + items.append({ + 'hostname': hostname, + 'ip_address': ip, + 'device_type': device_type, + 'team': team, + 'metric_id': sheet_name, + 'metric_desc': metric_desc, + 'category': category, + 'extra_json': extra, + }) + + return items + + +def extract_report_date(filepath): + """Try to pull YYYY-MM-DD from the filename, e.g. NTS_AEO_2026_03_25.xlsx""" + stem = Path(filepath).stem + m = re.search(r'(\d{4})_(\d{2})_(\d{2})', stem) + if m: + return f"{m.group(1)}-{m.group(2)}-{m.group(3)}" + return None + + +def main(): + if len(sys.argv) < 2: + print(json.dumps({'error': 'No file path provided'})) + sys.exit(1) + + filepath = sys.argv[1] + + try: + xl = pd.ExcelFile(filepath) + except Exception as e: + print(json.dumps({'error': f'Cannot open file: {str(e)}'})) + sys.exit(1) + + try: + summary = parse_summary(xl) + except Exception as e: + summary = {'entries': [], 'overall_scores': {}, 'parse_error': str(e)} + + all_items = [] + for sheet_name in xl.sheet_names: + if sheet_name in SKIP_SHEETS: + continue + items = parse_sheet(xl, sheet_name, summary.get('entries', [])) + all_items.extend(items) + + print(json.dumps({ + 'items': all_items, + 'summary': summary, + 'report_date': extract_report_date(filepath), + 'total': len(all_items), + })) + + +if __name__ == '__main__': + main() diff --git a/backend/server.js b/backend/server.js index 0cd3059..b51d585 100644 --- a/backend/server.js +++ b/backend/server.js @@ -23,6 +23,7 @@ const createArcherTicketsRouter = require('./routes/archerTickets'); const createIvantiWorkflowsRouter = require('./routes/ivantiWorkflows'); const createIvantiFindingsRouter = require('./routes/ivantiFindings'); const createIvantiTodoQueueRouter = require('./routes/ivantiTodoQueue'); +const createComplianceRouter = require('./routes/compliance'); const app = express(); const PORT = process.env.PORT || 3001; @@ -218,6 +219,9 @@ app.use('/api/ivanti/findings', createIvantiFindingsRouter(db, requireAuth)); // Ivanti queue routes — per-user staging queue for FP / Archer workflows app.use('/api/ivanti/todo-queue', createIvantiTodoQueueRouter(db, requireAuth)); +// AEO compliance routes — xlsx upload, non-compliant item tracking, notes +app.use('/api/compliance', createComplianceRouter(db, upload, requireAuth, requireRole)); + // ========== CVE ENDPOINTS ========== // Get all CVEs with optional filters (authenticated users)