cve-dashboard/backend/routes/compliance.js

// Compliance Routes — AEO metric tracking
// Handles xlsx upload/parse, non-compliant item history, and notes.
//
// Endpoints:
//   POST /preview              — parse xlsx, run drift check, compute diff (no DB write)
//   POST /reconcile-config     — patch compliance_config.json to resolve drift findings
//   POST /commit               — commit a previewed upload to DB
//   GET  /uploads              — list all uploads
//   POST /rollback/:uploadId   — roll back the most recent upload (Admin only)
//   GET  /summary              — metric health cards for a team (from latest upload)
//   GET  /items                — non-compliant devices grouped by hostname (?team=X&status=active)
//   GET  /items/:hostname      — detail panel: all metrics + notes + upload history for a device
//   POST /notes                — add a note to one or more (hostname, metric_id) pairs
//   GET  /notes/:hostname/:metricId — notes for a specific device+metric
//   GET  /trends               — per-upload totals + per-team counts for time-series charts
//   GET  /mttr                 — mean time to resolution per team
//   GET  /top-recurring        — chronic compliance gaps sorted by seen_count
//   GET  /category-trend       — active counts per category per upload for stacked area chart

const express  = require('express');
const path     = require('path');
const fs       = require('fs');
const crypto   = require('crypto');
const { spawn } = require('child_process');
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
const logAudit = require('../helpers/auditLog');

const PARSER_SCRIPT  = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
const SCHEMA_SCRIPT  = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
const CONFIG_PATH    = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
const PYTHON_BIN     = process.env.PYTHON_BIN || 'python3';
const TEMP_DIR       = path.join(process.cwd(), 'uploads', 'temp');
const ALLOWED_TEAMS  = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);

// ---------------------------------------------------------------------------
// DB helpers
// ---------------------------------------------------------------------------
function dbRun(db, sql, params = []) {
    return new Promise((resolve, reject) => {
        db.run(sql, params, function (err) {
            if (err) reject(err);
            else resolve({ lastID: this.lastID, changes: this.changes });
        });
    });
}
function dbGet(db, sql, params = []) {
    return new Promise((resolve, reject) => {
        db.get(sql, params, (err, row) => { if (err) reject(err); else resolve(row || null); });
    });
}
function dbAll(db, sql, params = []) {
    return new Promise((resolve, reject) => {
        db.all(sql, params, (err, rows) => { if (err) reject(err); else resolve(rows || []); });
    });
}

// ---------------------------------------------------------------------------
// Run Python parser, return parsed object
// ---------------------------------------------------------------------------
function parseXlsx(filePath) {
    return new Promise((resolve, reject) => {
        const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
        let out = '';
        let err = '';
        py.stdout.on('data', d => { out += d; });
        py.stderr.on('data', d => { err += d; });
        py.on('close', code => {
            if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
            try { resolve(JSON.parse(out)); }
            catch (e) { reject(new Error('Parser returned invalid JSON')); }
        });
        py.on('error', reject);
    });
}

// ---------------------------------------------------------------------------
// Run Python schema extractor, return xlsx schema object
// ---------------------------------------------------------------------------
function extractXlsxSchema(filePath) {
    return new Promise((resolve, reject) => {
        const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
        let out = '';
        let err = '';
        py.stdout.on('data', d => { out += d; });
        py.stderr.on('data', d => { err += d; });
        py.on('close', code => {
            if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
            try { resolve(JSON.parse(out)); }
            catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
        });
        py.on('error', reject);
    });
}

// ---------------------------------------------------------------------------
// Validate that a temp file path is safely within uploads/temp/
// ---------------------------------------------------------------------------
function isSafeTempPath(filePath) {
    const resolved = path.resolve(filePath);
    return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
}

// ---------------------------------------------------------------------------
// Compute diff: new / recurring / resolved
// ---------------------------------------------------------------------------
async function computeDiff(db, incomingItems) {
    const activeRows = await dbAll(db,
        `SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
    );
    const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
    const newKeys    = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));

    let newCount = 0, recurringCount = 0, resolvedCount = 0;
    for (const k of newKeys)    { if (activeKeys.has(k)) recurringCount++; else newCount++; }
    for (const k of activeKeys) { if (!newKeys.has(k))   resolvedCount++; }

    return { newCount, recurringCount, resolvedCount };
}

// ---------------------------------------------------------------------------
// Write a parsed upload to the DB (within a transaction)
// ---------------------------------------------------------------------------
async function persistUpload(db, { items, summary, reportDate, filename, userId }) {
    // Pull current active items before we modify anything
    const activeRows = await dbAll(db,
        `SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
    );
    const activeMap = {};
    activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });

    const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));

    await dbRun(db, 'BEGIN TRANSACTION');
    try {
        // 1. Insert the upload record
        const { lastID: uploadId } = await dbRun(db,
            `INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
             VALUES (?, ?, ?, datetime('now'), ?)`,
            [filename, reportDate || null, userId || null, JSON.stringify(summary)]
        );

        let newCount = 0, recurringCount = 0, resolvedCount = 0;

        // 2. Upsert each incoming non-compliant item
        for (const item of items) {
            const key      = `${item.hostname}|||${item.metric_id}`;
            const existing = activeMap[key];
            const extraStr = JSON.stringify(item.extra_json || {});

            if (existing) {
                // Recurring — bump seen_count, refresh snapshot fields
                await dbRun(db,
                    `UPDATE compliance_items
                     SET upload_id = ?, seen_count = ?, ip_address = ?, device_type = ?, extra_json = ?
                     WHERE id = ?`,
                    [uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
                );
                recurringCount++;
            } else {
                // New item (or previously resolved and re-appearing)
                await dbRun(db,
                    `INSERT INTO compliance_items
                     (upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
                      category, extra_json, status, first_seen_upload_id, seen_count)
                     VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, 'active', ?, 1)`,
                    [uploadId, item.hostname, item.ip_address, item.device_type, item.team,
                     item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
                );
                newCount++;
            }
        }

        // 3. Mark items not present in this upload as resolved
        for (const [key, row] of Object.entries(activeMap)) {
            if (!newKeys.has(key)) {
                await dbRun(db,
                    `UPDATE compliance_items
                     SET status = 'resolved', resolved_upload_id = ?
                     WHERE id = ?`,
                    [uploadId, row.id]
                );
                resolvedCount++;
            }
        }

        // 4. Update upload with final counts
        await dbRun(db,
            `UPDATE compliance_uploads
             SET new_count = ?, resolved_count = ?, recurring_count = ?
             WHERE id = ?`,
            [newCount, resolvedCount, recurringCount, uploadId]
        );

        await dbRun(db, 'COMMIT');
        return { uploadId, newCount, recurringCount, resolvedCount };

    } catch (err) {
        await dbRun(db, 'ROLLBACK').catch(() => {});
        throw err;
    }
}

// ---------------------------------------------------------------------------
// Group flat compliance_items rows into per-device objects
// ---------------------------------------------------------------------------
function groupByHostname(rows, noteHostnames) {
    const deviceMap = {};

    for (const row of rows) {
        if (!deviceMap[row.hostname]) {
            deviceMap[row.hostname] = {
                hostname:        row.hostname,
                ip_address:      row.ip_address  || '',
                device_type:     row.device_type || '',
                team:            row.team        || '',
                status:          row.status,
                failing_metrics: [],
                seen_count:      row.seen_count  || 1,
                first_seen:      row.first_seen  || null,
                last_seen:       row.last_seen   || null,
                resolved_on:     row.resolved_on || null,
                has_notes:       noteHostnames.has(row.hostname),
            };
        }

        const dev = deviceMap[row.hostname];
        dev.failing_metrics.push({
            metric_id:   row.metric_id,
            metric_desc: row.metric_desc  || '',
            category:    row.category     || '',
        });
        // Use the highest seen_count and earliest first_seen across all metrics
        if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count;
        if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen))
            dev.first_seen = row.first_seen;
        if (row.last_seen  && (!dev.last_seen  || row.last_seen  > dev.last_seen))
            dev.last_seen  = row.last_seen;
    }

    return Object.values(deviceMap);
}

// ---------------------------------------------------------------------------
// Router factory
// ---------------------------------------------------------------------------
function createComplianceRouter(db, upload, requireAuth, requireGroup) {
    const router = express.Router();

    // Idempotent column additions — errors mean column already exists, which is fine
    db.run(`ALTER TABLE compliance_items   ADD COLUMN seen_count   INTEGER DEFAULT 1`, () => {});
    db.run(`ALTER TABLE compliance_uploads ADD COLUMN summary_json TEXT`,              () => {});

    // All compliance routes require authentication
    router.use(requireAuth(db));

    // -----------------------------------------------------------------------
    // POST /preview
    // Parse the uploaded xlsx, compute diff, save parsed data to a temp JSON.
    // Returns diff counts + tempFile path for the commit step.
    //
    // Body: multipart/form-data with `file` field (xlsx)
    // Response: {
    //   drift: { breaking: [], silent_miss: [], cosmetic: [] } | null,
    //   drift_error: string | null,
    //   diff: { new_count, recurring_count, resolved_count },
    //   tempFile: string, filename: string,
    //   report_date: string, total_items: number
    // }
    // -----------------------------------------------------------------------
    router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
        upload.single('file')(req, res, async (uploadErr) => {
            if (uploadErr) {
                return res.status(400).json({ error: uploadErr.message });
            }
            if (!req.file) {
                return res.status(400).json({ error: 'No file uploaded' });
            }
            if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
                fs.unlink(req.file.path, () => {});
                return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
            }

            try {
                // --- Drift check: load config, extract schema, compare ---
                let drift = null;
                let drift_error = null;

                let config;
                try {
                    config = loadConfig(CONFIG_PATH);
                } catch (configErr) {
                    fs.unlink(req.file.path, () => {});
                    return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
                }

                let xlsxSchema = null;
                try {
                    xlsxSchema = await extractXlsxSchema(req.file.path);
                    if (xlsxSchema.error) {
                        throw new Error(xlsxSchema.error);
                    }
                    drift = compareSchemaToDrift(xlsxSchema, config);
                } catch (driftErr) {
                    drift = null;
                    drift_error = driftErr.message || 'Drift check failed';
                }

                // --- Existing parse flow ---
                const parsed = await parseXlsx(req.file.path);

                if (parsed.error) {
                    fs.unlink(req.file.path, () => {});
                    return res.status(422).json({ error: parsed.error });
                }

                const diff = await computeDiff(db, parsed.items);

                // Save parsed data to temp JSON — the commit step reads this
                if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
                const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
                const tempFilePath = path.join(TEMP_DIR, tempFilename);

                fs.writeFileSync(tempFilePath, JSON.stringify({
                    items:       parsed.items,
                    summary:     parsed.summary,
                    report_date: parsed.report_date,
                    filename:    req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
                }));

                // Delete the original xlsx from temp (we only need the JSON now)
                fs.unlink(req.file.path, () => {});

                res.json({
                    drift,
                    drift_error,
                    schema: xlsxSchema,
                    diff: {
                        new_count:       diff.newCount,
                        recurring_count: diff.recurringCount,
                        resolved_count:  diff.resolvedCount,
                    },
                    tempFile:    tempFilePath,
                    filename:    req.file.originalname,
                    report_date: parsed.report_date,
                    total_items: parsed.total,
                });

            } catch (err) {
                fs.unlink(req.file.path, () => {});
                console.error('[Compliance] Preview error:', err.message);
                res.status(500).json({ error: 'Failed to parse file: ' + err.message });
            }
        });
    });

    // -----------------------------------------------------------------------
    // POST /reconcile-config
    // Admin-only. Patches compliance_config.json to resolve breaking and
    // silent-miss drift findings, then re-runs the drift check and returns
    // the updated report. Logs every change to the audit trail.
    //
    // Body: { drift: { breaking: [...], silent_miss: [...] } }
    // Response: { changes: [{ action, key, value, detail }], message: string }
    // -----------------------------------------------------------------------
    router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
        const { drift, schema } = req.body;

        if (!drift || typeof drift !== 'object') {
            return res.status(400).json({ error: 'drift report is required in request body' });
        }

        const hasFindings = (drift.breaking && drift.breaking.length > 0) ||
                            (drift.silent_miss && drift.silent_miss.length > 0);
        if (!hasFindings) {
            return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });
        }

        try {
            const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);

            if (changes.length === 0) {
                return res.json({ changes: [], message: 'No changes needed' });
            }

            // Audit log each change
            for (const change of changes) {
                logAudit(db, {
                    userId:     req.user.id,
                    username:   req.user.username,
                    action:     'compliance_config_reconcile',
                    entityType: 'compliance_config',
                    entityId:   change.value,
                    details:    { action: change.action, key: change.key, detail: change.detail },
                    ipAddress:  req.ip,
                });
            }

            res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });

        } catch (err) {
            console.error('[Compliance] Reconcile config error:', err.message);
            res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
        }
    });

    // -----------------------------------------------------------------------
    // POST /commit
    // Commit a previewed upload to the DB.
    //
    // Body: { tempFile: string, filename: string, report_date: string }
    // Response: { upload: { id, filename, report_date, uploaded_at,
    //   new_count, resolved_count, recurring_count } }
    // -----------------------------------------------------------------------
    router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const { tempFile, filename, report_date } = req.body;

        if (!tempFile || typeof tempFile !== 'string') {
            return res.status(400).json({ error: 'tempFile is required' });
        }
        if (!isSafeTempPath(tempFile)) {
            return res.status(400).json({ error: 'Invalid tempFile path' });
        }
        if (!fs.existsSync(tempFile)) {
            return res.status(400).json({ error: 'Preview session expired — please upload again' });
        }

        let parsed;
        try {
            parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8'));
        } catch {
            return res.status(400).json({ error: 'Could not read preview data — please upload again' });
        }

        try {
            const result = await persistUpload(db, {
                items:      parsed.items,
                summary:    parsed.summary,
                reportDate: report_date || parsed.report_date,
                filename:   filename   || parsed.filename,
                userId:     req.user?.id || null,
            });

            fs.unlink(tempFile, () => {});

            const upload = await dbGet(db,
                `SELECT id, filename, report_date, uploaded_at,
                        new_count, resolved_count, recurring_count
                 FROM compliance_uploads WHERE id = ?`,
                [result.uploadId]
            );

            res.json({ upload });

        } catch (err) {
            console.error('[Compliance] Commit error:', err.message);
            res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
        }
    });

    // -----------------------------------------------------------------------
    // GET /uploads
    // List all uploads, most recent first.
    //
    // Response: { uploads: [{ id, filename, report_date, uploaded_at,
    //   new_count, resolved_count, recurring_count }] }
    // -----------------------------------------------------------------------
    router.get('/uploads', async (req, res) => {
        try {
            const rows = await dbAll(db,
                `SELECT id, filename, report_date, uploaded_at,
                        new_count, resolved_count, recurring_count
                 FROM compliance_uploads
                 ORDER BY id DESC`
            );
            res.json({ uploads: rows });
        } catch (err) {
            console.error('[Compliance] GET /uploads error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // POST /rollback/:uploadId
    // Admin-only. Rolls back a specific upload. Only the most recent upload
    // can be rolled back to avoid cascading data integrity issues.
    //
    // Params: uploadId — integer ID of the upload to roll back
    // Response: { message: string, rolled_back: { upload_id, filename,
    //   report_date, items_deleted, items_reactivated } }
    //
    // Reversal logic:
    //   1. Delete items first seen in this upload (new items)
    //   2. Re-activate items resolved by this upload
    //   3. Revert recurring items: decrement seen_count, point upload_id
    //      back to the previous upload
    //   4. Delete the upload record
    // -----------------------------------------------------------------------
    router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
        const uploadId = parseInt(req.params.uploadId, 10);
        if (isNaN(uploadId)) {
            return res.status(400).json({ error: 'Invalid upload ID' });
        }

        try {
            // Verify the upload exists
            const upload = await dbGet(db,
                `SELECT id, filename, report_date, new_count, resolved_count, recurring_count
                 FROM compliance_uploads WHERE id = ?`,
                [uploadId]
            );
            if (!upload) {
                return res.status(404).json({ error: 'Upload not found' });
            }

            // Only allow rolling back the most recent upload
            const latest = await dbGet(db,
                `SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
            );
            if (latest.id !== uploadId) {
                return res.status(400).json({
                    error: 'Only the most recent upload can be rolled back',
                    latest_upload_id: latest.id
                });
            }

            // Find the previous upload (to restore recurring items' upload_id)
            const previousUpload = await dbGet(db,
                `SELECT id FROM compliance_uploads WHERE id < ? ORDER BY id DESC LIMIT 1`,
                [uploadId]
            );

            await dbRun(db, 'BEGIN TRANSACTION');

            try {
                // 1. Delete items that were NEW in this upload
                const deleteNew = await dbRun(db,
                    `DELETE FROM compliance_items WHERE first_seen_upload_id = ? AND upload_id = ?`,
                    [uploadId, uploadId]
                );

                // 2. Re-activate items that were RESOLVED by this upload
                const reactivate = await dbRun(db,
                    `UPDATE compliance_items
                     SET status = 'active', resolved_upload_id = NULL
                     WHERE resolved_upload_id = ?`,
                    [uploadId]
                );

                // 3. Revert RECURRING items: decrement seen_count, restore upload_id
                if (previousUpload) {
                    await dbRun(db,
                        `UPDATE compliance_items
                         SET upload_id = ?, seen_count = MAX(seen_count - 1, 1)
                         WHERE upload_id = ? AND first_seen_upload_id != ?`,
                        [previousUpload.id, uploadId, uploadId]
                    );
                }

                // 4. Delete the upload record
                await dbRun(db, `DELETE FROM compliance_uploads WHERE id = ?`, [uploadId]);

                await dbRun(db, 'COMMIT');

                // Audit log
                logAudit(db, {
                    userId:     req.user.id,
                    username:   req.user.username,
                    action:     'compliance_upload_rollback',
                    entityType: 'compliance_upload',
                    entityId:   String(uploadId),
                    details:    {
                        filename:    upload.filename,
                        report_date: upload.report_date,
                        items_deleted:     deleteNew.changes,
                        items_reactivated: reactivate.changes,
                    },
                    ipAddress:  req.ip,
                });

                res.json({
                    message: `Rolled back upload "${upload.filename}"`,
                    rolled_back: {
                        upload_id:         uploadId,
                        filename:          upload.filename,
                        report_date:       upload.report_date,
                        items_deleted:     deleteNew.changes,
                        items_reactivated: reactivate.changes,
                    },
                });

            } catch (err) {
                await dbRun(db, 'ROLLBACK').catch(() => {});
                throw err;
            }

        } catch (err) {
            console.error('[Compliance] Rollback error:', err.message);
            res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
        }
    });

    // -----------------------------------------------------------------------
    // GET /summary?team=STEAM
    // Return metric health rows for a team from the latest upload's summary_json.
    //
    // Query: team — optional, one of ALLOWED_TEAMS
    // Response: { entries: [...], overall_scores: {}, upload: { id,
    //   report_date, uploaded_at } | null }
    // -----------------------------------------------------------------------
    router.get('/summary', async (req, res) => {
        const team = req.query.team;
        if (team && !ALLOWED_TEAMS.has(team)) {
            return res.status(400).json({ error: 'Invalid team' });
        }

        try {
            const latestUpload = await dbGet(db,
                `SELECT id, summary_json, report_date, uploaded_at
                 FROM compliance_uploads ORDER BY id DESC LIMIT 1`
            );
            if (!latestUpload || !latestUpload.summary_json) {
                return res.json({ entries: [], overall_scores: {}, upload: null });
            }

            let summary;
            try { summary = JSON.parse(latestUpload.summary_json); }
            catch { return res.json({ entries: [], overall_scores: {}, upload: null }); }

            let entries = summary.entries || [];
            if (team) {
                entries = entries.filter(e => e.team === team);
            }

            res.json({
                entries,
                overall_scores: summary.overall_scores || {},
                upload: {
                    id:          latestUpload.id,
                    report_date: latestUpload.report_date,
                    uploaded_at: latestUpload.uploaded_at,
                },
            });
        } catch (err) {
            console.error('[Compliance] GET /summary error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /items?team=STEAM&status=active
    // Return non-compliant devices grouped by hostname.
    //
    // Query: team — required, one of ALLOWED_TEAMS
    //        status — optional, 'active' (default) or 'resolved'
    // Response: { devices: [{ hostname, ip_address, device_type, team,
    //   status, failing_metrics, seen_count, first_seen, last_seen,
    //   resolved_on, has_notes }], team, status }
    // -----------------------------------------------------------------------
    router.get('/items', async (req, res) => {
        const { team, status = 'active' } = req.query;

        if (!team) return res.status(400).json({ error: 'team is required' });
        if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
        if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' });

        try {
            const rows = await dbAll(db,
                `SELECT
                     ci.hostname, ci.ip_address, ci.device_type, ci.team,
                     ci.metric_id, ci.metric_desc, ci.category,
                     ci.status, ci.seen_count,
                     fu.report_date AS first_seen,
                     lu.report_date AS last_seen,
                     ru.report_date AS resolved_on
                 FROM compliance_items ci
                 LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
                 LEFT JOIN compliance_uploads lu ON ci.upload_id             = lu.id
                 LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id    = ru.id
                 WHERE ci.team = ? AND ci.status = ?
                 ORDER BY ci.hostname, ci.metric_id`,
                [team, status]
            );

            // Fetch hostnames that have any notes (for the has_notes indicator)
            const noteRows = await dbAll(db,
                `SELECT DISTINCT hostname FROM compliance_notes`
            );
            const noteHostnames = new Set(noteRows.map(r => r.hostname));

            const devices = groupByHostname(rows, noteHostnames);

            res.json({ devices, team, status });

        } catch (err) {
            console.error('[Compliance] GET /items error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /items/:hostname
    // Detail panel: all metric rows for this hostname + notes + upload history.
    //
    // Params: hostname — device hostname string
    // Response: { hostname, ip_address, device_type, team,
    //   metrics: [{ metric_id, metric_desc, category, status, seen_count,
    //     extra, first_seen, last_seen, resolved_on, ... }],
    //   notes: [{ id, metric_id, note, group_id, created_at, created_by }] }
    // -----------------------------------------------------------------------
    router.get('/items/:hostname', async (req, res) => {
        const hostname = req.params.hostname;
        if (!hostname || hostname.length > 300) {
            return res.status(400).json({ error: 'Invalid hostname' });
        }

        try {
            // All metric rows for this hostname
            const metricRows = await dbAll(db,
                `SELECT
                     ci.metric_id, ci.metric_desc, ci.category, ci.status,
                     ci.ip_address, ci.device_type, ci.team,
                     ci.seen_count, ci.extra_json,
                     fu.report_date AS first_seen,
                     fu.uploaded_at AS first_seen_at,
                     lu.report_date AS last_seen,
                     lu.uploaded_at AS last_seen_at,
                     ru.report_date AS resolved_on
                 FROM compliance_items ci
                 LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
                 LEFT JOIN compliance_uploads lu ON ci.upload_id             = lu.id
                 LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id    = ru.id
                 WHERE ci.hostname = ?
                 ORDER BY ci.status DESC, ci.metric_id`,
                [hostname]
            );

            if (metricRows.length === 0) {
                return res.status(404).json({ error: 'Device not found' });
            }

            // Parse extra_json on each row
            const metrics = metricRows.map(r => ({
                ...r,
                extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(),
                extra_json: undefined,
            }));

            // Notes (all metrics for this hostname, sorted newest first)
            const notes = await dbAll(db,
                `SELECT cn.id, cn.metric_id, cn.note, cn.group_id, cn.created_at,
                        u.username AS created_by
                 FROM compliance_notes cn
                 LEFT JOIN users u ON cn.created_by = u.id
                 WHERE cn.hostname = ?
                 ORDER BY cn.created_at DESC`,
                [hostname]
            );

            // Derive device identity from the first active row, else any row
            const identity = metricRows.find(r => r.status === 'active') || metricRows[0];

            res.json({
                hostname,
                ip_address:  identity.ip_address  || '',
                device_type: identity.device_type || '',
                team:        identity.team        || '',
                metrics,
                notes,
            });

        } catch (err) {
            console.error('[Compliance] GET /items/:hostname error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // POST /notes
    // Add a note to one or more (hostname, metric_id) pairs.
    //
    // Body: { hostname: string, metric_ids: string[], note: string }
    //   — or legacy: { hostname: string, metric_id: string, note: string }
    // Response: { notes: [{ id, hostname, metric_id, note, group_id,
    //   created_at, created_by }] }
    // -----------------------------------------------------------------------
    router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const { hostname, metric_id, metric_ids, note } = req.body;

        if (!hostname || typeof hostname !== 'string' || hostname.length > 300 || !/^[a-zA-Z0-9._-]+$/.test(hostname)) {
            return res.status(400).json({ error: 'Invalid hostname format' });
        }

        // --- Resolve metric IDs: metric_ids takes precedence over metric_id ---
        let resolvedIds;
        if (metric_ids !== undefined) {
            if (!Array.isArray(metric_ids)) {
                return res.status(400).json({ error: 'metric_ids must be an array' });
            }
            resolvedIds = metric_ids;
        } else if (metric_id !== undefined && metric_id !== null && metric_id !== '') {
            if (typeof metric_id !== 'string' || metric_id.length > 50) {
                return res.status(400).json({ error: 'Invalid metric_id' });
            }
            resolvedIds = [metric_id];
        } else {
            return res.status(400).json({ error: 'metric_id or metric_ids is required' });
        }

        // --- Validate resolved metric IDs ---
        if (resolvedIds.length === 0) {
            return res.status(400).json({ error: 'At least one metric ID is required' });
        }
        for (let i = 0; i < resolvedIds.length; i++) {
            const mid = resolvedIds[i];
            if (!mid || typeof mid !== 'string' || mid.length === 0 || mid.length > 50) {
                return res.status(400).json({ error: `Invalid metric_id at index ${i}` });
            }
        }

        const noteText = String(note || '').trim().slice(0, 1000);
        if (!noteText) {
            return res.status(400).json({ error: 'Note cannot be empty' });
        }

        const groupId = crypto.randomUUID();
        const userId  = req.user?.id || null;

        try {
            await dbRun(db, 'BEGIN TRANSACTION');

            const insertedIds = [];
            for (const mid of resolvedIds) {
                const { lastID } = await dbRun(db,
                    `INSERT INTO compliance_notes (hostname, metric_id, note, group_id, created_by, created_at)
                     VALUES (?, ?, ?, ?, ?, datetime('now'))`,
                    [hostname, mid, noteText, groupId, userId]
                );
                insertedIds.push(lastID);
            }

            await dbRun(db, 'COMMIT');

            // Fetch all created rows with username
            const placeholders = insertedIds.map(() => '?').join(', ');
            const notes = await dbAll(db,
                `SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.group_id, cn.created_at,
                        u.username AS created_by
                 FROM compliance_notes cn
                 LEFT JOIN users u ON cn.created_by = u.id
                 WHERE cn.id IN (${placeholders})
                 ORDER BY cn.id ASC`,
                insertedIds
            );

            res.status(201).json({ notes });

        } catch (err) {
            await dbRun(db, 'ROLLBACK').catch(() => {});
            console.error('[Compliance] POST /notes error:', err.message);
            res.status(500).json({ error: 'Failed to save note' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /notes/:hostname/:metricId
    // Return all notes for a (hostname, metric_id) pair.
    //
    // Params: hostname — device hostname string
    //         metricId — metric identifier string
    // Response: { notes: [{ id, note, created_at, created_by }] }
    // -----------------------------------------------------------------------
    router.get('/notes/:hostname/:metricId', async (req, res) => {
        const { hostname, metricId } = req.params;

        if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
        if (!metricId || metricId.length > 50)  return res.status(400).json({ error: 'Invalid metricId' });

        try {
            const notes = await dbAll(db,
                `SELECT cn.id, cn.note, cn.created_at, u.username AS created_by
                 FROM compliance_notes cn
                 LEFT JOIN users u ON cn.created_by = u.id
                 WHERE cn.hostname = ? AND cn.metric_id = ?
                 ORDER BY cn.created_at DESC`,
                [hostname, metricId]
            );
            res.json({ notes });
        } catch (err) {
            console.error('[Compliance] GET /notes error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // DELETE /notes/:id
    // Delete a note (or all notes in the same group_id) by note ID.
    // Only the note author or an Admin can delete.
    //
    // Params: id — note row ID
    // Query:  ?group=true — delete all notes sharing the same group_id
    // Response: { deleted: number }
    // -----------------------------------------------------------------------
    router.delete('/notes/:id', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const noteId = parseInt(req.params.id, 10);
        if (isNaN(noteId)) return res.status(400).json({ error: 'Invalid note ID' });

        const deleteGroup = req.query.group === 'true';

        try {
            // Fetch the note to verify ownership
            const note = await dbGet(db,
                `SELECT id, hostname, metric_id, note, group_id, created_by FROM compliance_notes WHERE id = ?`,
                [noteId]
            );
            if (!note) return res.status(404).json({ error: 'Note not found' });

            // Only the author or an Admin can delete
            const isAuthor = req.user && String(req.user.id) === String(note.created_by);
            const isAdminUser = req.user && req.user.group === 'Admin';
            if (!isAuthor && !isAdminUser) {
                return res.status(403).json({ error: 'You can only delete your own notes' });
            }

            let deleted = 0;
            if (deleteGroup && note.group_id) {
                const result = await dbRun(db,
                    `DELETE FROM compliance_notes WHERE group_id = ?`,
                    [note.group_id]
                );
                deleted = result.changes || 0;
            } else {
                const result = await dbRun(db,
                    `DELETE FROM compliance_notes WHERE id = ?`,
                    [noteId]
                );
                deleted = result.changes || 0;
            }

            logAudit(db, {
                userId:     req.user.id,
                username:   req.user.username,
                action:     'compliance_note_delete',
                entityType: 'compliance_note',
                entityId:   String(noteId),
                details:    JSON.stringify({ hostname: note.hostname, group_id: note.group_id, deleted_count: deleted }),
                ipAddress:  req.ip,
            });

            res.json({ deleted });
        } catch (err) {
            console.error('[Compliance] DELETE /notes error:', err.message);
            res.status(500).json({ error: 'Failed to delete note' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /trends
    // Per-upload active totals + per-team counts for time-series charts.
    // Returns rows ordered ascending by report_date.
    //
    // Response: { trends: [{ report_date, new_count, recurring_count,
    //   resolved_count, total_active, STEAM, ACCESS-ENG, ACCESS-OPS,
    //   INTELDEV }] }
    // -----------------------------------------------------------------------
    router.get('/trends', async (req, res) => {
        try {
            const uploads = await dbAll(db,
                `SELECT id, report_date,
                        COALESCE(new_count, 0)       AS new_count,
                        COALESCE(recurring_count, 0) AS recurring_count,
                        COALESCE(resolved_count, 0)  AS resolved_count,
                        COALESCE(new_count, 0) + COALESCE(recurring_count, 0) AS total_active
                 FROM compliance_uploads
                 ORDER BY report_date ASC`
            );

            if (uploads.length === 0) return res.json({ trends: [] });

            // Per-team active counts — items whose upload_id matches the upload
            // (recurring items have upload_id bumped each cycle, so this is accurate)
            const teamRows = await dbAll(db,
                `SELECT ci.upload_id, ci.team, COUNT(ci.id) AS count
                 FROM compliance_items ci
                 WHERE ci.team IS NOT NULL
                 GROUP BY ci.upload_id, ci.team`
            );

            const teamMap = {};
            teamRows.forEach(r => {
                if (!teamMap[r.upload_id]) teamMap[r.upload_id] = {};
                teamMap[r.upload_id][r.team] = r.count;
            });

            const trends = uploads.map(u => ({
                report_date:      u.report_date,
                new_count:        u.new_count,
                recurring_count:  u.recurring_count,
                resolved_count:   u.resolved_count,
                total_active:     u.total_active,
                STEAM:            teamMap[u.id]?.STEAM            || 0,
                'ACCESS-ENG':     teamMap[u.id]?.['ACCESS-ENG']   || 0,
                'ACCESS-OPS':     teamMap[u.id]?.['ACCESS-OPS']   || 0,
                INTELDEV:         teamMap[u.id]?.INTELDEV          || 0,
            }));

            res.json({ trends });
        } catch (err) {
            console.error('[Compliance] GET /trends error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /mttr
    // Mean time to resolution (calendar days) per team, for resolved items.
    //
    // Response: { mttr: [{ team, avg_days, resolved_count }] }
    // -----------------------------------------------------------------------
    router.get('/mttr', async (req, res) => {
        try {
            const rows = await dbAll(db,
                `SELECT
                     ci.team,
                     ROUND(AVG(JULIANDAY(ru.report_date) - JULIANDAY(fu.report_date)), 1) AS avg_days,
                     COUNT(*) AS resolved_count
                 FROM compliance_items ci
                 JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
                 JOIN compliance_uploads ru ON ci.resolved_upload_id    = ru.id
                 WHERE ci.resolved_upload_id IS NOT NULL
                   AND fu.report_date IS NOT NULL
                   AND ru.report_date IS NOT NULL
                 GROUP BY ci.team
                 ORDER BY avg_days DESC`
            );
            res.json({ mttr: rows });
        } catch (err) {
            console.error('[Compliance] GET /mttr error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /top-recurring
    // Active findings grouped by team + metric_id, sorted by seen_count desc.
    // Identifies chronic compliance gaps that keep reappearing.
    //
    // Response: { items: [{ team, metric_id, metric_desc, seen_count,
    //   host_count }] }  — limited to top 20
    // -----------------------------------------------------------------------
    router.get('/top-recurring', async (req, res) => {
        try {
            const rows = await dbAll(db,
                `SELECT team, metric_id, metric_desc, seen_count, COUNT(*) AS host_count
                 FROM compliance_items
                 WHERE status = 'active'
                 GROUP BY team, metric_id
                 ORDER BY seen_count DESC, host_count DESC
                 LIMIT 20`
            );
            res.json({ items: rows });
        } catch (err) {
            console.error('[Compliance] GET /top-recurring error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /category-trend
    // Active item counts per category per upload, for stacked area chart.
    //
    // Response: { categoryTrend: [{ report_date, category, count }] }
    // -----------------------------------------------------------------------
    router.get('/category-trend', async (req, res) => {
        try {
            const rows = await dbAll(db,
                `SELECT cu.report_date, COALESCE(ci.category, 'Unknown') AS category, COUNT(ci.id) AS count
                 FROM compliance_uploads cu
                 JOIN compliance_items ci ON ci.upload_id = cu.id
                 GROUP BY cu.id, category
                 ORDER BY cu.report_date ASC`
            );
            res.json({ categoryTrend: rows });
        } catch (err) {
            console.error('[Compliance] GET /category-trend error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    return router;
}

module.exports = createComplianceRouter;