cve-dashboard/backend/routes/compliance.js

// Compliance Routes — AEO metric tracking
// Handles xlsx upload/parse, non-compliant item history, and notes.

const express  = require('express');
const path     = require('path');
const fs       = require('fs');
const crypto   = require('crypto');
const { spawn } = require('child_process');
const pool     = require('../db');
const { requireAuth, requireGroup } = require('../middleware/auth');
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
const { isValidDateString, validateRemediationPlan, computeVCLStats, categorizeNonCompliant, rankHeavyHitters, computeForecastBurndown, matchByHostname, computeBulkDiff, mapColumnHeaders } = require('../helpers/vclHelpers');
const logAudit = require('../helpers/auditLog');

const PARSER_SCRIPT  = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
const SCHEMA_SCRIPT  = path.join(__dirname, '../scripts/extract_xlsx_schema.py');
const CONFIG_PATH    = path.join(__dirname, '..', 'scripts', 'compliance_config.json');
const PYTHON_BIN     = process.env.PYTHON_BIN || 'python3';
const TEMP_DIR       = path.join(process.cwd(), 'uploads', 'temp');
const ALLOWED_TEAMS  = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);

// ---------------------------------------------------------------------------
// Run Python parser, return parsed object
// ---------------------------------------------------------------------------
function parseXlsx(filePath) {
    return new Promise((resolve, reject) => {
        const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
        let out = '';
        let err = '';
        py.stdout.on('data', d => { out += d; });
        py.stderr.on('data', d => { err += d; });
        py.on('close', code => {
            if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
            try { resolve(JSON.parse(out)); }
            catch (e) { reject(new Error('Parser returned invalid JSON')); }
        });
        py.on('error', reject);
    });
}

function extractXlsxSchema(filePath) {
    return new Promise((resolve, reject) => {
        const py = spawn(PYTHON_BIN, [SCHEMA_SCRIPT, filePath]);
        let out = '';
        let err = '';
        py.stdout.on('data', d => { out += d; });
        py.stderr.on('data', d => { err += d; });
        py.on('close', code => {
            if (code !== 0) return reject(new Error(err || `Schema extractor exited with code ${code}`));
            try { resolve(JSON.parse(out)); }
            catch (e) { reject(new Error('Schema extractor returned invalid JSON')); }
        });
        py.on('error', reject);
    });
}

function isSafeTempPath(filePath) {
    const resolved = path.resolve(filePath);
    return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
}

// ---------------------------------------------------------------------------
// Compute diff: new / recurring / resolved
// ---------------------------------------------------------------------------
async function computeDiff(incomingItems) {
    const { rows: activeRows } = await pool.query(
        `SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
    );
    const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
    const newKeys    = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));

    let newCount = 0, recurringCount = 0, resolvedCount = 0;
    for (const k of newKeys)    { if (activeKeys.has(k)) recurringCount++; else newCount++; }
    for (const k of activeKeys) { if (!newKeys.has(k))   resolvedCount++; }

    return { newCount, recurringCount, resolvedCount };
}

// ---------------------------------------------------------------------------
// Write a parsed upload to the DB (within a transaction)
// ---------------------------------------------------------------------------
async function persistUpload({ items, summary, reportDate, filename, userId }) {
    const { rows: activeRows } = await pool.query(
        `SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
    );
    const activeMap = {};
    activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });

    const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));

    const client = await pool.connect();
    try {
        await client.query('BEGIN');

        // 1. Insert the upload record
        const uploadResult = await client.query(
            `INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
             VALUES ($1, $2, $3, NOW(), $4)
             RETURNING id`,
            [filename, reportDate || null, userId || null, JSON.stringify(summary)]
        );
        const uploadId = uploadResult.rows[0].id;

        let newCount = 0, recurringCount = 0, resolvedCount = 0;

        // 2. Upsert each incoming non-compliant item
        for (const item of items) {
            const key      = `${item.hostname}|||${item.metric_id}`;
            const existing = activeMap[key];
            const extraStr = JSON.stringify(item.extra_json || {});

            if (existing) {
                await client.query(
                    `UPDATE compliance_items
                     SET upload_id = $1, seen_count = $2, ip_address = $3, device_type = $4, extra_json = $5
                     WHERE id = $6`,
                    [uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
                );
                recurringCount++;
            } else {
                await client.query(
                    `INSERT INTO compliance_items
                     (upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
                      category, extra_json, status, first_seen_upload_id, seen_count)
                     VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, 1)`,
                    [uploadId, item.hostname, item.ip_address, item.device_type, item.team,
                     item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
                );
                newCount++;
            }
        }

        // 3. Mark items not present in this upload as resolved
        for (const [key, row] of Object.entries(activeMap)) {
            if (!newKeys.has(key)) {
                await client.query(
                    `UPDATE compliance_items SET status = 'resolved', resolved_upload_id = $1 WHERE id = $2`,
                    [uploadId, row.id]
                );
                resolvedCount++;
            }
        }

        // 4. Update upload with final counts
        await client.query(
            `UPDATE compliance_uploads SET new_count = $1, resolved_count = $2, recurring_count = $3 WHERE id = $4`,
            [newCount, resolvedCount, recurringCount, uploadId]
        );

        await client.query('COMMIT');

        // Task 7: Create/update compliance_snapshots for the current month
        try {
            const currentMonth = new Date().toISOString().slice(0, 7); // YYYY-MM

            // Compute per-vertical compliance percentages from current state
            const { rows: verticalStats } = await pool.query(
                `SELECT team AS vertical,
                        COUNT(DISTINCT hostname)::int AS total_devices,
                        COUNT(DISTINCT CASE WHEN status = 'resolved' THEN hostname END)::int AS compliant,
                        COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
                 FROM compliance_items
                 WHERE team IS NOT NULL
                 GROUP BY team`
            );

            for (const vs of verticalStats) {
                const total = vs.total_devices;
                const compPct = total > 0 ? Math.round((vs.compliant / total) * 100 * 100) / 100 : 0;

                await pool.query(
                    `INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
                     VALUES ($1, $2, $3, $4, $5, $6)
                     ON CONFLICT (snapshot_month, vertical)
                     DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
                    [currentMonth, vs.vertical, total, vs.compliant, vs.non_compliant, compPct]
                );
            }
        } catch (snapshotErr) {
            // Snapshot creation is non-critical — log but don't fail the upload
            console.error('[Compliance] Snapshot creation error:', snapshotErr.message);
        }

        return { uploadId, newCount, recurringCount, resolvedCount };
    } catch (err) {
        await client.query('ROLLBACK');
        throw err;
    } finally {
        client.release();
    }
}

// ---------------------------------------------------------------------------
// Group flat compliance_items rows into per-device objects
// ---------------------------------------------------------------------------
function groupByHostname(rows, noteHostnames) {
    const deviceMap = {};
    for (const row of rows) {
        if (!deviceMap[row.hostname]) {
            deviceMap[row.hostname] = {
                hostname: row.hostname, ip_address: row.ip_address || '', device_type: row.device_type || '',
                team: row.team || '', status: row.status, failing_metrics: [],
                seen_count: row.seen_count || 1, first_seen: row.first_seen || null,
                last_seen: row.last_seen || null, resolved_on: row.resolved_on || null,
                has_notes: noteHostnames.has(row.hostname),
            };
        }
        const dev = deviceMap[row.hostname];
        dev.failing_metrics.push({ metric_id: row.metric_id, metric_desc: row.metric_desc || '', category: row.category || '' });
        if ((row.seen_count || 1) > dev.seen_count) dev.seen_count = row.seen_count;
        if (row.first_seen && (!dev.first_seen || row.first_seen < dev.first_seen)) dev.first_seen = row.first_seen;
        if (row.last_seen && (!dev.last_seen || row.last_seen > dev.last_seen)) dev.last_seen = row.last_seen;
    }
    return Object.values(deviceMap);
}

// ---------------------------------------------------------------------------
// Pure helpers
// ---------------------------------------------------------------------------
const BUCKET_ORDER = ['1 cycle', '2–3 cycles', '4–6 cycles', '7+ cycles'];

function bucketAgingItems(items) {
    const teams = ['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV'];
    const buckets = {};
    for (const b of BUCKET_ORDER) {
        buckets[b] = { bucket: b, total: 0 };
        for (const t of teams) buckets[b][t] = 0;
    }
    for (const item of items) {
        const sc = item.seen_count;
        let label;
        if (sc === 1) label = '1 cycle';
        else if (sc >= 2 && sc <= 3) label = '2–3 cycles';
        else if (sc >= 4 && sc <= 6) label = '4–6 cycles';
        else label = '7+ cycles';
        buckets[label].total += 1;
        if (item.team in buckets[label]) buckets[label][item.team] += 1;
    }
    return BUCKET_ORDER.map(b => buckets[b]);
}

function computeWaterfall(uploads) {
    let start = 0;
    return uploads.map((row) => {
        const end = start + row.new_count + row.recurring_count - row.resolved_count;
        const entry = { date: row.report_date, start, new_count: row.new_count, recurring_count: row.recurring_count, resolved_count: row.resolved_count, end };
        start = end;
        return entry;
    });
}

// ---------------------------------------------------------------------------
// Router factory
// ---------------------------------------------------------------------------
function createComplianceRouter(upload) {
    const router = express.Router();

    // All compliance routes require authentication
    router.use(requireAuth());

    // POST /preview
    router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
        upload.single('file')(req, res, async (uploadErr) => {
            if (uploadErr) return res.status(400).json({ error: uploadErr.message });
            if (!req.file) return res.status(400).json({ error: 'No file uploaded' });
            if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
                fs.unlink(req.file.path, () => {});
                return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
            }

            try {
                let drift = null, drift_error = null;
                let config;
                try { config = loadConfig(CONFIG_PATH); } catch (configErr) {
                    fs.unlink(req.file.path, () => {});
                    return res.status(500).json({ error: 'Configuration file could not be loaded: ' + configErr.message });
                }

                let xlsxSchema = null;
                try {
                    xlsxSchema = await extractXlsxSchema(req.file.path);
                    if (xlsxSchema.error) throw new Error(xlsxSchema.error);
                    drift = compareSchemaToDrift(xlsxSchema, config);
                } catch (driftErr) {
                    drift = null;
                    drift_error = driftErr.message || 'Drift check failed';
                }

                const parsed = await parseXlsx(req.file.path);
                if (parsed.error) {
                    fs.unlink(req.file.path, () => {});
                    return res.status(422).json({ error: parsed.error });
                }

                const diff = await computeDiff(parsed.items);

                if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
                const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
                const tempFilePath = path.join(TEMP_DIR, tempFilename);

                fs.writeFileSync(tempFilePath, JSON.stringify({
                    items: parsed.items, summary: parsed.summary,
                    report_date: parsed.report_date,
                    filename: req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
                }));

                fs.unlink(req.file.path, () => {});

                res.json({
                    drift, drift_error, schema: xlsxSchema,
                    diff: { new_count: diff.newCount, recurring_count: diff.recurringCount, resolved_count: diff.resolvedCount },
                    tempFile: tempFilePath, filename: req.file.originalname,
                    report_date: parsed.report_date, total_items: parsed.total,
                });
            } catch (err) {
                fs.unlink(req.file.path, () => {});
                console.error('[Compliance] Preview error:', err.message);
                res.status(500).json({ error: 'Failed to parse file: ' + err.message });
            }
        });
    });

    // POST /reconcile-config
    router.post('/reconcile-config', requireGroup('Admin'), async (req, res) => {
        const { drift, schema } = req.body;
        if (!drift || typeof drift !== 'object') return res.status(400).json({ error: 'drift report is required in request body' });
        const hasFindings = (drift.breaking && drift.breaking.length > 0) || (drift.silent_miss && drift.silent_miss.length > 0);
        if (!hasFindings) return res.status(400).json({ error: 'No breaking or silent-miss findings to reconcile' });

        try {
            const { changes } = reconcileConfig(CONFIG_PATH, drift, schema || null);
            if (changes.length === 0) return res.json({ changes: [], message: 'No changes needed' });

            for (const change of changes) {
                logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_config_reconcile', entityType: 'compliance_config', entityId: change.value, details: { action: change.action, key: change.key, detail: change.detail }, ipAddress: req.ip });
            }
            res.json({ changes, message: `Reconciled ${changes.length} config change(s)` });
        } catch (err) {
            console.error('[Compliance] Reconcile config error:', err.message);
            res.status(500).json({ error: 'Failed to reconcile config: ' + err.message });
        }
    });

    // POST /commit
    router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const { tempFile, filename, report_date } = req.body;
        if (!tempFile || typeof tempFile !== 'string') return res.status(400).json({ error: 'tempFile is required' });
        if (!isSafeTempPath(tempFile)) return res.status(400).json({ error: 'Invalid tempFile path' });
        if (!fs.existsSync(tempFile)) return res.status(400).json({ error: 'Preview session expired — please upload again' });

        let parsed;
        try { parsed = JSON.parse(fs.readFileSync(tempFile, 'utf8')); }
        catch { return res.status(400).json({ error: 'Could not read preview data — please upload again' }); }

        try {
            const result = await persistUpload({
                items: parsed.items, summary: parsed.summary,
                reportDate: report_date || parsed.report_date,
                filename: filename || parsed.filename,
                userId: req.user?.id || null,
            });
            fs.unlink(tempFile, () => {});

            const { rows } = await pool.query(
                `SELECT id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count
                 FROM compliance_uploads WHERE id = $1`, [result.uploadId]
            );
            res.json({ upload: rows[0] });
        } catch (err) {
            console.error('[Compliance] Commit error:', err.message);
            res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
        }
    });

    // GET /uploads
    router.get('/uploads', async (req, res) => {
        try {
            const { rows } = await pool.query(
                `SELECT id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count
                 FROM compliance_uploads ORDER BY id DESC`
            );
            res.json({ uploads: rows });
        } catch (err) {
            console.error('[Compliance] GET /uploads error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // POST /rollback/:uploadId
    router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
        const uploadId = parseInt(req.params.uploadId, 10);
        if (isNaN(uploadId)) return res.status(400).json({ error: 'Invalid upload ID' });

        try {
            const { rows: uploadRows } = await pool.query(
                `SELECT id, filename, report_date, new_count, resolved_count, recurring_count FROM compliance_uploads WHERE id = $1`, [uploadId]
            );
            const upload = uploadRows[0];
            if (!upload) return res.status(404).json({ error: 'Upload not found' });

            const { rows: latestRows } = await pool.query(`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`);
            if (latestRows[0].id !== uploadId) {
                return res.status(400).json({ error: 'Only the most recent upload can be rolled back', latest_upload_id: latestRows[0].id });
            }

            const { rows: prevRows } = await pool.query(`SELECT id FROM compliance_uploads WHERE id < $1 ORDER BY id DESC LIMIT 1`, [uploadId]);
            const previousUpload = prevRows[0];

            const client = await pool.connect();
            try {
                await client.query('BEGIN');

                const deleteNew = await client.query(
                    `DELETE FROM compliance_items WHERE first_seen_upload_id = $1 AND upload_id = $1`, [uploadId]
                );
                const reactivate = await client.query(
                    `UPDATE compliance_items SET status = 'active', resolved_upload_id = NULL WHERE resolved_upload_id = $1`, [uploadId]
                );
                if (previousUpload) {
                    await client.query(
                        `UPDATE compliance_items SET upload_id = $1, seen_count = GREATEST(seen_count - 1, 1) WHERE upload_id = $2 AND first_seen_upload_id != $2`,
                        [previousUpload.id, uploadId]
                    );
                }
                await client.query(`DELETE FROM compliance_uploads WHERE id = $1`, [uploadId]);
                await client.query('COMMIT');

                logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_upload_rollback', entityType: 'compliance_upload', entityId: String(uploadId), details: { filename: upload.filename, report_date: upload.report_date, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount }, ipAddress: req.ip });

                res.json({ message: `Rolled back upload "${upload.filename}"`, rolled_back: { upload_id: uploadId, filename: upload.filename, report_date: upload.report_date, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount } });
            } catch (err) {
                await client.query('ROLLBACK');
                throw err;
            } finally {
                client.release();
            }
        } catch (err) {
            console.error('[Compliance] Rollback error:', err.message);
            res.status(500).json({ error: 'Failed to rollback upload: ' + err.message });
        }
    });

    // GET /summary
    router.get('/summary', async (req, res) => {
        const team = req.query.team;
        if (team && !ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });

        try {
            const { rows: latestRows } = await pool.query(
                `SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads ORDER BY id DESC LIMIT 1`
            );
            const latestUpload = latestRows[0];
            if (!latestUpload || !latestUpload.summary_json) return res.json({ entries: [], overall_scores: {}, upload: null });

            let summary;
            try { summary = JSON.parse(latestUpload.summary_json); } catch { return res.json({ entries: [], overall_scores: {}, upload: null }); }

            let entries = summary.entries || [];
            if (team) entries = entries.filter(e => e.team === team);

            res.json({ entries, overall_scores: summary.overall_scores || {}, upload: { id: latestUpload.id, report_date: latestUpload.report_date, uploaded_at: latestUpload.uploaded_at } });
        } catch (err) {
            console.error('[Compliance] GET /summary error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // GET /items
    router.get('/items', async (req, res) => {
        const { team, status = 'active' } = req.query;
        if (!team) return res.status(400).json({ error: 'team is required' });
        if (!ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
        if (!['active', 'resolved'].includes(status)) return res.status(400).json({ error: 'Invalid status' });

        try {
            const { rows } = await pool.query(
                `SELECT ci.hostname, ci.ip_address, ci.device_type, ci.team, ci.metric_id, ci.metric_desc, ci.category, ci.status, ci.seen_count,
                     fu.report_date AS first_seen, lu.report_date AS last_seen, ru.report_date AS resolved_on
                 FROM compliance_items ci
                 LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
                 LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
                 LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
                 WHERE ci.team = $1 AND ci.status = $2
                 ORDER BY ci.hostname, ci.metric_id`,
                [team, status]
            );

            const { rows: noteRows } = await pool.query(`SELECT DISTINCT hostname FROM compliance_notes`);
            const noteHostnames = new Set(noteRows.map(r => r.hostname));
            const devices = groupByHostname(rows, noteHostnames);
            res.json({ devices, team, status });
        } catch (err) {
            console.error('[Compliance] GET /items error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // GET /items/:hostname
    router.get('/items/:hostname', async (req, res) => {
        const hostname = req.params.hostname;
        if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });

        try {
            const { rows: metricRows } = await pool.query(
                `SELECT ci.metric_id, ci.metric_desc, ci.category, ci.status, ci.ip_address, ci.device_type, ci.team, ci.seen_count, ci.extra_json,
                     fu.report_date AS first_seen, fu.uploaded_at AS first_seen_at, lu.report_date AS last_seen, lu.uploaded_at AS last_seen_at, ru.report_date AS resolved_on
                 FROM compliance_items ci
                 LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
                 LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
                 LEFT JOIN compliance_uploads ru ON ci.resolved_upload_id = ru.id
                 WHERE ci.hostname = $1
                 ORDER BY ci.status DESC, ci.metric_id`, [hostname]
            );
            if (metricRows.length === 0) return res.status(404).json({ error: 'Device not found' });

            const metrics = metricRows.map(r => ({ ...r, extra: (() => { try { return JSON.parse(r.extra_json || '{}'); } catch { return {}; } })(), extra_json: undefined }));

            const { rows: notes } = await pool.query(
                `SELECT cn.id, cn.metric_id, cn.note, cn.group_id, cn.created_at, u.username AS created_by
                 FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
                 WHERE cn.hostname = $1 ORDER BY cn.created_at DESC`, [hostname]
            );

            const identity = metricRows.find(r => r.status === 'active') || metricRows[0];
            res.json({ hostname, ip_address: identity.ip_address || '', device_type: identity.device_type || '', team: identity.team || '', metrics, notes });
        } catch (err) {
            console.error('[Compliance] GET /items/:hostname error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // POST /notes
    router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const { hostname, metric_id, metric_ids, note } = req.body;
        if (!hostname || typeof hostname !== 'string' || hostname.length > 300 || !/^[a-zA-Z0-9._-]+$/.test(hostname)) return res.status(400).json({ error: 'Invalid hostname format' });

        let resolvedIds;
        if (metric_ids !== undefined) {
            if (!Array.isArray(metric_ids)) return res.status(400).json({ error: 'metric_ids must be an array' });
            resolvedIds = metric_ids;
        } else if (metric_id !== undefined && metric_id !== null && metric_id !== '') {
            if (typeof metric_id !== 'string' || metric_id.length > 50) return res.status(400).json({ error: 'Invalid metric_id' });
            resolvedIds = [metric_id];
        } else {
            return res.status(400).json({ error: 'metric_id or metric_ids is required' });
        }

        if (resolvedIds.length === 0) return res.status(400).json({ error: 'At least one metric ID is required' });
        for (let i = 0; i < resolvedIds.length; i++) {
            const mid = resolvedIds[i];
            if (!mid || typeof mid !== 'string' || mid.length === 0 || mid.length > 50) return res.status(400).json({ error: `Invalid metric_id at index ${i}` });
        }

        const noteText = String(note || '').trim().slice(0, 1000);
        if (!noteText) return res.status(400).json({ error: 'Note cannot be empty' });

        const groupId = crypto.randomUUID();
        const userId  = req.user?.id || null;

        const client = await pool.connect();
        try {
            await client.query('BEGIN');
            const insertedIds = [];
            for (const mid of resolvedIds) {
                const { rows } = await client.query(
                    `INSERT INTO compliance_notes (hostname, metric_id, note, group_id, created_by, created_at) VALUES ($1, $2, $3, $4, $5, NOW()) RETURNING id`,
                    [hostname, mid, noteText, groupId, userId]
                );
                insertedIds.push(rows[0].id);
            }
            await client.query('COMMIT');

            const { rows: notes } = await pool.query(
                `SELECT cn.id, cn.hostname, cn.metric_id, cn.note, cn.group_id, cn.created_at, u.username AS created_by
                 FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
                 WHERE cn.id = ANY($1) ORDER BY cn.id ASC`, [insertedIds]
            );
            res.status(201).json({ notes });
        } catch (err) {
            await client.query('ROLLBACK');
            console.error('[Compliance] POST /notes error:', err.message);
            res.status(500).json({ error: 'Failed to save note' });
        } finally {
            client.release();
        }
    });

    // GET /notes/:hostname/:metricId
    router.get('/notes/:hostname/:metricId', async (req, res) => {
        const { hostname, metricId } = req.params;
        if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
        if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metricId' });

        try {
            const { rows: notes } = await pool.query(
                `SELECT cn.id, cn.note, cn.created_at, u.username AS created_by
                 FROM compliance_notes cn LEFT JOIN users u ON cn.created_by = u.id
                 WHERE cn.hostname = $1 AND cn.metric_id = $2 ORDER BY cn.created_at DESC`, [hostname, metricId]
            );
            res.json({ notes });
        } catch (err) {
            console.error('[Compliance] GET /notes error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // DELETE /notes/:id
    router.delete('/notes/:id', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const noteId = parseInt(req.params.id, 10);
        if (isNaN(noteId)) return res.status(400).json({ error: 'Invalid note ID' });
        const deleteGroup = req.query.group === 'true';

        try {
            const { rows } = await pool.query(`SELECT id, hostname, metric_id, note, group_id, created_by FROM compliance_notes WHERE id = $1`, [noteId]);
            const noteRow = rows[0];
            if (!noteRow) return res.status(404).json({ error: 'Note not found' });

            const isAuthor = req.user && String(req.user.id) === String(noteRow.created_by);
            const isAdminUser = req.user && req.user.group === 'Admin';
            if (!isAuthor && !isAdminUser) return res.status(403).json({ error: 'You can only delete your own notes' });

            let deleted = 0;
            if (deleteGroup && noteRow.group_id) {
                const result = await pool.query(`DELETE FROM compliance_notes WHERE group_id = $1`, [noteRow.group_id]);
                deleted = result.rowCount;
            } else {
                const result = await pool.query(`DELETE FROM compliance_notes WHERE id = $1`, [noteId]);
                deleted = result.rowCount;
            }

            logAudit({ userId: req.user.id, username: req.user.username, action: 'compliance_note_delete', entityType: 'compliance_note', entityId: String(noteId), details: JSON.stringify({ hostname: noteRow.hostname, group_id: noteRow.group_id, deleted_count: deleted }), ipAddress: req.ip });
            res.json({ deleted });
        } catch (err) {
            console.error('[Compliance] DELETE /notes error:', err.message);
            res.status(500).json({ error: 'Failed to delete note' });
        }
    });

    // GET /trends
    router.get('/trends', async (req, res) => {
        try {
            const { rows: uploads } = await pool.query(
                `SELECT id, report_date, COALESCE(new_count, 0) AS new_count, COALESCE(recurring_count, 0) AS recurring_count, COALESCE(resolved_count, 0) AS resolved_count, COALESCE(new_count, 0) + COALESCE(recurring_count, 0) AS total_active FROM compliance_uploads ORDER BY report_date ASC`
            );
            if (uploads.length === 0) return res.json({ trends: [] });

            const { rows: teamRows } = await pool.query(
                `SELECT ci.upload_id, ci.team, COUNT(ci.id)::int AS count FROM compliance_items ci WHERE ci.team IS NOT NULL GROUP BY ci.upload_id, ci.team`
            );
            const teamMap = {};
            teamRows.forEach(r => { if (!teamMap[r.upload_id]) teamMap[r.upload_id] = {}; teamMap[r.upload_id][r.team] = r.count; });

            const trends = uploads.map(u => ({
                report_date: u.report_date, new_count: u.new_count, recurring_count: u.recurring_count, resolved_count: u.resolved_count, total_active: u.total_active,
                STEAM: teamMap[u.id]?.STEAM || 0, 'ACCESS-ENG': teamMap[u.id]?.['ACCESS-ENG'] || 0, 'ACCESS-OPS': teamMap[u.id]?.['ACCESS-OPS'] || 0, INTELDEV: teamMap[u.id]?.INTELDEV || 0,
            }));
            res.json({ trends });
        } catch (err) {
            console.error('[Compliance] GET /trends error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // GET /mttr
    router.get('/mttr', async (req, res) => {
        try {
            const { rows } = await pool.query(`SELECT COALESCE(seen_count, 1) AS seen_count, team FROM compliance_items WHERE status = 'active'`);
            if (rows.length === 0) return res.json({ aging: [] });
            const aging = bucketAgingItems(rows);
            res.json({ aging });
        } catch (err) {
            console.error('[Compliance] GET /mttr error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // GET /top-recurring
    router.get('/top-recurring', async (req, res) => {
        try {
            const { rows } = await pool.query(
                `SELECT id, report_date, COALESCE(new_count, 0) AS new_count, COALESCE(recurring_count, 0) AS recurring_count, COALESCE(resolved_count, 0) AS resolved_count FROM compliance_uploads ORDER BY report_date ASC`
            );
            const waterfall = computeWaterfall(rows);
            res.json({ waterfall });
        } catch (err) {
            console.error('[Compliance] GET /top-recurring error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // GET /category-trend
    router.get('/category-trend', async (req, res) => {
        try {
            const { rows } = await pool.query(
                `SELECT cu.report_date, COALESCE(ci.category, 'Unknown') AS category, COUNT(ci.id)::int AS count
                 FROM compliance_uploads cu JOIN compliance_items ci ON ci.upload_id = cu.id
                 GROUP BY cu.id, cu.report_date, category ORDER BY cu.report_date ASC`
            );
            res.json({ categoryTrend: rows });
        } catch (err) {
            console.error('[Compliance] GET /category-trend error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // PATCH /items/:hostname/metadata — Update resolution_date / remediation_plan
    // -----------------------------------------------------------------------
    router.patch('/items/:hostname/metadata', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const hostname = req.params.hostname;
        if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });

        const { resolution_date, remediation_plan } = req.body;

        // Validate resolution_date: must be a valid ISO date string or null
        if (resolution_date !== undefined && resolution_date !== null) {
            if (!isValidDateString(resolution_date)) {
                return res.status(400).json({ error: 'Invalid resolution_date format' });
            }
        }

        // Validate remediation_plan: must be <= 2000 chars or null
        if (remediation_plan !== undefined && remediation_plan !== null) {
            const planValidation = validateRemediationPlan(remediation_plan);
            if (!planValidation.valid) {
                return res.status(400).json({ error: planValidation.error });
            }
        }

        try {
            // Build dynamic SET clause for provided fields only
            const setClauses = [];
            const values = [];
            let paramIdx = 1;

            if (resolution_date !== undefined) {
                setClauses.push(`resolution_date = $${paramIdx++}`);
                values.push(resolution_date);
            }
            if (remediation_plan !== undefined) {
                setClauses.push(`remediation_plan = $${paramIdx++}`);
                values.push(remediation_plan);
            }

            if (setClauses.length === 0) {
                return res.status(400).json({ error: 'No fields to update' });
            }

            values.push(hostname);
            const result = await pool.query(
                `UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
                values
            );

            if (result.rowCount === 0) {
                return res.status(404).json({ error: 'Device not found' });
            }

            logAudit({
                userId: req.user.id,
                username: req.user.username,
                action: 'compliance_metadata_update',
                entityType: 'compliance_item',
                entityId: hostname,
                details: { resolution_date, remediation_plan },
                ipAddress: req.ip,
            });

            res.json({ updated: result.rowCount });
        } catch (err) {
            console.error('[Compliance] PATCH /items/:hostname/metadata error:', err.message);
            res.status(500).json({ error: 'Failed to update device metadata' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /vcl/stats — VCL executive summary statistics
    // -----------------------------------------------------------------------
    const VCL_TARGET_PCT = parseInt(process.env.VCL_TARGET_PCT, 10) || 95;

    router.get('/vcl/stats', async (req, res) => {
        try {
            // Fetch all active compliance items
            const { rows: items } = await pool.query(
                `SELECT hostname, team, status, resolution_date, remediation_plan,
                        CASE WHEN status = 'resolved' THEN true ELSE false END AS is_compliant,
                        true AS in_scope
                 FROM compliance_items WHERE status = 'active'`
            );

            // For stats computation, all active items are non-compliant (they are findings)
            // We need total in-scope devices (active + resolved from latest upload)
            const { rows: latestUploadRows } = await pool.query(
                `SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
            );

            let allDeviceItems = [];
            if (latestUploadRows.length > 0) {
                const { rows: allItems } = await pool.query(
                    `SELECT hostname, team, status, resolution_date, remediation_plan,
                            CASE WHEN status = 'resolved' THEN true ELSE false END AS is_compliant,
                            true AS in_scope
                     FROM compliance_items`
                );
                // Deduplicate by hostname — a device is compliant if it has no active findings
                const deviceMap = new Map();
                for (const item of allItems) {
                    const existing = deviceMap.get(item.hostname);
                    if (!existing) {
                        deviceMap.set(item.hostname, { ...item, is_compliant: item.status !== 'active', in_scope: true });
                    } else if (item.status === 'active') {
                        existing.is_compliant = false;
                    }
                }
                allDeviceItems = Array.from(deviceMap.values());
            }

            const stats = computeVCLStats(allDeviceItems, VCL_TARGET_PCT);

            // Donut: categorize non-compliant items by resolution_date presence
            const nonCompliantItems = items.filter(i => i.status === 'active');
            const donut = categorizeNonCompliant(nonCompliantItems);

            // Heavy hitters: group by team, count non-compliant per team
            const teamCounts = {};
            for (const item of nonCompliantItems) {
                const team = item.team || 'Unknown';
                if (!teamCounts[team]) {
                    teamCounts[team] = { vertical: team, team: team, non_compliant: 0, compliance_date: null, notes: '' };
                }
                teamCounts[team].non_compliant++;
                // Use the latest resolution_date as the team's compliance_date
                if (item.resolution_date && (!teamCounts[team].compliance_date || item.resolution_date > teamCounts[team].compliance_date)) {
                    teamCounts[team].compliance_date = item.resolution_date;
                }
            }
            const heavy_hitters = rankHeavyHitters(Object.values(teamCounts));

            // Vertical breakdown with burndown
            const verticalBreakdown = [];
            for (const team of Object.keys(teamCounts)) {
                const teamItems = nonCompliantItems.filter(i => (i.team || 'Unknown') === team);
                const teamAllDevices = allDeviceItems.filter(i => (i.team || 'Unknown') === team);
                const teamTotal = teamAllDevices.length;
                const teamCompliant = teamAllDevices.filter(i => i.is_compliant).length;
                const compliance_pct = teamTotal > 0 ? Math.round((teamCompliant / teamTotal) * 100) : 0;

                const actual_burndown = computeForecastBurndown(teamItems.filter(i => i.resolution_date));
                const forecast_burndown = computeForecastBurndown(teamItems.filter(i => i.resolution_date));
                const blockers = teamItems.filter(i => !i.resolution_date).length;

                verticalBreakdown.push({
                    vertical: team,
                    compliance_pct,
                    team: team,
                    non_compliant: teamItems.length,
                    actual_burndown,
                    forecast_burndown,
                    blockers,
                    risk_acceptances: 0,
                    notes: '',
                });
            }

            res.json({ stats, donut, heavy_hitters, vertical_breakdown: verticalBreakdown });
        } catch (err) {
            console.error('[Compliance] GET /vcl/stats error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // GET /vcl/trend — Monthly compliance trend with forecast
    // -----------------------------------------------------------------------
    router.get('/vcl/trend', async (req, res) => {
        try {
            const { rows: snapshots } = await pool.query(
                `SELECT snapshot_month, SUM(compliant)::int AS compliant_count,
                        CASE WHEN SUM(total_devices) > 0
                             THEN ROUND((SUM(compliant)::numeric / SUM(total_devices)::numeric) * 100, 1)
                             ELSE 0 END AS compliance_pct
                 FROM compliance_snapshots
                 GROUP BY snapshot_month
                 ORDER BY snapshot_month ASC`
            );

            // Build months array with actuals
            const months = snapshots.map(s => ({
                month: s.snapshot_month,
                compliant_count: s.compliant_count,
                compliance_pct: parseFloat(s.compliance_pct),
                forecast_pct: null,
                target_pct: VCL_TARGET_PCT,
            }));

            // Compute forecast using linear regression if we have 3+ months
            if (months.length >= 3) {
                const n = months.length;
                // Use last data points for regression
                let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
                for (let i = 0; i < n; i++) {
                    sumX += i;
                    sumY += months[i].compliance_pct;
                    sumXY += i * months[i].compliance_pct;
                    sumX2 += i * i;
                }
                const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
                const intercept = (sumY - slope * sumX) / n;

                // Project forward 3 months
                for (let i = 0; i < 3; i++) {
                    const futureIdx = n + i;
                    const forecastPct = Math.min(100, Math.max(0, Math.round((slope * futureIdx + intercept) * 10) / 10));

                    // Compute the future month string
                    const lastMonth = months[months.length - 1].month;
                    const [year, mon] = lastMonth.split('-').map(Number);
                    const futureDate = new Date(year, mon - 1 + i + 1, 1);
                    const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`;

                    months.push({
                        month: futureMonth,
                        compliant_count: null,
                        compliance_pct: null,
                        forecast_pct: forecastPct,
                        target_pct: VCL_TARGET_PCT,
                    });
                }

                // Also add forecast_pct to the last actual month as the starting point
                if (months.length > 0 && n > 0) {
                    months[n - 1].forecast_pct = months[n - 1].compliance_pct;
                }
            }

            res.json({ months });
        } catch (err) {
            console.error('[Compliance] GET /vcl/trend error:', err.message);
            res.status(500).json({ error: 'Database error' });
        }
    });

    // -----------------------------------------------------------------------
    // POST /vcl/bulk-preview — Bulk upload diff preview
    // -----------------------------------------------------------------------
    router.post('/vcl/bulk-preview', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const { rows, headers } = req.body;

        // Validate: require rows array
        if (!rows || !Array.isArray(rows)) {
            return res.status(400).json({ error: 'rows array is required' });
        }

        // Enforce 2000 row limit
        if (rows.length === 0) {
            return res.status(400).json({ error: 'File contains no data rows' });
        }
        if (rows.length > 2000) {
            return res.status(400).json({ error: 'File exceeds maximum of 2000 rows' });
        }

        // Map column headers if provided
        let columnMapping = {};
        if (headers && Array.isArray(headers)) {
            columnMapping = mapColumnHeaders(headers);
        }

        // Require hostname field
        const hasHostname = rows.every(r => r.hostname != null && r.hostname !== '');
        if (!hasHostname) {
            return res.status(400).json({ error: 'File must contain a Hostname column' });
        }

        // Check for updatable fields (resolution_date, remediation_plan, or notes)
        const sampleRow = rows[0] || {};
        const updatableFields = ['resolution_date', 'remediation_plan', 'notes'];
        const hasUpdatableFields = updatableFields.some(f => f in sampleRow);
        if (!hasUpdatableFields && headers) {
            // Check via column mapping
            const mappedFields = Object.keys(columnMapping).filter(k => k !== 'hostname');
            if (mappedFields.length === 0) {
                return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
            }
        } else if (!hasUpdatableFields && !headers) {
            return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
        }

        try {
            // Get existing hostnames from DB
            const { rows: existingRows } = await pool.query(
                `SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active'`
            );
            const existingHostnames = new Set(existingRows.map(r => r.hostname));

            // Match by hostname
            const { matched, unmatched } = matchByHostname(rows, existingHostnames);

            // Validate fields on matched rows
            const validRows = [];
            const invalidRows = [];

            for (const row of matched) {
                const errors = [];

                if (row.resolution_date !== undefined && row.resolution_date !== null && row.resolution_date !== '') {
                    if (!isValidDateString(row.resolution_date)) {
                        errors.push('resolution_date: invalid date format');
                    }
                }

                if (row.remediation_plan !== undefined && row.remediation_plan !== null) {
                    const planCheck = validateRemediationPlan(row.remediation_plan);
                    if (!planCheck.valid) {
                        errors.push('remediation_plan: ' + planCheck.error);
                    }
                }

                if (errors.length > 0) {
                    invalidRows.push({ hostname: row.hostname, errors });
                } else {
                    validRows.push(row);
                }
            }

            // Get current data for diff computation
            const { rows: currentRows } = await pool.query(
                `SELECT DISTINCT ON (hostname) hostname, resolution_date, remediation_plan
                 FROM compliance_items WHERE status = 'active' AND hostname = ANY($1)
                 ORDER BY hostname, id DESC`,
                [validRows.map(r => r.hostname)]
            );
            const currentData = new Map();
            for (const row of currentRows) {
                currentData.set(row.hostname, {
                    resolution_date: row.resolution_date ? row.resolution_date.toISOString?.().slice(0, 10) || String(row.resolution_date).slice(0, 10) : null,
                    remediation_plan: row.remediation_plan || null,
                    notes: null,
                });
            }

            // Compute diff
            const diffResults = computeBulkDiff(validRows, currentData);
            const changedRows = diffResults.filter(r => r.status === 'changed');

            res.json({
                matched: matched.length,
                unmatched: unmatched.length,
                changes: changedRows.length,
                invalid: invalidRows.length,
                details: diffResults,
                unmatched_rows: unmatched.map(r => r.hostname),
                invalid_rows: invalidRows,
            });
        } catch (err) {
            console.error('[Compliance] POST /vcl/bulk-preview error:', err.message);
            res.status(500).json({ error: 'Failed to process bulk preview' });
        }
    });

    // -----------------------------------------------------------------------
    // POST /vcl/bulk-commit — Commit validated bulk changes
    // -----------------------------------------------------------------------
    router.post('/vcl/bulk-commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
        const { changes } = req.body;

        if (!changes || !Array.isArray(changes) || changes.length === 0) {
            return res.status(400).json({ error: 'changes array is required' });
        }

        const client = await pool.connect();
        try {
            await client.query('BEGIN');

            let committedCount = 0;
            for (const change of changes) {
                const setClauses = [];
                const values = [];
                let paramIdx = 1;

                if (change.resolution_date !== undefined) {
                    setClauses.push(`resolution_date = $${paramIdx++}`);
                    values.push(change.resolution_date);
                }
                if (change.remediation_plan !== undefined) {
                    setClauses.push(`remediation_plan = $${paramIdx++}`);
                    values.push(change.remediation_plan);
                }
                if (change.notes !== undefined) {
                    // Notes are stored separately in compliance_notes, but we can update a field if it exists
                    // For now, skip notes in the direct update
                }

                if (setClauses.length === 0) continue;

                values.push(change.hostname);
                const result = await client.query(
                    `UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
                    values
                );
                if (result.rowCount > 0) committedCount++;
            }

            await client.query('COMMIT');

            logAudit({
                userId: req.user.id,
                username: req.user.username,
                action: 'compliance_bulk_update',
                entityType: 'compliance_items',
                entityId: null,
                details: { rows_updated: committedCount, total_changes: changes.length },
                ipAddress: req.ip,
            });

            res.json({ committed: committedCount });
        } catch (err) {
            await client.query('ROLLBACK');
            console.error('[Compliance] POST /vcl/bulk-commit error:', err.message);
            res.status(500).json({ error: 'Failed to commit changes' });
        } finally {
            client.release();
        }
    });

    return router;
}

module.exports = { createComplianceRouter, bucketAgingItems, computeWaterfall };