04360cc4bc
New feature: multi-file per-vertical compliance xlsx upload with scoped resolution logic, executive-level aggregated reporting, and drill-down by vertical and metric. Supports daily upload cadence and batch commit. Backend: - Migration: add vertical column to compliance_items/uploads, create vcl_multi_vertical_summary table - New route module: routes/vclMultiVertical.js with preview, commit, stats, trend, metric drill-down, device list, and burndown endpoints - New helpers: parseVerticalFilename(), computeVerticalBurndown() - Vertical-scoped resolution: uploading one vertical never resolves items from other verticals Frontend: - CCPMetricsPage with stats bar, trend chart, donut, vertical table - Drill-down: vertical -> metrics by category -> device list - Per-vertical burndown forecast chart - MultiVerticalUploadModal: multi-file drag-drop, batch preview, commit - Nav entry: CCP Metrics (Building2 icon) Docs: - Design brief for stakeholder meeting (docs/vcl-multi-vertical-design-brief.md)
881 lines
35 KiB
JavaScript
881 lines
35 KiB
JavaScript
// VCL Multi-Vertical Routes — Cross-organizational compliance reporting
|
||
// Handles multi-file per-vertical xlsx upload, scoped resolution, and executive reporting.
|
||
|
||
const express = require('express');
|
||
const path = require('path');
|
||
const fs = require('fs');
|
||
const { spawn } = require('child_process');
|
||
const pool = require('../db');
|
||
const { requireAuth, requireGroup } = require('../middleware/auth');
|
||
const { parseVerticalFilename, computeVerticalBurndown, isValidDateString, categorizeNonCompliant } = require('../helpers/vclHelpers');
|
||
const logAudit = require('../helpers/auditLog');
|
||
|
||
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
|
||
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
|
||
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
|
||
const VCL_TARGET_PCT = parseInt(process.env.VCL_TARGET_PCT, 10) || 95;
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Run Python parser, return parsed object
|
||
// ---------------------------------------------------------------------------
|
||
function parseXlsx(filePath) {
|
||
return new Promise((resolve, reject) => {
|
||
const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
|
||
let out = '';
|
||
let err = '';
|
||
py.stdout.on('data', d => { out += d; });
|
||
py.stderr.on('data', d => { err += d; });
|
||
py.on('close', code => {
|
||
if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
|
||
try { resolve(JSON.parse(out)); }
|
||
catch (e) { reject(new Error('Parser returned invalid JSON')); }
|
||
});
|
||
py.on('error', reject);
|
||
});
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Compute scoped diff: only considers items within the same vertical
|
||
// ---------------------------------------------------------------------------
|
||
async function computeScopedDiff(incomingItems, vertical) {
|
||
const { rows: activeRows } = await pool.query(
|
||
`SELECT hostname, metric_id FROM compliance_items WHERE status = 'active' AND vertical = $1`,
|
||
[vertical]
|
||
);
|
||
const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
|
||
const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));
|
||
|
||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
|
||
for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; }
|
||
|
||
return { newCount, recurringCount, resolvedCount };
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Persist a single vertical's upload with scoped resolution
|
||
// ---------------------------------------------------------------------------
|
||
async function persistMultiVerticalUpload({ items, summary, reportDate, filename, vertical, userId }, client) {
|
||
// Get active items for THIS vertical only
|
||
const { rows: activeRows } = await client.query(
|
||
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items
|
||
WHERE status = 'active' AND vertical = $1`,
|
||
[vertical]
|
||
);
|
||
const activeMap = {};
|
||
activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });
|
||
|
||
const newKeys = new Set(items.map(i => `${i.hostname}|||${i.metric_id}`));
|
||
|
||
// 1. Insert the upload record
|
||
const uploadResult = await client.query(
|
||
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, vertical, summary_json)
|
||
VALUES ($1, $2, $3, NOW(), $4, $5)
|
||
RETURNING id`,
|
||
[filename, reportDate || null, userId || null, vertical, JSON.stringify(summary)]
|
||
);
|
||
const uploadId = uploadResult.rows[0].id;
|
||
|
||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||
|
||
// 2. Upsert each incoming non-compliant item
|
||
for (const item of items) {
|
||
const key = `${item.hostname}|||${item.metric_id}`;
|
||
const existing = activeMap[key];
|
||
const extraStr = JSON.stringify(item.extra_json || {});
|
||
|
||
if (existing) {
|
||
await client.query(
|
||
`UPDATE compliance_items
|
||
SET upload_id = $1, seen_count = $2, ip_address = $3, device_type = $4, extra_json = $5,
|
||
metric_desc = $6, category = $7, team = $8
|
||
WHERE id = $9`,
|
||
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr,
|
||
item.metric_desc, item.category, item.team, existing.id]
|
||
);
|
||
recurringCount++;
|
||
} else {
|
||
await client.query(
|
||
`INSERT INTO compliance_items
|
||
(upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
|
||
category, extra_json, status, first_seen_upload_id, seen_count, vertical)
|
||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, 1, $11)`,
|
||
[uploadId, item.hostname, item.ip_address, item.device_type, item.team,
|
||
item.metric_id, item.metric_desc, item.category, extraStr, uploadId, vertical]
|
||
);
|
||
newCount++;
|
||
}
|
||
}
|
||
|
||
// 3. Resolve items NOT present in this upload — SCOPED to this vertical only
|
||
for (const [key, row] of Object.entries(activeMap)) {
|
||
if (!newKeys.has(key)) {
|
||
await client.query(
|
||
`UPDATE compliance_items SET status = 'resolved', resolved_upload_id = $1 WHERE id = $2`,
|
||
[uploadId, row.id]
|
||
);
|
||
resolvedCount++;
|
||
}
|
||
}
|
||
|
||
// 4. Update upload with final counts
|
||
await client.query(
|
||
`UPDATE compliance_uploads SET new_count = $1, resolved_count = $2, recurring_count = $3 WHERE id = $4`,
|
||
[newCount, resolvedCount, recurringCount, uploadId]
|
||
);
|
||
|
||
// 5. Store summary entries in vcl_multi_vertical_summary
|
||
if (summary && summary.entries && summary.entries.length > 0) {
|
||
for (const entry of summary.entries) {
|
||
await client.query(
|
||
`INSERT INTO vcl_multi_vertical_summary
|
||
(upload_id, vertical, metric_id, metric_desc, category, team, priority,
|
||
non_compliant, compliant, total, compliance_pct, target, status)
|
||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13)`,
|
||
[uploadId, vertical, entry.metric_id, entry.description || entry.metric_desc || '',
|
||
entry.category || 'Other', entry.team || '', entry.priority || '',
|
||
entry.non_compliant || 0, entry.compliant || 0, entry.total || 0,
|
||
entry.compliance_pct || 0, entry.target || 0, entry.status || '']
|
||
);
|
||
}
|
||
}
|
||
|
||
// 6. Create/update compliance_snapshots for this vertical
|
||
const currentMonth = new Date().toISOString().slice(0, 7);
|
||
const { rows: verticalStats } = await client.query(
|
||
`SELECT
|
||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||
FROM compliance_items
|
||
WHERE vertical = $1`,
|
||
[vertical]
|
||
);
|
||
const vs = verticalStats[0] || { total_devices: 0, non_compliant: 0 };
|
||
const totalDevices = vs.total_devices;
|
||
const compliant = totalDevices - vs.non_compliant;
|
||
const compPct = totalDevices > 0 ? Math.round((compliant / totalDevices) * 100 * 100) / 100 : 0;
|
||
|
||
await client.query(
|
||
`INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||
VALUES ($1, $2, $3, $4, $5, $6)
|
||
ON CONFLICT (snapshot_month, vertical)
|
||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||
[currentMonth, vertical, totalDevices, compliant, vs.non_compliant, compPct]
|
||
);
|
||
|
||
return { uploadId, newCount, recurringCount, resolvedCount };
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Safe temp path check
|
||
// ---------------------------------------------------------------------------
|
||
function isSafeTempPath(filePath) {
|
||
const resolved = path.resolve(filePath);
|
||
return resolved.startsWith(path.resolve(TEMP_DIR) + path.sep) && path.extname(resolved) === '.json';
|
||
}
|
||
|
||
// ---------------------------------------------------------------------------
|
||
// Router factory
|
||
// ---------------------------------------------------------------------------
|
||
function createVCLMultiVerticalRouter(upload) {
|
||
const router = express.Router();
|
||
|
||
// All routes require authentication
|
||
router.use(requireAuth());
|
||
|
||
/**
|
||
* POST /preview
|
||
* Accepts multiple xlsx files, parses each, extracts vertical from filename,
|
||
* computes per-vertical scoped diffs, and stores parsed data in temp files.
|
||
*
|
||
* @method POST
|
||
* @route /preview
|
||
* @group Admin, Standard_User
|
||
*
|
||
* @body multipart/form-data
|
||
* - files: File[] — 1–14 xlsx files with naming convention <VERTICAL>_YYYY_MM_DD.xlsx
|
||
*
|
||
* @response 200
|
||
* {
|
||
* files: Array<{
|
||
* filename: string,
|
||
* vertical: string,
|
||
* report_date: string,
|
||
* total_items: number,
|
||
* summary_entries: number,
|
||
* diff: { new_count: number, recurring_count: number, resolved_count: number },
|
||
* tempFile: string
|
||
* }>,
|
||
* unrecognized: Array<{ filename: string, error: string }>
|
||
* }
|
||
* @response 400 { error: string } — upload error or no files provided
|
||
*/
|
||
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
|
||
upload.array('files', 14)(req, res, async (uploadErr) => {
|
||
if (uploadErr) return res.status(400).json({ error: uploadErr.message });
|
||
if (!req.files || req.files.length === 0) return res.status(400).json({ error: 'No files uploaded' });
|
||
|
||
const results = [];
|
||
const unrecognized = [];
|
||
const seenVerticals = new Set();
|
||
|
||
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||
|
||
for (const file of req.files) {
|
||
const ext = path.extname(file.originalname).toLowerCase();
|
||
if (ext !== '.xlsx') {
|
||
unrecognized.push({ filename: file.originalname, error: 'Not an xlsx file' });
|
||
fs.unlink(file.path, () => {});
|
||
continue;
|
||
}
|
||
|
||
// Extract vertical from filename
|
||
const parsed = parseVerticalFilename(file.originalname);
|
||
if (!parsed) {
|
||
unrecognized.push({ filename: file.originalname, error: 'Filename does not match pattern <VERTICAL>_YYYY_MM_DD.xlsx' });
|
||
fs.unlink(file.path, () => {});
|
||
continue;
|
||
}
|
||
|
||
// Check for duplicate verticals in the same batch
|
||
if (seenVerticals.has(parsed.vertical)) {
|
||
unrecognized.push({ filename: file.originalname, error: `Duplicate vertical "${parsed.vertical}" in batch` });
|
||
fs.unlink(file.path, () => {});
|
||
continue;
|
||
}
|
||
seenVerticals.add(parsed.vertical);
|
||
|
||
try {
|
||
const xlsxData = await parseXlsx(file.path);
|
||
if (xlsxData.error) {
|
||
unrecognized.push({ filename: file.originalname, error: xlsxData.error });
|
||
fs.unlink(file.path, () => {});
|
||
continue;
|
||
}
|
||
|
||
// Compute scoped diff for this vertical
|
||
const diff = await computeScopedDiff(xlsxData.items, parsed.vertical);
|
||
|
||
// Store parsed data in temp file
|
||
const tempFilename = `vcl_multi_${parsed.vertical}_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
|
||
const tempFilePath = path.join(TEMP_DIR, tempFilename);
|
||
fs.writeFileSync(tempFilePath, JSON.stringify({
|
||
items: xlsxData.items,
|
||
summary: xlsxData.summary,
|
||
report_date: parsed.date,
|
||
vertical: parsed.vertical,
|
||
filename: file.originalname.replace(/[^\w.\-() ]/g, '_'),
|
||
}));
|
||
|
||
results.push({
|
||
filename: file.originalname,
|
||
vertical: parsed.vertical,
|
||
report_date: parsed.date,
|
||
total_items: xlsxData.total || xlsxData.items.length,
|
||
summary_entries: (xlsxData.summary && xlsxData.summary.entries) ? xlsxData.summary.entries.length : 0,
|
||
diff: { new_count: diff.newCount, recurring_count: diff.recurringCount, resolved_count: diff.resolvedCount },
|
||
tempFile: tempFilePath,
|
||
});
|
||
} catch (parseErr) {
|
||
unrecognized.push({ filename: file.originalname, error: parseErr.message });
|
||
} finally {
|
||
fs.unlink(file.path, () => {});
|
||
}
|
||
}
|
||
|
||
res.json({ files: results, unrecognized });
|
||
});
|
||
});
|
||
|
||
/**
|
||
* POST /commit
|
||
* Commits all previewed files in a single transaction with vertical-scoped resolution.
|
||
*
|
||
* @method POST
|
||
* @route /commit
|
||
* @group Admin, Standard_User
|
||
*
|
||
* @body application/json
|
||
* {
|
||
* files: Array<{
|
||
* tempFile: string,
|
||
* vertical?: string,
|
||
* report_date?: string,
|
||
* filename?: string
|
||
* }>
|
||
* }
|
||
*
|
||
* @response 200
|
||
* {
|
||
* committed: Array<{
|
||
* vertical: string,
|
||
* upload_id: number,
|
||
* new_count: number,
|
||
* recurring_count: number,
|
||
* resolved_count: number
|
||
* }>,
|
||
* total_new: number,
|
||
* total_resolved: number
|
||
* }
|
||
* @response 400 { error: string } — invalid/missing tempFile or expired preview session
|
||
* @response 500 { error: string } — transaction failure
|
||
*/
|
||
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||
const { files } = req.body;
|
||
if (!files || !Array.isArray(files) || files.length === 0) {
|
||
return res.status(400).json({ error: 'files array is required' });
|
||
}
|
||
|
||
// Validate all temp files exist before starting transaction
|
||
for (const file of files) {
|
||
if (!file.tempFile || !isSafeTempPath(file.tempFile)) {
|
||
return res.status(400).json({ error: `Invalid tempFile path for ${file.vertical || 'unknown'}` });
|
||
}
|
||
if (!fs.existsSync(file.tempFile)) {
|
||
return res.status(400).json({ error: `Preview session expired for ${file.vertical || 'unknown'} — please upload again` });
|
||
}
|
||
}
|
||
|
||
const client = await pool.connect();
|
||
try {
|
||
await client.query('BEGIN');
|
||
|
||
const committed = [];
|
||
for (const file of files) {
|
||
let parsed;
|
||
try { parsed = JSON.parse(fs.readFileSync(file.tempFile, 'utf8')); }
|
||
catch { throw new Error(`Could not read preview data for ${file.vertical}`); }
|
||
|
||
const result = await persistMultiVerticalUpload({
|
||
items: parsed.items,
|
||
summary: parsed.summary,
|
||
reportDate: file.report_date || parsed.report_date,
|
||
filename: file.filename || parsed.filename,
|
||
vertical: file.vertical || parsed.vertical,
|
||
userId: req.user?.id || null,
|
||
}, client);
|
||
|
||
committed.push({
|
||
vertical: file.vertical || parsed.vertical,
|
||
upload_id: result.uploadId,
|
||
new_count: result.newCount,
|
||
recurring_count: result.recurringCount,
|
||
resolved_count: result.resolvedCount,
|
||
});
|
||
}
|
||
|
||
await client.query('COMMIT');
|
||
|
||
// Clean up temp files
|
||
for (const file of files) {
|
||
fs.unlink(file.tempFile, () => {});
|
||
}
|
||
|
||
// Audit log
|
||
logAudit({
|
||
userId: req.user.id,
|
||
username: req.user.username,
|
||
action: 'vcl_multi_vertical_upload',
|
||
entityType: 'compliance_uploads',
|
||
entityId: null,
|
||
details: {
|
||
verticals: committed.map(c => c.vertical),
|
||
total_new: committed.reduce((s, c) => s + c.new_count, 0),
|
||
total_resolved: committed.reduce((s, c) => s + c.resolved_count, 0),
|
||
},
|
||
ipAddress: req.ip,
|
||
});
|
||
|
||
res.json({
|
||
committed,
|
||
total_new: committed.reduce((s, c) => s + c.new_count, 0),
|
||
total_resolved: committed.reduce((s, c) => s + c.resolved_count, 0),
|
||
});
|
||
} catch (err) {
|
||
await client.query('ROLLBACK');
|
||
// Clean up temp files on failure too
|
||
for (const file of files) {
|
||
if (file.tempFile) fs.unlink(file.tempFile, () => {});
|
||
}
|
||
console.error('[VCL Multi] Commit error:', err.message);
|
||
res.status(500).json({ error: 'Failed to commit batch: ' + err.message });
|
||
} finally {
|
||
client.release();
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /stats
|
||
* Returns aggregated cross-vertical executive summary statistics.
|
||
*
|
||
* @method GET
|
||
* @route /stats
|
||
*
|
||
* @response 200
|
||
* {
|
||
* stats: {
|
||
* total_devices: number,
|
||
* compliant: number,
|
||
* non_compliant: number,
|
||
* compliance_pct: number,
|
||
* target_pct: number
|
||
* },
|
||
* donut: { blocked: number, in_progress: number },
|
||
* vertical_breakdown: Array<{
|
||
* vertical: string,
|
||
* total_devices: number,
|
||
* compliant: number,
|
||
* non_compliant: number,
|
||
* compliance_pct: number,
|
||
* blockers: number,
|
||
* forecast_burndown: Array<{ month: string, projected_remaining: number }>,
|
||
* last_upload: string|null
|
||
* }>,
|
||
* last_upload_date: string|null
|
||
* }
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/stats', async (req, res) => {
|
||
try {
|
||
// Aggregate device-level stats across all multi-vertical items
|
||
const { rows: statsRows } = await pool.query(`
|
||
SELECT
|
||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||
FROM compliance_items
|
||
WHERE vertical IS NOT NULL
|
||
`);
|
||
|
||
const raw = statsRows[0] || { total_devices: 0, non_compliant: 0 };
|
||
const total_devices = raw.total_devices;
|
||
const non_compliant = raw.non_compliant;
|
||
const compliant = total_devices - non_compliant;
|
||
const compliance_pct = total_devices > 0 ? Math.round((compliant / total_devices) * 100) : 0;
|
||
|
||
// Donut: blocked vs in-progress across all verticals
|
||
const { rows: donutRows } = await pool.query(`
|
||
SELECT hostname, MAX(resolution_date) AS resolution_date
|
||
FROM compliance_items
|
||
WHERE vertical IS NOT NULL AND status = 'active'
|
||
GROUP BY hostname
|
||
`);
|
||
const donutItems = donutRows.map(r => ({ resolution_date: r.resolution_date }));
|
||
const donut = categorizeNonCompliant(donutItems);
|
||
|
||
// Per-vertical breakdown
|
||
const { rows: verticalRows } = await pool.query(`
|
||
SELECT
|
||
vertical,
|
||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||
FROM compliance_items
|
||
WHERE vertical IS NOT NULL
|
||
GROUP BY vertical
|
||
ORDER BY vertical
|
||
`);
|
||
|
||
// Get last upload date per vertical
|
||
const { rows: uploadDates } = await pool.query(`
|
||
SELECT vertical, MAX(report_date) AS last_upload
|
||
FROM compliance_uploads
|
||
WHERE vertical IS NOT NULL
|
||
GROUP BY vertical
|
||
`);
|
||
const uploadDateMap = {};
|
||
uploadDates.forEach(r => { uploadDateMap[r.vertical] = r.last_upload; });
|
||
|
||
// Get burndown data per vertical
|
||
const { rows: burndownRows } = await pool.query(`
|
||
SELECT vertical, hostname, resolution_date
|
||
FROM compliance_items
|
||
WHERE vertical IS NOT NULL AND status = 'active'
|
||
`);
|
||
// Group by vertical for burndown computation
|
||
const burndownByVertical = {};
|
||
for (const row of burndownRows) {
|
||
if (!burndownByVertical[row.vertical]) burndownByVertical[row.vertical] = [];
|
||
burndownByVertical[row.vertical].push(row);
|
||
}
|
||
|
||
const vertical_breakdown = verticalRows.map(v => {
|
||
const totalDev = v.total_devices;
|
||
const comp = totalDev - v.non_compliant;
|
||
const pct = totalDev > 0 ? Math.round((comp / totalDev) * 100) : 0;
|
||
const items = burndownByVertical[v.vertical] || [];
|
||
const burndown = computeVerticalBurndown(items);
|
||
|
||
return {
|
||
vertical: v.vertical,
|
||
total_devices: totalDev,
|
||
compliant: comp,
|
||
non_compliant: v.non_compliant,
|
||
compliance_pct: pct,
|
||
blockers: burndown.blockers,
|
||
forecast_burndown: burndown.monthly,
|
||
last_upload: uploadDateMap[v.vertical] || null,
|
||
};
|
||
});
|
||
|
||
res.json({
|
||
stats: {
|
||
total_devices,
|
||
compliant,
|
||
non_compliant,
|
||
compliance_pct,
|
||
target_pct: VCL_TARGET_PCT,
|
||
},
|
||
donut,
|
||
vertical_breakdown,
|
||
last_upload_date: uploadDates.length > 0 ? uploadDates.reduce((max, r) => r.last_upload > max ? r.last_upload : max, '') : null,
|
||
});
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /stats error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /trend
|
||
* Returns monthly compliance trend data aggregated across all verticals.
|
||
* Includes linear regression forecast when 3+ months of data exist.
|
||
*
|
||
* @method GET
|
||
* @route /trend
|
||
*
|
||
* @response 200
|
||
* {
|
||
* months: Array<{
|
||
* month: string,
|
||
* compliant_count: number|null,
|
||
* compliance_pct: number|null,
|
||
* forecast_pct: number|null,
|
||
* target_pct: number
|
||
* }>
|
||
* }
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/trend', async (req, res) => {
|
||
try {
|
||
// Get snapshots for multi-vertical data (vertical IS NOT NULL)
|
||
const { rows: snapshots } = await pool.query(`
|
||
SELECT snapshot_month, SUM(total_devices)::int AS total_devices,
|
||
SUM(compliant)::int AS compliant, SUM(non_compliant)::int AS non_compliant
|
||
FROM compliance_snapshots
|
||
WHERE vertical IS NOT NULL AND vertical != ''
|
||
GROUP BY snapshot_month
|
||
ORDER BY snapshot_month ASC
|
||
`);
|
||
|
||
if (snapshots.length === 0) return res.json({ months: [] });
|
||
|
||
const months = snapshots.map(s => {
|
||
const total = s.total_devices;
|
||
const pct = total > 0 ? Math.round((s.compliant / total) * 100 * 10) / 10 : 0;
|
||
return {
|
||
month: s.snapshot_month,
|
||
compliant_count: s.compliant,
|
||
compliance_pct: pct,
|
||
forecast_pct: null,
|
||
target_pct: VCL_TARGET_PCT,
|
||
};
|
||
});
|
||
|
||
// Compute forecast using linear regression if we have 3+ months
|
||
if (months.length >= 3) {
|
||
const n = months.length;
|
||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
|
||
for (let i = 0; i < n; i++) {
|
||
sumX += i;
|
||
sumY += months[i].compliance_pct;
|
||
sumXY += i * months[i].compliance_pct;
|
||
sumX2 += i * i;
|
||
}
|
||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
|
||
const intercept = (sumY - slope * sumX) / n;
|
||
|
||
// Project forward 3 months
|
||
for (let i = 0; i < 3; i++) {
|
||
const futureIdx = n + i;
|
||
const forecastPct = Math.min(100, Math.max(0, Math.round((slope * futureIdx + intercept) * 10) / 10));
|
||
|
||
const lastMonth = months[months.length - 1].month;
|
||
const [year, mon] = lastMonth.split('-').map(Number);
|
||
const futureDate = new Date(year, mon - 1 + i + 1, 1);
|
||
const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`;
|
||
|
||
months.push({
|
||
month: futureMonth,
|
||
compliant_count: null,
|
||
compliance_pct: null,
|
||
forecast_pct: forecastPct,
|
||
target_pct: VCL_TARGET_PCT,
|
||
});
|
||
}
|
||
|
||
// Add forecast_pct to last actual month as starting point
|
||
if (n > 0) {
|
||
months[n - 1].forecast_pct = months[n - 1].compliance_pct;
|
||
}
|
||
}
|
||
|
||
res.json({ months });
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /trend error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /vertical/:code/metrics
|
||
* Returns per-metric breakdown for a specific vertical from the latest upload's summary data.
|
||
*
|
||
* @method GET
|
||
* @route /vertical/:code/metrics
|
||
* @param {string} code — vertical code (e.g., "NTS_AEO", "SDIT_CISO")
|
||
*
|
||
* @response 200
|
||
* {
|
||
* vertical: string,
|
||
* metrics: Array<{
|
||
* metric_id: string,
|
||
* metric_desc: string,
|
||
* category: string,
|
||
* team: string,
|
||
* priority: string,
|
||
* non_compliant: number,
|
||
* compliant: number,
|
||
* total: number,
|
||
* compliance_pct: number,
|
||
* target: number,
|
||
* status: string
|
||
* }>,
|
||
* categories: Array<{
|
||
* category: string,
|
||
* non_compliant: number,
|
||
* compliant: number,
|
||
* total: number,
|
||
* compliance_pct: number
|
||
* }>
|
||
* }
|
||
* @response 400 { error: string } — invalid vertical code
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/vertical/:code/metrics', async (req, res) => {
|
||
const vertical = req.params.code;
|
||
if (!vertical || vertical.length > 100) return res.status(400).json({ error: 'Invalid vertical code' });
|
||
|
||
try {
|
||
// Get the latest upload for this vertical
|
||
const { rows: latestUpload } = await pool.query(
|
||
`SELECT id FROM compliance_uploads WHERE vertical = $1 ORDER BY id DESC LIMIT 1`,
|
||
[vertical]
|
||
);
|
||
|
||
if (latestUpload.length === 0) return res.json({ vertical, metrics: [], categories: [] });
|
||
|
||
const uploadId = latestUpload[0].id;
|
||
|
||
// Get per-metric summary data
|
||
const { rows: metrics } = await pool.query(
|
||
`SELECT metric_id, metric_desc, category, team, priority,
|
||
non_compliant, compliant, total, compliance_pct, target, status
|
||
FROM vcl_multi_vertical_summary
|
||
WHERE upload_id = $1 AND vertical = $2
|
||
ORDER BY category, metric_id`,
|
||
[uploadId, vertical]
|
||
);
|
||
|
||
// Aggregate by category
|
||
const categoryMap = {};
|
||
for (const m of metrics) {
|
||
const cat = m.category || 'Other';
|
||
if (!categoryMap[cat]) categoryMap[cat] = { category: cat, non_compliant: 0, compliant: 0, total: 0 };
|
||
categoryMap[cat].non_compliant += m.non_compliant;
|
||
categoryMap[cat].compliant += m.compliant;
|
||
categoryMap[cat].total += m.total;
|
||
}
|
||
const categories = Object.values(categoryMap).map(c => ({
|
||
...c,
|
||
compliance_pct: c.total > 0 ? Math.round((c.compliant / c.total) * 100 * 10) / 10 : 0,
|
||
}));
|
||
|
||
res.json({ vertical, metrics, categories });
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /vertical/:code/metrics error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /vertical/:code/metric/:metricId/devices
|
||
* Returns the list of non-compliant devices for a specific vertical + metric.
|
||
*
|
||
* @method GET
|
||
* @route /vertical/:code/metric/:metricId/devices
|
||
* @param {string} code — vertical code (e.g., "NTS_AEO")
|
||
* @param {string} metricId — metric identifier (e.g., "VM-001")
|
||
*
|
||
* @response 200
|
||
* {
|
||
* vertical: string,
|
||
* metric_id: string,
|
||
* devices: Array<{
|
||
* hostname: string,
|
||
* ip_address: string,
|
||
* device_type: string,
|
||
* team: string,
|
||
* seen_count: number,
|
||
* resolution_date: string|null,
|
||
* remediation_plan: string|null,
|
||
* first_seen: string|null,
|
||
* last_seen: string|null
|
||
* }>
|
||
* }
|
||
* @response 400 { error: string } — invalid vertical code or metric ID
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/vertical/:code/metric/:metricId/devices', async (req, res) => {
|
||
const vertical = req.params.code;
|
||
const metricId = req.params.metricId;
|
||
if (!vertical || vertical.length > 100) return res.status(400).json({ error: 'Invalid vertical code' });
|
||
if (!metricId || metricId.length > 50) return res.status(400).json({ error: 'Invalid metric ID' });
|
||
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT ci.hostname, ci.ip_address, ci.device_type, ci.team, ci.seen_count,
|
||
ci.resolution_date, ci.remediation_plan,
|
||
fu.report_date AS first_seen, lu.report_date AS last_seen
|
||
FROM compliance_items ci
|
||
LEFT JOIN compliance_uploads fu ON ci.first_seen_upload_id = fu.id
|
||
LEFT JOIN compliance_uploads lu ON ci.upload_id = lu.id
|
||
WHERE ci.vertical = $1 AND ci.metric_id = $2 AND ci.status = 'active'
|
||
ORDER BY ci.hostname`,
|
||
[vertical, metricId]
|
||
);
|
||
|
||
res.json({ vertical, metric_id: metricId, devices: rows });
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /vertical/:code/metric/:metricId/devices error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /vertical/:code/burndown
|
||
* Returns burndown forecast for a specific vertical.
|
||
* Deduplicates devices by hostname and computes monthly projected resolution.
|
||
*
|
||
* @method GET
|
||
* @route /vertical/:code/burndown
|
||
* @param {string} code — vertical code (e.g., "TSI")
|
||
*
|
||
* @response 200
|
||
* {
|
||
* vertical: string,
|
||
* blockers: number,
|
||
* in_progress: number,
|
||
* monthly: Array<{ month: string, projected_remaining: number }>
|
||
* }
|
||
* @response 400 { error: string } — invalid vertical code
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/vertical/:code/burndown', async (req, res) => {
|
||
const vertical = req.params.code;
|
||
if (!vertical || vertical.length > 100) return res.status(400).json({ error: 'Invalid vertical code' });
|
||
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT hostname, resolution_date
|
||
FROM compliance_items
|
||
WHERE vertical = $1 AND status = 'active'`,
|
||
[vertical]
|
||
);
|
||
|
||
// Deduplicate by hostname (a device may have multiple failing metrics)
|
||
const deviceMap = {};
|
||
for (const row of rows) {
|
||
if (!deviceMap[row.hostname]) {
|
||
deviceMap[row.hostname] = { hostname: row.hostname, resolution_date: row.resolution_date };
|
||
} else if (row.resolution_date && !deviceMap[row.hostname].resolution_date) {
|
||
// If any metric has a resolution date, the device counts as "in progress"
|
||
deviceMap[row.hostname].resolution_date = row.resolution_date;
|
||
}
|
||
}
|
||
const devices = Object.values(deviceMap);
|
||
const burndown = computeVerticalBurndown(devices);
|
||
|
||
res.json({ vertical, ...burndown });
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /vertical/:code/burndown error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /uploads
|
||
* Returns upload history for multi-vertical uploads (most recent 100).
|
||
*
|
||
* @method GET
|
||
* @route /uploads
|
||
*
|
||
* @response 200
|
||
* {
|
||
* uploads: Array<{
|
||
* id: number,
|
||
* filename: string,
|
||
* report_date: string|null,
|
||
* uploaded_at: string,
|
||
* vertical: string,
|
||
* new_count: number,
|
||
* resolved_count: number,
|
||
* recurring_count: number
|
||
* }>
|
||
* }
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/uploads', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT id, filename, report_date, uploaded_at, vertical, new_count, resolved_count, recurring_count
|
||
FROM compliance_uploads
|
||
WHERE vertical IS NOT NULL
|
||
ORDER BY id DESC
|
||
LIMIT 100`
|
||
);
|
||
res.json({ uploads: rows });
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /uploads error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
/**
|
||
* GET /verticals
|
||
* Returns the list of known verticals derived from compliance_items records.
|
||
*
|
||
* @method GET
|
||
* @route /verticals
|
||
*
|
||
* @response 200
|
||
* {
|
||
* verticals: string[]
|
||
* }
|
||
* @response 500 { error: string }
|
||
*/
|
||
router.get('/verticals', async (req, res) => {
|
||
try {
|
||
const { rows } = await pool.query(
|
||
`SELECT DISTINCT vertical FROM compliance_items WHERE vertical IS NOT NULL ORDER BY vertical`
|
||
);
|
||
res.json({ verticals: rows.map(r => r.vertical) });
|
||
} catch (err) {
|
||
console.error('[VCL Multi] GET /verticals error:', err.message);
|
||
res.status(500).json({ error: 'Database error' });
|
||
}
|
||
});
|
||
|
||
return router;
|
||
}
|
||
|
||
module.exports = { createVCLMultiVerticalRouter };
|