Add VCL compliance reporting: exec report page, device metadata fields, bulk upload
This commit is contained in:
Jordan Ramos
501
backend/__tests__/vcl-compliance-reporting.property.test.js
Normal file
501
backend/__tests__/vcl-compliance-reporting.property.test.js
Normal file
@@ -0,0 +1,501 @@
|
||||
/**
|
||||
* Property-Based Tests: VCL Compliance Reporting
|
||||
*
|
||||
* Feature: vcl-compliance-reporting
|
||||
*
|
||||
* Tests the pure helper functions used for VCL compliance reporting computations.
|
||||
*
|
||||
* Validates: Requirements 2.4, 2.5, 3.2, 3.3, 5.2, 5.3, 6.1, 6.3, 7.5, 8.2, 8.3, 8.4, 8.7, 9.2, 9.3, 9.6
|
||||
*/
|
||||
|
||||
const fc = require('fast-check');
|
||||
|
||||
// Mock db pool before importing anything (avoids DATABASE_URL requirement)
|
||||
jest.mock('../db', () => ({
|
||||
query: jest.fn(() => Promise.resolve({ rows: [], rowCount: 0 })),
|
||||
connect: jest.fn(() => Promise.resolve({
|
||||
query: jest.fn(() => Promise.resolve({ rows: [], rowCount: 0 })),
|
||||
release: jest.fn(),
|
||||
})),
|
||||
}));
|
||||
|
||||
// Mock dependencies that the route module imports
|
||||
jest.mock('../helpers/auditLog', () => jest.fn());
|
||||
jest.mock('../helpers/ivantiApi', () => ({
|
||||
ivantiFormPost: jest.fn(),
|
||||
ivantiPost: jest.fn(),
|
||||
}));
|
||||
|
||||
const {
|
||||
truncateText,
|
||||
validateRemediationPlan,
|
||||
computeVCLStats,
|
||||
formatPct,
|
||||
categorizeNonCompliant,
|
||||
rankHeavyHitters,
|
||||
computeForecastBurndown,
|
||||
matchByHostname,
|
||||
computeBulkDiff,
|
||||
mapColumnHeaders,
|
||||
isValidDateString,
|
||||
} = require('../helpers/vclHelpers');
|
||||
|
||||
// --- Generators ---
|
||||
|
||||
const hostnameArb = fc.stringMatching(/^[a-zA-Z0-9._-]+$/, { minLength: 1, maxLength: 30 });
|
||||
|
||||
const validDateArb = fc.record({
|
||||
year: fc.integer({ min: 2020, max: 2030 }),
|
||||
month: fc.integer({ min: 1, max: 12 }),
|
||||
day: fc.integer({ min: 1, max: 28 }), // 1-28 always valid
|
||||
}).map(({ year, month, day }) =>
|
||||
`${year}-${String(month).padStart(2, '0')}-${String(day).padStart(2, '0')}`
|
||||
);
|
||||
|
||||
const complianceItemArb = fc.record({
|
||||
hostname: hostnameArb,
|
||||
is_compliant: fc.boolean(),
|
||||
in_scope: fc.constant(true),
|
||||
});
|
||||
|
||||
const nonCompliantItemArb = fc.record({
|
||||
hostname: hostnameArb,
|
||||
resolution_date: fc.oneof(fc.constant(null), validDateArb),
|
||||
});
|
||||
|
||||
const verticalArb = fc.record({
|
||||
vertical: fc.string({ minLength: 1, maxLength: 20 }),
|
||||
team: fc.string({ minLength: 1, maxLength: 20 }),
|
||||
non_compliant: fc.integer({ min: 0, max: 1000 }),
|
||||
});
|
||||
|
||||
// --- Property 2: Text Truncation ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 2: Text Truncation', () => {
|
||||
/**
|
||||
* For any string, truncateText(text, 80) should return the original string if its
|
||||
* length is <= 80, or the first 80 characters followed by "…" if its length exceeds 80.
|
||||
*
|
||||
* **Validates: Requirements 2.4**
|
||||
*/
|
||||
it('returns original for short strings, truncated + ellipsis for long strings', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.string({ minLength: 0, maxLength: 200 }),
|
||||
fc.integer({ min: 1, max: 100 }),
|
||||
(text, maxLen) => {
|
||||
const result = truncateText(text, maxLen);
|
||||
if (text.length <= maxLen) {
|
||||
expect(result).toBe(text);
|
||||
} else {
|
||||
expect(result).toBe(text.slice(0, maxLen) + '\u2026');
|
||||
expect(result.length).toBe(maxLen + 1);
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 3: Remediation Plan Length Validation ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 3: Remediation Plan Length Validation', () => {
|
||||
/**
|
||||
* For any string, validateRemediationPlan(text) should return valid if and only if
|
||||
* the string length is <= 2000 characters.
|
||||
*
|
||||
* **Validates: Requirements 2.5, 9.4**
|
||||
*/
|
||||
it('accepts strings <= 2000 chars, rejects longer', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.string({ minLength: 1, maxLength: 3000 }),
|
||||
(text) => {
|
||||
const result = validateRemediationPlan(text);
|
||||
if (text.length <= 2000) {
|
||||
expect(result.valid).toBe(true);
|
||||
} else {
|
||||
expect(result.valid).toBe(false);
|
||||
expect(result.error).toBeDefined();
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 4: Summary Statistics Computation Invariants ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 4: Summary Statistics Computation Invariants', () => {
|
||||
/**
|
||||
* For any set of compliance items, computeVCLStats produces correct arithmetic:
|
||||
* non_compliant + compliant = in_scope, and correct percentage.
|
||||
*
|
||||
* **Validates: Requirements 3.2, 7.3**
|
||||
*/
|
||||
it('non_compliant + compliant = in_scope, correct percentage', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(complianceItemArb, { minLength: 0, maxLength: 50 }),
|
||||
fc.integer({ min: 0, max: 100 }),
|
||||
(items, targetPct) => {
|
||||
const stats = computeVCLStats(items, targetPct);
|
||||
|
||||
// in_scope items are those with in_scope === true
|
||||
const in_scope = items.filter(i => i.in_scope).length;
|
||||
const compliant = items.filter(i => i.is_compliant).length;
|
||||
|
||||
expect(stats.non_compliant + stats.compliant).toBe(stats.in_scope);
|
||||
expect(stats.in_scope).toBe(in_scope);
|
||||
expect(stats.compliant).toBe(compliant);
|
||||
|
||||
if (in_scope > 0) {
|
||||
expect(stats.compliance_pct).toBe(Math.round((compliant / in_scope) * 100));
|
||||
} else {
|
||||
expect(stats.compliance_pct).toBe(0);
|
||||
}
|
||||
|
||||
expect(stats.target_pct).toBe(targetPct);
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 5: Percentage Formatting ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 5: Percentage Formatting', () => {
|
||||
/**
|
||||
* For any decimal number between 0 and 1, formatPct produces a string matching /^\d{1,3}%$/.
|
||||
*
|
||||
* **Validates: Requirements 3.3**
|
||||
*/
|
||||
it('produces correct percentage string matching /^\\d{1,3}%$/', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.double({ min: 0, max: 1, noNaN: true }),
|
||||
(decimal) => {
|
||||
const result = formatPct(decimal);
|
||||
expect(result).toMatch(/^\d{1,3}%$/);
|
||||
expect(result).toBe(Math.round(decimal * 100) + '%');
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 6: Non-Compliant Device Categorization Partition ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 6: Non-Compliant Device Categorization Partition', () => {
|
||||
/**
|
||||
* For any array of non-compliant device objects, categorizeNonCompliant produces
|
||||
* two groups (blocked, in_progress) where blocked.count + in_progress.count = items.length.
|
||||
*
|
||||
* **Validates: Requirements 5.2, 5.3**
|
||||
*/
|
||||
it('two groups sum to total', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(nonCompliantItemArb, { minLength: 0, maxLength: 50 }),
|
||||
(items) => {
|
||||
const result = categorizeNonCompliant(items);
|
||||
|
||||
expect(result.blocked.count + result.in_progress.count).toBe(items.length);
|
||||
|
||||
if (items.length > 0) {
|
||||
expect(result.blocked.pct).toBe(Math.round((result.blocked.count / items.length) * 100));
|
||||
expect(result.in_progress.pct).toBe(Math.round((result.in_progress.count / items.length) * 100));
|
||||
} else {
|
||||
expect(result.blocked.pct).toBe(0);
|
||||
expect(result.in_progress.pct).toBe(0);
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 7: Heavy Hitters Descending Sort ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 7: Heavy Hitters Descending Sort', () => {
|
||||
/**
|
||||
* For any array of vertical objects, rankHeavyHitters returns the array sorted
|
||||
* in non-increasing order by non_compliant.
|
||||
*
|
||||
* **Validates: Requirements 6.1, 6.3**
|
||||
*/
|
||||
it('sorted non-increasing by non_compliant', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(verticalArb, { minLength: 0, maxLength: 30 }),
|
||||
(verticals) => {
|
||||
const result = rankHeavyHitters(verticals);
|
||||
|
||||
expect(result.length).toBe(verticals.length);
|
||||
|
||||
for (let i = 1; i < result.length; i++) {
|
||||
expect(result[i - 1].non_compliant).toBeGreaterThanOrEqual(result[i].non_compliant);
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 8: Forecasted Burndown Projection ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 8: Forecasted Burndown Projection', () => {
|
||||
/**
|
||||
* For any set of non-compliant devices with resolution_date values,
|
||||
* computeForecastBurndown produces monthly buckets where the sum of all
|
||||
* monthly forecast counts equals the number of items with non-null resolution_dates.
|
||||
*
|
||||
* **Validates: Requirements 7.5**
|
||||
*/
|
||||
it('bucket sum = count of items with non-null resolution_dates', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(nonCompliantItemArb, { minLength: 0, maxLength: 50 }),
|
||||
(items) => {
|
||||
const buckets = computeForecastBurndown(items);
|
||||
const bucketSum = Object.values(buckets).reduce((sum, count) => sum + count, 0);
|
||||
const itemsWithDate = items.filter(i => i.resolution_date != null).length;
|
||||
|
||||
expect(bucketSum).toBe(itemsWithDate);
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 9: Hostname Matching with Unmatched Flagging ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 9: Hostname Matching with Unmatched Flagging', () => {
|
||||
/**
|
||||
* For any array of uploaded rows and a set of existing hostnames,
|
||||
* matchByHostname produces matched + unmatched = total, and matched hostnames
|
||||
* all exist in the set.
|
||||
*
|
||||
* **Validates: Requirements 8.2, 8.7**
|
||||
*/
|
||||
it('matched + unmatched = total, matched hostnames in set', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(fc.record({ hostname: hostnameArb }), { minLength: 0, maxLength: 30 }),
|
||||
fc.array(hostnameArb, { minLength: 0, maxLength: 20 }),
|
||||
(rows, existingList) => {
|
||||
const existingSet = new Set(existingList);
|
||||
const { matched, unmatched } = matchByHostname(rows, existingSet);
|
||||
|
||||
expect(matched.length + unmatched.length).toBe(rows.length);
|
||||
|
||||
for (const row of matched) {
|
||||
expect(existingSet.has(row.hostname)).toBe(true);
|
||||
}
|
||||
|
||||
for (const row of unmatched) {
|
||||
expect(existingSet.has(row.hostname)).toBe(false);
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 10: Bulk Diff Change Detection ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 10: Bulk Diff Change Detection', () => {
|
||||
/**
|
||||
* For any array of matched row pairs, computeBulkDiff flags a row as "changed"
|
||||
* if and only if at least one field value differs.
|
||||
*
|
||||
* **Validates: Requirements 8.3, 8.4**
|
||||
*/
|
||||
it('changed iff at least one field differs', () => {
|
||||
const fieldValueArb = fc.oneof(fc.constant(null), fc.string({ minLength: 1, maxLength: 20 }));
|
||||
|
||||
// When uploaded values match current data exactly, status should be 'unchanged'
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(hostnameArb, { minLength: 1, maxLength: 20 }).chain(hostnames => {
|
||||
// Ensure unique hostnames to avoid map overwrite issues
|
||||
const uniqueHostnames = [...new Set(hostnames)];
|
||||
return fc.tuple(
|
||||
...uniqueHostnames.map(h =>
|
||||
fc.record({
|
||||
hostname: fc.constant(h),
|
||||
resolution_date: fieldValueArb,
|
||||
remediation_plan: fieldValueArb,
|
||||
notes: fieldValueArb,
|
||||
})
|
||||
)
|
||||
);
|
||||
}),
|
||||
(matchedRows) => {
|
||||
// Build currentData with same values as uploaded
|
||||
const currentData = new Map();
|
||||
for (const row of matchedRows) {
|
||||
currentData.set(row.hostname, {
|
||||
resolution_date: row.resolution_date,
|
||||
remediation_plan: row.remediation_plan,
|
||||
notes: row.notes,
|
||||
});
|
||||
}
|
||||
|
||||
const results = computeBulkDiff(matchedRows, currentData);
|
||||
for (const r of results) {
|
||||
expect(r.status).toBe('unchanged');
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 50 }
|
||||
);
|
||||
|
||||
// When at least one field differs, status should be 'changed'
|
||||
fc.assert(
|
||||
fc.property(
|
||||
hostnameArb,
|
||||
fc.string({ minLength: 1, maxLength: 20 }),
|
||||
fc.string({ minLength: 1, maxLength: 20 }),
|
||||
(hostname, oldVal, newVal) => {
|
||||
fc.pre(oldVal !== newVal);
|
||||
|
||||
const matchedRows = [{ hostname, resolution_date: newVal }];
|
||||
const currentData = new Map();
|
||||
currentData.set(hostname, { resolution_date: oldVal, remediation_plan: null, notes: null });
|
||||
|
||||
const results = computeBulkDiff(matchedRows, currentData);
|
||||
expect(results[0].status).toBe('changed');
|
||||
}
|
||||
),
|
||||
{ numRuns: 50 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 11: Column Header Mapping ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 11: Column Header Mapping', () => {
|
||||
/**
|
||||
* mapColumnHeaders correctly identifies known columns case-insensitively.
|
||||
*
|
||||
* **Validates: Requirements 9.2**
|
||||
*/
|
||||
it('identifies known columns case-insensitively', () => {
|
||||
const knownHeaders = ['Hostname', 'Resolution Date', 'Remediation Plan', 'Notes',
|
||||
'hostname', 'resolution_date', 'remediation_plan', 'notes',
|
||||
'HOSTNAME', 'RESOLUTION DATE', 'REMEDIATION PLAN', 'NOTES'];
|
||||
|
||||
const caseVariantArb = fc.constantFrom(...knownHeaders);
|
||||
const unknownHeaderArb = fc.stringMatching(/^[a-z]{5,10}$/).filter(
|
||||
s => !['hostname', 'notes'].includes(s.toLowerCase())
|
||||
);
|
||||
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(fc.oneof(caseVariantArb, unknownHeaderArb), { minLength: 1, maxLength: 10 }),
|
||||
(headers) => {
|
||||
const mapping = mapColumnHeaders(headers);
|
||||
|
||||
// Every mapped key should be a known field
|
||||
const validKeys = new Set(['hostname', 'resolution_date', 'remediation_plan', 'notes']);
|
||||
for (const key of Object.keys(mapping)) {
|
||||
expect(validKeys.has(key)).toBe(true);
|
||||
}
|
||||
|
||||
// Check that known headers are mapped correctly
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
const normalized = headers[i].trim().toLowerCase();
|
||||
if (normalized === 'hostname') {
|
||||
expect(mapping.hostname).toBeDefined();
|
||||
}
|
||||
if (normalized === 'resolution date' || normalized === 'resolution_date') {
|
||||
expect(mapping.resolution_date).toBeDefined();
|
||||
}
|
||||
if (normalized === 'remediation plan' || normalized === 'remediation_plan') {
|
||||
expect(mapping.remediation_plan).toBeDefined();
|
||||
}
|
||||
if (normalized === 'notes') {
|
||||
expect(mapping.notes).toBeDefined();
|
||||
}
|
||||
}
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 12: Date String Validation ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 12: Date String Validation', () => {
|
||||
/**
|
||||
* isValidDateString rejects invalid calendar dates and non-date strings.
|
||||
* Returns true only for valid YYYY-MM-DD dates.
|
||||
*
|
||||
* **Validates: Requirements 9.3**
|
||||
*/
|
||||
it('rejects invalid dates and non-date strings', () => {
|
||||
// Valid dates should return true
|
||||
fc.assert(
|
||||
fc.property(validDateArb, (dateStr) => {
|
||||
expect(isValidDateString(dateStr)).toBe(true);
|
||||
}),
|
||||
{ numRuns: 50 }
|
||||
);
|
||||
|
||||
// Invalid dates should return false
|
||||
const invalidDateArb = fc.oneof(
|
||||
fc.constant(null),
|
||||
fc.constant(''),
|
||||
fc.constant('not-a-date'),
|
||||
fc.constant('2026-02-30'),
|
||||
fc.constant('2026-13-01'),
|
||||
fc.constant('2026-00-15'),
|
||||
fc.constant('abcd-ef-gh'),
|
||||
fc.integer().map(n => String(n)),
|
||||
fc.string({ minLength: 1, maxLength: 5 }),
|
||||
);
|
||||
|
||||
fc.assert(
|
||||
fc.property(invalidDateArb, (val) => {
|
||||
expect(isValidDateString(val)).toBe(false);
|
||||
}),
|
||||
{ numRuns: 50 }
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// --- Property 13: Row Count Arithmetic Invariant ---
|
||||
|
||||
describe('Feature: vcl-compliance-reporting, Property 13: Row Count Arithmetic (matched + unmatched = total)', () => {
|
||||
/**
|
||||
* For any bulk upload, matched + unmatched = total input rows.
|
||||
*
|
||||
* **Validates: Requirements 9.6**
|
||||
*/
|
||||
it('matched + unmatched = total invariant holds', () => {
|
||||
fc.assert(
|
||||
fc.property(
|
||||
fc.array(fc.record({ hostname: hostnameArb }), { minLength: 0, maxLength: 50 }),
|
||||
fc.array(hostnameArb, { minLength: 0, maxLength: 30 }),
|
||||
(rows, existingList) => {
|
||||
const existingSet = new Set(existingList);
|
||||
const { matched, unmatched } = matchByHostname(rows, existingSet);
|
||||
|
||||
// Core invariant: matched + unmatched = total
|
||||
expect(matched.length + unmatched.length).toBe(rows.length);
|
||||
}
|
||||
),
|
||||
{ numRuns: 100 }
|
||||
);
|
||||
});
|
||||
});
|
||||
316
backend/__tests__/vcl-compliance-reporting.test.js
Normal file
316
backend/__tests__/vcl-compliance-reporting.test.js
Normal file
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* Unit and Integration Tests: VCL Compliance Reporting
|
||||
*
|
||||
* Feature: vcl-compliance-reporting
|
||||
*
|
||||
* Tests cover:
|
||||
* - PATCH /items/:hostname/metadata (happy path, invalid date, plan too long, not found)
|
||||
* - GET /vcl/stats with no data (zero/empty response)
|
||||
* - Bulk preview with all unmatched hostnames
|
||||
* - Bulk preview with mixed valid/invalid rows
|
||||
* - Integration test for full bulk flow (preview → commit)
|
||||
* - Trend endpoint with < 2 months (no forecast)
|
||||
*/
|
||||
|
||||
const http = require('http');
|
||||
const express = require('express');
|
||||
|
||||
// Mock auth middleware to bypass real session checks
|
||||
jest.mock('../middleware/auth', () => ({
|
||||
requireAuth: () => (req, res, next) => {
|
||||
req.user = { id: 1, username: 'testuser', group: 'Admin' };
|
||||
next();
|
||||
},
|
||||
requireGroup: () => (req, res, next) => next(),
|
||||
}));
|
||||
|
||||
// Mock audit log as a no-op
|
||||
jest.mock('../helpers/auditLog', () => jest.fn());
|
||||
|
||||
// Mock ivantiApi to avoid real network calls
|
||||
jest.mock('../helpers/ivantiApi', () => ({
|
||||
ivantiFormPost: jest.fn(),
|
||||
ivantiPost: jest.fn(),
|
||||
}));
|
||||
|
||||
// Mock the db pool
|
||||
const mockPool = {
|
||||
query: jest.fn(() => Promise.resolve({ rows: [], rowCount: 0 })),
|
||||
connect: jest.fn(() => Promise.resolve({
|
||||
query: jest.fn(() => Promise.resolve({ rows: [], rowCount: 0 })),
|
||||
release: jest.fn(),
|
||||
})),
|
||||
};
|
||||
jest.mock('../db', () => mockPool);
|
||||
|
||||
// Mock driftChecker to avoid file system dependencies
|
||||
jest.mock('../helpers/driftChecker', () => ({
|
||||
loadConfig: jest.fn(() => ({})),
|
||||
compareSchemaToDrift: jest.fn(() => null),
|
||||
reconcileConfig: jest.fn(() => ({ changes: [] })),
|
||||
}));
|
||||
|
||||
const { createComplianceRouter } = require('../routes/compliance');
|
||||
|
||||
// --- HTTP helper ---
|
||||
|
||||
function request(server, method, path, body) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const addr = server.address();
|
||||
const options = {
|
||||
hostname: '127.0.0.1',
|
||||
port: addr.port,
|
||||
path,
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
};
|
||||
|
||||
const req = http.request(options, (res) => {
|
||||
const chunks = [];
|
||||
res.on('data', (chunk) => chunks.push(chunk));
|
||||
res.on('end', () => {
|
||||
const rawBody = Buffer.concat(chunks).toString();
|
||||
let json;
|
||||
try { json = JSON.parse(rawBody); } catch (e) { json = null; }
|
||||
resolve({ statusCode: res.statusCode, body: json });
|
||||
});
|
||||
});
|
||||
|
||||
req.on('error', reject);
|
||||
if (body) req.write(JSON.stringify(body));
|
||||
req.end();
|
||||
});
|
||||
}
|
||||
|
||||
// --- Setup ---
|
||||
|
||||
let app, server;
|
||||
|
||||
beforeAll((done) => {
|
||||
app = express();
|
||||
app.use(express.json());
|
||||
|
||||
// Mock multer upload middleware
|
||||
const mockUpload = { single: () => (req, res, next) => next() };
|
||||
const router = createComplianceRouter(mockUpload);
|
||||
app.use('/api/compliance', router);
|
||||
|
||||
server = app.listen(0, '127.0.0.1', done);
|
||||
});
|
||||
|
||||
afterAll((done) => {
|
||||
server.close(done);
|
||||
});
|
||||
|
||||
beforeEach(() => {
|
||||
mockPool.query.mockReset();
|
||||
mockPool.connect.mockReset();
|
||||
mockPool.query.mockResolvedValue({ rows: [], rowCount: 0 });
|
||||
});
|
||||
|
||||
// --- 18.1: PATCH /items/:hostname/metadata ---
|
||||
|
||||
describe('PATCH /items/:hostname/metadata', () => {
|
||||
it('happy path — updates resolution_date and remediation_plan', async () => {
|
||||
mockPool.query.mockResolvedValueOnce({ rows: [], rowCount: 2 });
|
||||
|
||||
const res = await request(server, 'PATCH', '/api/compliance/items/srv-001/metadata', {
|
||||
resolution_date: '2026-06-15',
|
||||
remediation_plan: 'Patch in next maintenance window',
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.body.updated).toBe(2);
|
||||
});
|
||||
|
||||
it('returns 400 for invalid date format', async () => {
|
||||
const res = await request(server, 'PATCH', '/api/compliance/items/srv-001/metadata', {
|
||||
resolution_date: 'not-a-date',
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(res.body.error).toContain('Invalid resolution_date format');
|
||||
});
|
||||
|
||||
it('returns 400 when remediation plan exceeds 2000 characters', async () => {
|
||||
const longPlan = 'x'.repeat(2001);
|
||||
const res = await request(server, 'PATCH', '/api/compliance/items/srv-001/metadata', {
|
||||
remediation_plan: longPlan,
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(400);
|
||||
expect(res.body.error).toContain('2000 characters');
|
||||
});
|
||||
|
||||
it('returns 404 when hostname not found', async () => {
|
||||
mockPool.query.mockResolvedValueOnce({ rows: [], rowCount: 0 });
|
||||
|
||||
const res = await request(server, 'PATCH', '/api/compliance/items/nonexistent-host/metadata', {
|
||||
resolution_date: '2026-06-15',
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(404);
|
||||
expect(res.body.error).toBe('Device not found');
|
||||
});
|
||||
});
|
||||
|
||||
// --- 18.2: GET /vcl/stats with no data ---
|
||||
|
||||
describe('GET /vcl/stats with no data', () => {
|
||||
it('returns zero/empty response when no compliance data exists', async () => {
|
||||
// First query: active items
|
||||
mockPool.query.mockResolvedValueOnce({ rows: [] });
|
||||
// Second query: latest upload
|
||||
mockPool.query.mockResolvedValueOnce({ rows: [] });
|
||||
|
||||
const res = await request(server, 'GET', '/api/compliance/vcl/stats');
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.body.stats).toBeDefined();
|
||||
expect(res.body.stats.total).toBe(0);
|
||||
expect(res.body.stats.in_scope).toBe(0);
|
||||
expect(res.body.stats.compliant).toBe(0);
|
||||
expect(res.body.stats.non_compliant).toBe(0);
|
||||
expect(res.body.stats.compliance_pct).toBe(0);
|
||||
expect(res.body.donut).toBeDefined();
|
||||
expect(res.body.heavy_hitters).toEqual([]);
|
||||
expect(res.body.vertical_breakdown).toEqual([]);
|
||||
});
|
||||
});
|
||||
|
||||
// --- 18.3: Bulk preview with all unmatched hostnames ---
|
||||
|
||||
describe('POST /vcl/bulk-preview — all unmatched', () => {
|
||||
it('returns all rows as unmatched when no hostnames exist in DB', async () => {
|
||||
// Query for existing hostnames returns empty
|
||||
mockPool.query.mockResolvedValueOnce({ rows: [] });
|
||||
|
||||
const res = await request(server, 'POST', '/api/compliance/vcl/bulk-preview', {
|
||||
rows: [
|
||||
{ hostname: 'unknown-1', resolution_date: '2026-06-15' },
|
||||
{ hostname: 'unknown-2', resolution_date: '2026-07-01' },
|
||||
{ hostname: 'unknown-3', resolution_date: '2026-08-01' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.body.matched).toBe(0);
|
||||
expect(res.body.unmatched).toBe(3);
|
||||
expect(res.body.changes).toBe(0);
|
||||
expect(res.body.unmatched_rows).toEqual(['unknown-1', 'unknown-2', 'unknown-3']);
|
||||
});
|
||||
});
|
||||
|
||||
// --- 18.4: Bulk preview with mixed valid/invalid rows ---
|
||||
|
||||
describe('POST /vcl/bulk-preview — mixed valid/invalid', () => {
|
||||
it('correctly classifies valid and invalid rows', async () => {
|
||||
// Query for existing hostnames
|
||||
mockPool.query
|
||||
.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ hostname: 'srv-001' },
|
||||
{ hostname: 'srv-002' },
|
||||
{ hostname: 'srv-003' },
|
||||
],
|
||||
})
|
||||
// Query for current data (DISTINCT ON)
|
||||
.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ hostname: 'srv-001', resolution_date: null, remediation_plan: null },
|
||||
{ hostname: 'srv-003', resolution_date: null, remediation_plan: null },
|
||||
],
|
||||
});
|
||||
|
||||
const res = await request(server, 'POST', '/api/compliance/vcl/bulk-preview', {
|
||||
rows: [
|
||||
{ hostname: 'srv-001', resolution_date: '2026-06-15' }, // valid, matched
|
||||
{ hostname: 'srv-002', resolution_date: 'bad-date' }, // invalid date, matched
|
||||
{ hostname: 'srv-003', resolution_date: '2026-07-01' }, // valid, matched
|
||||
{ hostname: 'unknown-1', resolution_date: '2026-08-01' }, // unmatched
|
||||
],
|
||||
});
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.body.matched).toBe(3);
|
||||
expect(res.body.unmatched).toBe(1);
|
||||
expect(res.body.invalid).toBe(1);
|
||||
expect(res.body.invalid_rows[0].hostname).toBe('srv-002');
|
||||
expect(res.body.invalid_rows[0].errors[0]).toContain('invalid date');
|
||||
expect(res.body.unmatched_rows).toEqual(['unknown-1']);
|
||||
});
|
||||
});
|
||||
|
||||
// --- 18.5: Integration test for full bulk flow ---
|
||||
|
||||
describe('Integration: full bulk upload flow (preview → commit)', () => {
|
||||
it('preview shows changes, commit updates DB', async () => {
|
||||
// --- Preview phase ---
|
||||
// Query for existing hostnames
|
||||
mockPool.query
|
||||
.mockResolvedValueOnce({
|
||||
rows: [{ hostname: 'srv-001' }, { hostname: 'srv-002' }],
|
||||
})
|
||||
// Query for current data
|
||||
.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ hostname: 'srv-001', resolution_date: null, remediation_plan: null },
|
||||
{ hostname: 'srv-002', resolution_date: '2026-01-01', remediation_plan: 'Old plan' },
|
||||
],
|
||||
});
|
||||
|
||||
const previewRes = await request(server, 'POST', '/api/compliance/vcl/bulk-preview', {
|
||||
rows: [
|
||||
{ hostname: 'srv-001', resolution_date: '2026-06-15', remediation_plan: 'New plan' },
|
||||
{ hostname: 'srv-002', resolution_date: '2026-01-01', remediation_plan: 'Old plan' }, // unchanged
|
||||
],
|
||||
});
|
||||
|
||||
expect(previewRes.statusCode).toBe(200);
|
||||
expect(previewRes.body.matched).toBe(2);
|
||||
expect(previewRes.body.changes).toBe(1); // only srv-001 changed
|
||||
|
||||
// --- Commit phase ---
|
||||
const mockClient = {
|
||||
query: jest.fn(),
|
||||
release: jest.fn(),
|
||||
};
|
||||
mockClient.query
|
||||
.mockResolvedValueOnce({}) // BEGIN
|
||||
.mockResolvedValueOnce({ rowCount: 1 }) // UPDATE srv-001
|
||||
.mockResolvedValueOnce({}); // COMMIT
|
||||
|
||||
mockPool.connect.mockResolvedValueOnce(mockClient);
|
||||
|
||||
const commitRes = await request(server, 'POST', '/api/compliance/vcl/bulk-commit', {
|
||||
changes: [
|
||||
{ hostname: 'srv-001', resolution_date: '2026-06-15', remediation_plan: 'New plan' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(commitRes.statusCode).toBe(200);
|
||||
expect(commitRes.body.committed).toBe(1);
|
||||
expect(mockClient.query).toHaveBeenCalledWith('BEGIN');
|
||||
expect(mockClient.query).toHaveBeenCalledWith('COMMIT');
|
||||
});
|
||||
});
|
||||
|
||||
// --- 18.6: Trend endpoint with < 2 months (no forecast) ---
|
||||
|
||||
describe('GET /vcl/trend — fewer than 2 months', () => {
|
||||
it('returns data without forecast when < 2 months exist', async () => {
|
||||
mockPool.query.mockResolvedValueOnce({
|
||||
rows: [
|
||||
{ snapshot_month: '2026-01', compliant_count: 900, compliance_pct: '82.0' },
|
||||
],
|
||||
});
|
||||
|
||||
const res = await request(server, 'GET', '/api/compliance/vcl/trend');
|
||||
|
||||
expect(res.statusCode).toBe(200);
|
||||
expect(res.body.months).toHaveLength(1);
|
||||
expect(res.body.months[0].month).toBe('2026-01');
|
||||
expect(res.body.months[0].forecast_pct).toBeNull();
|
||||
expect(res.body.months[0].target_pct).toBe(95);
|
||||
});
|
||||
});
|
||||
220
backend/helpers/vclHelpers.js
Normal file
220
backend/helpers/vclHelpers.js
Normal file
@@ -0,0 +1,220 @@
|
||||
// Pure helper functions for VCL Compliance Reporting
|
||||
// No database dependencies — all functions are stateless and testable in isolation.
|
||||
|
||||
/**
|
||||
* Truncates text to maxLen characters with an ellipsis.
|
||||
* Returns '' for null/undefined input.
|
||||
*/
|
||||
function truncateText(text, maxLen = 80) {
|
||||
if (text == null) return '';
|
||||
if (text.length <= maxLen) return text;
|
||||
return text.slice(0, maxLen) + '\u2026';
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates that a remediation plan does not exceed 2000 characters.
|
||||
* Null/undefined/empty values are considered valid (no plan documented).
|
||||
*/
|
||||
function validateRemediationPlan(text) {
|
||||
if (text == null || text === '') return { valid: true };
|
||||
if (text.length > 2000) return { valid: false, error: 'Remediation plan exceeds 2000 characters' };
|
||||
return { valid: true };
|
||||
}
|
||||
|
||||
/**
|
||||
* Returns true only for strings parseable as real calendar dates.
|
||||
* Rejects null, undefined, empty string, and invalid dates like "2026-02-30".
|
||||
*/
|
||||
function isValidDateString(str) {
|
||||
if (str == null || str === '') return false;
|
||||
if (typeof str !== 'string') return false;
|
||||
|
||||
// Expect YYYY-MM-DD format
|
||||
const match = str.match(/^(\d{4})-(\d{2})-(\d{2})$/);
|
||||
if (!match) return false;
|
||||
|
||||
const year = parseInt(match[1], 10);
|
||||
const month = parseInt(match[2], 10);
|
||||
const day = parseInt(match[3], 10);
|
||||
|
||||
// Month must be 1-12
|
||||
if (month < 1 || month > 12) return false;
|
||||
|
||||
// Create date and verify components match (catches invalid days like Feb 30)
|
||||
const date = new Date(year, month - 1, day);
|
||||
return (
|
||||
date.getFullYear() === year &&
|
||||
date.getMonth() === month - 1 &&
|
||||
date.getDate() === day
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Formats a decimal as a whole-number percentage string.
|
||||
* Returns '0%' for null, undefined, or NaN input.
|
||||
*/
|
||||
function formatPct(decimal) {
|
||||
if (decimal == null || isNaN(decimal)) return '0%';
|
||||
return Math.round(decimal * 100) + '%';
|
||||
}
|
||||
|
||||
/**
|
||||
* Computes VCL summary statistics from an array of device objects.
|
||||
* Each item should have at least { is_compliant: boolean, in_scope: boolean }.
|
||||
*/
|
||||
function computeVCLStats(items, targetPct) {
|
||||
const total = items.length;
|
||||
const in_scope = items.filter(item => item.in_scope).length;
|
||||
const compliant = items.filter(item => item.is_compliant).length;
|
||||
const non_compliant = in_scope - compliant;
|
||||
const remediations_required = non_compliant;
|
||||
const compliance_pct = in_scope > 0 ? Math.round((compliant / in_scope) * 100) : 0;
|
||||
|
||||
return {
|
||||
total,
|
||||
in_scope,
|
||||
compliant,
|
||||
non_compliant,
|
||||
remediations_required,
|
||||
compliance_pct,
|
||||
target_pct: targetPct,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Partitions non-compliant items into "blocked" (no resolution_date) and
|
||||
* "in_progress" (resolution_date set). Returns counts and percentages.
|
||||
*/
|
||||
function categorizeNonCompliant(items) {
|
||||
const total = items.length;
|
||||
const blocked = items.filter(item => item.resolution_date == null);
|
||||
const in_progress = items.filter(item => item.resolution_date != null);
|
||||
|
||||
return {
|
||||
blocked: {
|
||||
count: blocked.length,
|
||||
pct: total > 0 ? Math.round((blocked.length / total) * 100) : 0,
|
||||
},
|
||||
in_progress: {
|
||||
count: in_progress.length,
|
||||
pct: total > 0 ? Math.round((in_progress.length / total) * 100) : 0,
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Sorts verticals by non_compliant count in descending order.
|
||||
* Returns a new sorted array (does not mutate input).
|
||||
*/
|
||||
function rankHeavyHitters(verticalData) {
|
||||
return [...verticalData].sort((a, b) => b.non_compliant - a.non_compliant);
|
||||
}
|
||||
|
||||
/**
|
||||
* Buckets non-compliant items by resolution_date month (YYYY-MM).
|
||||
* Items with null resolution_date are skipped.
|
||||
* Returns an object like { '2026-05': 3, '2026-06': 7 }.
|
||||
*/
|
||||
function computeForecastBurndown(items) {
|
||||
const buckets = {};
|
||||
for (const item of items) {
|
||||
if (item.resolution_date == null) continue;
|
||||
const dateStr = typeof item.resolution_date === 'string'
|
||||
? item.resolution_date
|
||||
: item.resolution_date.toISOString().slice(0, 10);
|
||||
const month = dateStr.slice(0, 7); // YYYY-MM
|
||||
buckets[month] = (buckets[month] || 0) + 1;
|
||||
}
|
||||
return buckets;
|
||||
}
|
||||
|
||||
/**
|
||||
* Matches uploaded rows to existing hostnames.
|
||||
* Returns { matched: [...], unmatched: [...] }.
|
||||
*/
|
||||
function matchByHostname(uploadedRows, existingHostnames) {
|
||||
const matched = [];
|
||||
const unmatched = [];
|
||||
for (const row of uploadedRows) {
|
||||
if (existingHostnames.has(row.hostname)) {
|
||||
matched.push(row);
|
||||
} else {
|
||||
unmatched.push(row);
|
||||
}
|
||||
}
|
||||
return { matched, unmatched };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compares uploaded row values against current DB values.
|
||||
* currentData is a Map of hostname -> { resolution_date, remediation_plan, notes }.
|
||||
* Returns array of { hostname, status: 'changed'|'unchanged', fields: { fieldName: { old, new } } }.
|
||||
*/
|
||||
function computeBulkDiff(matchedRows, currentData) {
|
||||
const results = [];
|
||||
const COMPARE_FIELDS = ['resolution_date', 'remediation_plan', 'notes'];
|
||||
|
||||
for (const row of matchedRows) {
|
||||
const current = currentData.get(row.hostname) || {};
|
||||
const fields = {};
|
||||
let hasChange = false;
|
||||
|
||||
for (const field of COMPARE_FIELDS) {
|
||||
if (field in row) {
|
||||
const oldVal = current[field] != null ? current[field] : null;
|
||||
const newVal = row[field] != null ? row[field] : null;
|
||||
if (oldVal !== newVal) {
|
||||
fields[field] = { old: oldVal, new: newVal };
|
||||
hasChange = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
results.push({
|
||||
hostname: row.hostname,
|
||||
status: hasChange ? 'changed' : 'unchanged',
|
||||
fields,
|
||||
});
|
||||
}
|
||||
|
||||
return results;
|
||||
}
|
||||
|
||||
/**
|
||||
* Maps column header strings to known field names (case-insensitive).
|
||||
* Returns a mapping object like { hostname: 0, resolution_date: 3 } where values are column indices.
|
||||
*/
|
||||
function mapColumnHeaders(headers) {
|
||||
const mapping = {};
|
||||
const KNOWN_MAPPINGS = {
|
||||
hostname: 'hostname',
|
||||
'resolution date': 'resolution_date',
|
||||
resolution_date: 'resolution_date',
|
||||
'remediation plan': 'remediation_plan',
|
||||
remediation_plan: 'remediation_plan',
|
||||
notes: 'notes',
|
||||
};
|
||||
|
||||
for (let i = 0; i < headers.length; i++) {
|
||||
const normalized = headers[i].trim().toLowerCase();
|
||||
if (KNOWN_MAPPINGS[normalized]) {
|
||||
mapping[KNOWN_MAPPINGS[normalized]] = i;
|
||||
}
|
||||
}
|
||||
|
||||
return mapping;
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
truncateText,
|
||||
validateRemediationPlan,
|
||||
isValidDateString,
|
||||
formatPct,
|
||||
computeVCLStats,
|
||||
categorizeNonCompliant,
|
||||
rankHeavyHitters,
|
||||
computeForecastBurndown,
|
||||
matchByHostname,
|
||||
computeBulkDiff,
|
||||
mapColumnHeaders,
|
||||
};
|
||||
38
backend/migrations/add_vcl_reporting_columns.js
Normal file
38
backend/migrations/add_vcl_reporting_columns.js
Normal file
@@ -0,0 +1,38 @@
|
||||
// Migration: Add VCL reporting columns to compliance_items and create compliance_snapshots table
|
||||
const pool = require('../db');
|
||||
|
||||
async function run() {
|
||||
console.log('Starting VCL reporting migration...');
|
||||
try {
|
||||
await pool.query(`ALTER TABLE compliance_items ADD COLUMN IF NOT EXISTS resolution_date DATE DEFAULT NULL`);
|
||||
console.log('✓ resolution_date column added (or already exists)');
|
||||
|
||||
await pool.query(`ALTER TABLE compliance_items ADD COLUMN IF NOT EXISTS remediation_plan TEXT DEFAULT NULL`);
|
||||
console.log('✓ remediation_plan column added (or already exists)');
|
||||
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS compliance_snapshots (
|
||||
id SERIAL PRIMARY KEY,
|
||||
snapshot_month TEXT NOT NULL,
|
||||
vertical TEXT NOT NULL,
|
||||
total_devices INTEGER NOT NULL DEFAULT 0,
|
||||
compliant INTEGER NOT NULL DEFAULT 0,
|
||||
non_compliant INTEGER NOT NULL DEFAULT 0,
|
||||
compliance_pct NUMERIC(5,2) DEFAULT 0,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
UNIQUE(snapshot_month, vertical)
|
||||
)
|
||||
`);
|
||||
console.log('✓ compliance_snapshots table created (or already exists)');
|
||||
|
||||
await pool.query(`CREATE INDEX IF NOT EXISTS idx_compliance_snapshots_month ON compliance_snapshots(snapshot_month)`);
|
||||
console.log('✓ idx_compliance_snapshots_month index created (or already exists)');
|
||||
} catch (err) {
|
||||
console.error('Migration error:', err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
console.log('Migration complete.');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
run();
|
||||
@@ -16,6 +16,7 @@ const MIGRATIONS_DIR = __dirname;
|
||||
const POSTGRES_MIGRATIONS = [
|
||||
'add_decom_workflow_type.js',
|
||||
'add_fp_submissions_dismissed.js',
|
||||
'add_vcl_reporting_columns.js',
|
||||
];
|
||||
|
||||
async function runAll() {
|
||||
|
||||
@@ -9,6 +9,7 @@ const { spawn } = require('child_process');
|
||||
const pool = require('../db');
|
||||
const { requireAuth, requireGroup } = require('../middleware/auth');
|
||||
const { loadConfig, compareSchemaToDrift, reconcileConfig } = require('../helpers/driftChecker');
|
||||
const { isValidDateString, validateRemediationPlan, computeVCLStats, categorizeNonCompliant, rankHeavyHitters, computeForecastBurndown, matchByHostname, computeBulkDiff, mapColumnHeaders } = require('../helpers/vclHelpers');
|
||||
const logAudit = require('../helpers/auditLog');
|
||||
|
||||
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_compliance_xlsx.py');
|
||||
@@ -147,6 +148,39 @@ async function persistUpload({ items, summary, reportDate, filename, userId }) {
|
||||
);
|
||||
|
||||
await client.query('COMMIT');
|
||||
|
||||
// Task 7: Create/update compliance_snapshots for the current month
|
||||
try {
|
||||
const currentMonth = new Date().toISOString().slice(0, 7); // YYYY-MM
|
||||
|
||||
// Compute per-vertical compliance percentages from current state
|
||||
const { rows: verticalStats } = await pool.query(
|
||||
`SELECT team AS vertical,
|
||||
COUNT(DISTINCT hostname)::int AS total_devices,
|
||||
COUNT(DISTINCT CASE WHEN status = 'resolved' THEN hostname END)::int AS compliant,
|
||||
COUNT(DISTINCT CASE WHEN status = 'active' THEN hostname END)::int AS non_compliant
|
||||
FROM compliance_items
|
||||
WHERE team IS NOT NULL
|
||||
GROUP BY team`
|
||||
);
|
||||
|
||||
for (const vs of verticalStats) {
|
||||
const total = vs.total_devices;
|
||||
const compPct = total > 0 ? Math.round((vs.compliant / total) * 100 * 100) / 100 : 0;
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO compliance_snapshots (snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (snapshot_month, vertical)
|
||||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||||
[currentMonth, vs.vertical, total, vs.compliant, vs.non_compliant, compPct]
|
||||
);
|
||||
}
|
||||
} catch (snapshotErr) {
|
||||
// Snapshot creation is non-critical — log but don't fail the upload
|
||||
console.error('[Compliance] Snapshot creation error:', snapshotErr.message);
|
||||
}
|
||||
|
||||
return { uploadId, newCount, recurringCount, resolvedCount };
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
@@ -669,6 +703,425 @@ function createComplianceRouter(upload) {
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// PATCH /items/:hostname/metadata — Update resolution_date / remediation_plan
|
||||
// -----------------------------------------------------------------------
|
||||
router.patch('/items/:hostname/metadata', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const hostname = req.params.hostname;
|
||||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||||
|
||||
const { resolution_date, remediation_plan } = req.body;
|
||||
|
||||
// Validate resolution_date: must be a valid ISO date string or null
|
||||
if (resolution_date !== undefined && resolution_date !== null) {
|
||||
if (!isValidDateString(resolution_date)) {
|
||||
return res.status(400).json({ error: 'Invalid resolution_date format' });
|
||||
}
|
||||
}
|
||||
|
||||
// Validate remediation_plan: must be <= 2000 chars or null
|
||||
if (remediation_plan !== undefined && remediation_plan !== null) {
|
||||
const planValidation = validateRemediationPlan(remediation_plan);
|
||||
if (!planValidation.valid) {
|
||||
return res.status(400).json({ error: planValidation.error });
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
// Build dynamic SET clause for provided fields only
|
||||
const setClauses = [];
|
||||
const values = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (resolution_date !== undefined) {
|
||||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||||
values.push(resolution_date);
|
||||
}
|
||||
if (remediation_plan !== undefined) {
|
||||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||||
values.push(remediation_plan);
|
||||
}
|
||||
|
||||
if (setClauses.length === 0) {
|
||||
return res.status(400).json({ error: 'No fields to update' });
|
||||
}
|
||||
|
||||
values.push(hostname);
|
||||
const result = await pool.query(
|
||||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||||
values
|
||||
);
|
||||
|
||||
if (result.rowCount === 0) {
|
||||
return res.status(404).json({ error: 'Device not found' });
|
||||
}
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id,
|
||||
username: req.user.username,
|
||||
action: 'compliance_metadata_update',
|
||||
entityType: 'compliance_item',
|
||||
entityId: hostname,
|
||||
details: { resolution_date, remediation_plan },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.json({ updated: result.rowCount });
|
||||
} catch (err) {
|
||||
console.error('[Compliance] PATCH /items/:hostname/metadata error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to update device metadata' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// GET /vcl/stats — VCL executive summary statistics
|
||||
// -----------------------------------------------------------------------
|
||||
const VCL_TARGET_PCT = parseInt(process.env.VCL_TARGET_PCT, 10) || 95;
|
||||
|
||||
router.get('/vcl/stats', async (req, res) => {
|
||||
try {
|
||||
// Fetch all active compliance items
|
||||
const { rows: items } = await pool.query(
|
||||
`SELECT hostname, team, status, resolution_date, remediation_plan,
|
||||
CASE WHEN status = 'resolved' THEN true ELSE false END AS is_compliant,
|
||||
true AS in_scope
|
||||
FROM compliance_items WHERE status = 'active'`
|
||||
);
|
||||
|
||||
// For stats computation, all active items are non-compliant (they are findings)
|
||||
// We need total in-scope devices (active + resolved from latest upload)
|
||||
const { rows: latestUploadRows } = await pool.query(
|
||||
`SELECT id FROM compliance_uploads ORDER BY id DESC LIMIT 1`
|
||||
);
|
||||
|
||||
let allDeviceItems = [];
|
||||
if (latestUploadRows.length > 0) {
|
||||
const { rows: allItems } = await pool.query(
|
||||
`SELECT hostname, team, status, resolution_date, remediation_plan,
|
||||
CASE WHEN status = 'resolved' THEN true ELSE false END AS is_compliant,
|
||||
true AS in_scope
|
||||
FROM compliance_items`
|
||||
);
|
||||
// Deduplicate by hostname — a device is compliant if it has no active findings
|
||||
const deviceMap = new Map();
|
||||
for (const item of allItems) {
|
||||
const existing = deviceMap.get(item.hostname);
|
||||
if (!existing) {
|
||||
deviceMap.set(item.hostname, { ...item, is_compliant: item.status !== 'active', in_scope: true });
|
||||
} else if (item.status === 'active') {
|
||||
existing.is_compliant = false;
|
||||
}
|
||||
}
|
||||
allDeviceItems = Array.from(deviceMap.values());
|
||||
}
|
||||
|
||||
const stats = computeVCLStats(allDeviceItems, VCL_TARGET_PCT);
|
||||
|
||||
// Donut: categorize non-compliant items by resolution_date presence
|
||||
const nonCompliantItems = items.filter(i => i.status === 'active');
|
||||
const donut = categorizeNonCompliant(nonCompliantItems);
|
||||
|
||||
// Heavy hitters: group by team, count non-compliant per team
|
||||
const teamCounts = {};
|
||||
for (const item of nonCompliantItems) {
|
||||
const team = item.team || 'Unknown';
|
||||
if (!teamCounts[team]) {
|
||||
teamCounts[team] = { vertical: team, team: team, non_compliant: 0, compliance_date: null, notes: '' };
|
||||
}
|
||||
teamCounts[team].non_compliant++;
|
||||
// Use the latest resolution_date as the team's compliance_date
|
||||
if (item.resolution_date && (!teamCounts[team].compliance_date || item.resolution_date > teamCounts[team].compliance_date)) {
|
||||
teamCounts[team].compliance_date = item.resolution_date;
|
||||
}
|
||||
}
|
||||
const heavy_hitters = rankHeavyHitters(Object.values(teamCounts));
|
||||
|
||||
// Vertical breakdown with burndown
|
||||
const verticalBreakdown = [];
|
||||
for (const team of Object.keys(teamCounts)) {
|
||||
const teamItems = nonCompliantItems.filter(i => (i.team || 'Unknown') === team);
|
||||
const teamAllDevices = allDeviceItems.filter(i => (i.team || 'Unknown') === team);
|
||||
const teamTotal = teamAllDevices.length;
|
||||
const teamCompliant = teamAllDevices.filter(i => i.is_compliant).length;
|
||||
const compliance_pct = teamTotal > 0 ? Math.round((teamCompliant / teamTotal) * 100) : 0;
|
||||
|
||||
const actual_burndown = computeForecastBurndown(teamItems.filter(i => i.resolution_date));
|
||||
const forecast_burndown = computeForecastBurndown(teamItems.filter(i => i.resolution_date));
|
||||
const blockers = teamItems.filter(i => !i.resolution_date).length;
|
||||
|
||||
verticalBreakdown.push({
|
||||
vertical: team,
|
||||
compliance_pct,
|
||||
team: team,
|
||||
non_compliant: teamItems.length,
|
||||
actual_burndown,
|
||||
forecast_burndown,
|
||||
blockers,
|
||||
risk_acceptances: 0,
|
||||
notes: '',
|
||||
});
|
||||
}
|
||||
|
||||
res.json({ stats, donut, heavy_hitters, vertical_breakdown: verticalBreakdown });
|
||||
} catch (err) {
|
||||
console.error('[Compliance] GET /vcl/stats error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// GET /vcl/trend — Monthly compliance trend with forecast
|
||||
// -----------------------------------------------------------------------
|
||||
router.get('/vcl/trend', async (req, res) => {
|
||||
try {
|
||||
const { rows: snapshots } = await pool.query(
|
||||
`SELECT snapshot_month, SUM(compliant)::int AS compliant_count,
|
||||
CASE WHEN SUM(total_devices) > 0
|
||||
THEN ROUND((SUM(compliant)::numeric / SUM(total_devices)::numeric) * 100, 1)
|
||||
ELSE 0 END AS compliance_pct
|
||||
FROM compliance_snapshots
|
||||
GROUP BY snapshot_month
|
||||
ORDER BY snapshot_month ASC`
|
||||
);
|
||||
|
||||
// Build months array with actuals
|
||||
const months = snapshots.map(s => ({
|
||||
month: s.snapshot_month,
|
||||
compliant_count: s.compliant_count,
|
||||
compliance_pct: parseFloat(s.compliance_pct),
|
||||
forecast_pct: null,
|
||||
target_pct: VCL_TARGET_PCT,
|
||||
}));
|
||||
|
||||
// Compute forecast using linear regression if we have 3+ months
|
||||
if (months.length >= 3) {
|
||||
const n = months.length;
|
||||
// Use last data points for regression
|
||||
let sumX = 0, sumY = 0, sumXY = 0, sumX2 = 0;
|
||||
for (let i = 0; i < n; i++) {
|
||||
sumX += i;
|
||||
sumY += months[i].compliance_pct;
|
||||
sumXY += i * months[i].compliance_pct;
|
||||
sumX2 += i * i;
|
||||
}
|
||||
const slope = (n * sumXY - sumX * sumY) / (n * sumX2 - sumX * sumX);
|
||||
const intercept = (sumY - slope * sumX) / n;
|
||||
|
||||
// Project forward 3 months
|
||||
for (let i = 0; i < 3; i++) {
|
||||
const futureIdx = n + i;
|
||||
const forecastPct = Math.min(100, Math.max(0, Math.round((slope * futureIdx + intercept) * 10) / 10));
|
||||
|
||||
// Compute the future month string
|
||||
const lastMonth = months[months.length - 1].month;
|
||||
const [year, mon] = lastMonth.split('-').map(Number);
|
||||
const futureDate = new Date(year, mon - 1 + i + 1, 1);
|
||||
const futureMonth = `${futureDate.getFullYear()}-${String(futureDate.getMonth() + 1).padStart(2, '0')}`;
|
||||
|
||||
months.push({
|
||||
month: futureMonth,
|
||||
compliant_count: null,
|
||||
compliance_pct: null,
|
||||
forecast_pct: forecastPct,
|
||||
target_pct: VCL_TARGET_PCT,
|
||||
});
|
||||
}
|
||||
|
||||
// Also add forecast_pct to the last actual month as the starting point
|
||||
if (months.length > 0 && n > 0) {
|
||||
months[n - 1].forecast_pct = months[n - 1].compliance_pct;
|
||||
}
|
||||
}
|
||||
|
||||
res.json({ months });
|
||||
} catch (err) {
|
||||
console.error('[Compliance] GET /vcl/trend error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// POST /vcl/bulk-preview — Bulk upload diff preview
|
||||
// -----------------------------------------------------------------------
|
||||
router.post('/vcl/bulk-preview', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { rows, headers } = req.body;
|
||||
|
||||
// Validate: require rows array
|
||||
if (!rows || !Array.isArray(rows)) {
|
||||
return res.status(400).json({ error: 'rows array is required' });
|
||||
}
|
||||
|
||||
// Enforce 2000 row limit
|
||||
if (rows.length === 0) {
|
||||
return res.status(400).json({ error: 'File contains no data rows' });
|
||||
}
|
||||
if (rows.length > 2000) {
|
||||
return res.status(400).json({ error: 'File exceeds maximum of 2000 rows' });
|
||||
}
|
||||
|
||||
// Map column headers if provided
|
||||
let columnMapping = {};
|
||||
if (headers && Array.isArray(headers)) {
|
||||
columnMapping = mapColumnHeaders(headers);
|
||||
}
|
||||
|
||||
// Require hostname field
|
||||
const hasHostname = rows.every(r => r.hostname != null && r.hostname !== '');
|
||||
if (!hasHostname) {
|
||||
return res.status(400).json({ error: 'File must contain a Hostname column' });
|
||||
}
|
||||
|
||||
// Check for updatable fields (resolution_date, remediation_plan, or notes)
|
||||
const sampleRow = rows[0] || {};
|
||||
const updatableFields = ['resolution_date', 'remediation_plan', 'notes'];
|
||||
const hasUpdatableFields = updatableFields.some(f => f in sampleRow);
|
||||
if (!hasUpdatableFields && headers) {
|
||||
// Check via column mapping
|
||||
const mappedFields = Object.keys(columnMapping).filter(k => k !== 'hostname');
|
||||
if (mappedFields.length === 0) {
|
||||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||||
}
|
||||
} else if (!hasUpdatableFields && !headers) {
|
||||
return res.status(400).json({ error: 'No updatable fields found (need Resolution Date, Remediation Plan, or Notes)' });
|
||||
}
|
||||
|
||||
try {
|
||||
// Get existing hostnames from DB
|
||||
const { rows: existingRows } = await pool.query(
|
||||
`SELECT DISTINCT hostname FROM compliance_items WHERE status = 'active'`
|
||||
);
|
||||
const existingHostnames = new Set(existingRows.map(r => r.hostname));
|
||||
|
||||
// Match by hostname
|
||||
const { matched, unmatched } = matchByHostname(rows, existingHostnames);
|
||||
|
||||
// Validate fields on matched rows
|
||||
const validRows = [];
|
||||
const invalidRows = [];
|
||||
|
||||
for (const row of matched) {
|
||||
const errors = [];
|
||||
|
||||
if (row.resolution_date !== undefined && row.resolution_date !== null && row.resolution_date !== '') {
|
||||
if (!isValidDateString(row.resolution_date)) {
|
||||
errors.push('resolution_date: invalid date format');
|
||||
}
|
||||
}
|
||||
|
||||
if (row.remediation_plan !== undefined && row.remediation_plan !== null) {
|
||||
const planCheck = validateRemediationPlan(row.remediation_plan);
|
||||
if (!planCheck.valid) {
|
||||
errors.push('remediation_plan: ' + planCheck.error);
|
||||
}
|
||||
}
|
||||
|
||||
if (errors.length > 0) {
|
||||
invalidRows.push({ hostname: row.hostname, errors });
|
||||
} else {
|
||||
validRows.push(row);
|
||||
}
|
||||
}
|
||||
|
||||
// Get current data for diff computation
|
||||
const { rows: currentRows } = await pool.query(
|
||||
`SELECT DISTINCT ON (hostname) hostname, resolution_date, remediation_plan
|
||||
FROM compliance_items WHERE status = 'active' AND hostname = ANY($1)
|
||||
ORDER BY hostname, id DESC`,
|
||||
[validRows.map(r => r.hostname)]
|
||||
);
|
||||
const currentData = new Map();
|
||||
for (const row of currentRows) {
|
||||
currentData.set(row.hostname, {
|
||||
resolution_date: row.resolution_date ? row.resolution_date.toISOString?.().slice(0, 10) || String(row.resolution_date).slice(0, 10) : null,
|
||||
remediation_plan: row.remediation_plan || null,
|
||||
notes: null,
|
||||
});
|
||||
}
|
||||
|
||||
// Compute diff
|
||||
const diffResults = computeBulkDiff(validRows, currentData);
|
||||
const changedRows = diffResults.filter(r => r.status === 'changed');
|
||||
|
||||
res.json({
|
||||
matched: matched.length,
|
||||
unmatched: unmatched.length,
|
||||
changes: changedRows.length,
|
||||
invalid: invalidRows.length,
|
||||
details: diffResults,
|
||||
unmatched_rows: unmatched.map(r => r.hostname),
|
||||
invalid_rows: invalidRows,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Compliance] POST /vcl/bulk-preview error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to process bulk preview' });
|
||||
}
|
||||
});
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// POST /vcl/bulk-commit — Commit validated bulk changes
|
||||
// -----------------------------------------------------------------------
|
||||
router.post('/vcl/bulk-commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { changes } = req.body;
|
||||
|
||||
if (!changes || !Array.isArray(changes) || changes.length === 0) {
|
||||
return res.status(400).json({ error: 'changes array is required' });
|
||||
}
|
||||
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
let committedCount = 0;
|
||||
for (const change of changes) {
|
||||
const setClauses = [];
|
||||
const values = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (change.resolution_date !== undefined) {
|
||||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||||
values.push(change.resolution_date);
|
||||
}
|
||||
if (change.remediation_plan !== undefined) {
|
||||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||||
values.push(change.remediation_plan);
|
||||
}
|
||||
if (change.notes !== undefined) {
|
||||
// Notes are stored separately in compliance_notes, but we can update a field if it exists
|
||||
// For now, skip notes in the direct update
|
||||
}
|
||||
|
||||
if (setClauses.length === 0) continue;
|
||||
|
||||
values.push(change.hostname);
|
||||
const result = await client.query(
|
||||
`UPDATE compliance_items SET ${setClauses.join(', ')} WHERE hostname = $${paramIdx} AND status = 'active'`,
|
||||
values
|
||||
);
|
||||
if (result.rowCount > 0) committedCount++;
|
||||
}
|
||||
|
||||
await client.query('COMMIT');
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id,
|
||||
username: req.user.username,
|
||||
action: 'compliance_bulk_update',
|
||||
entityType: 'compliance_items',
|
||||
entityId: null,
|
||||
details: { rows_updated: committedCount, total_changes: changes.length },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.json({ committed: committedCount });
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
console.error('[Compliance] POST /vcl/bulk-commit error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to commit changes' });
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user