Add Granite supplemental workbook ingest with reconciliation
Separate upload path for the NTS_AEO_(supp only) workbook that tracks Granite CMDB hygiene findings (Missing OS, Missing App ID, Missing Device Function, Retired App ID) with weekly history and team scoping. Backend: - Python parser for supplemental xlsx (4 recognized sheets) - supplemental_uploads and supplemental_items tables with per-sheet scoped resolution - supplemental_notes table for per-device notes - resolution_date and remediation_plan on supplemental items - Full route module: preview, commit, summary, items, trends, uploads, rollback, device detail, metadata PATCH, notes CRUD Frontend: - SupplementalUploadModal with per-sheet diff preview - GraniteHygieneSection on CompliancePage with cards, device drill-down, pagination - SupplementalDetailPanel slide-out with findings, metadata editing, and notes - Upload Supplemental button in CompliancePage header
This commit is contained in:
Jordan Ramos
3
backend/.gitignore
vendored
3
backend/.gitignore
vendored
@@ -6,3 +6,6 @@ backend/add_vendor_to_documents.js
|
||||
|
||||
# TLS certificates (self-signed or CA-issued)
|
||||
certs/
|
||||
|
||||
# Database dumps taken before one-off data repairs
|
||||
backups/
|
||||
|
||||
@@ -0,0 +1,398 @@
|
||||
/**
|
||||
* Property Tests: Compliance Cross-Vertical Resolve Sweep
|
||||
*
|
||||
* Spec: .kiro/specs/compliance-cross-vertical-resolve-sweep/ (bugfix)
|
||||
*
|
||||
* BUG CONDITION (from bugfix.md):
|
||||
* persistUpload() built its active set with `WHERE status = 'active'` and no
|
||||
* vertical predicate, then resolved every row absent from the incoming
|
||||
* spreadsheet. A compliance spreadsheet covers exactly one vertical, so this
|
||||
* marked every OTHER vertical's open findings as resolved.
|
||||
*
|
||||
* Real-world instance: upload 91 (NTS_AEO_2026_07_20.xlsx) resolved 123,444
|
||||
* items, of which only 899 were NTS_AEO's own.
|
||||
*
|
||||
* Properties under test:
|
||||
* 1. Sweep isolation — status changes only within the upload's vertical
|
||||
* 2. Resolved-count accuracy — resolved_count counts only own vertical
|
||||
* 3. Ownership transfer preserves history
|
||||
* 4. Single-vertical equivalence — unchanged vs pre-fix for one vertical
|
||||
*
|
||||
* **Validates: Requirements 2.1, 2.2, 2.3, 2.4, 2.5, 2.6, 2.7, 4.1, 4.2, 4.4**
|
||||
*/
|
||||
|
||||
const fc = require('fast-check');
|
||||
|
||||
jest.mock('../middleware/auth', () => ({
|
||||
requireTeam: () => (req, res, next) => { req.teamScope = null; next(); },
|
||||
requireAuth: () => (req, res, next) => {
|
||||
req.user = { id: 1, username: 'testuser', group: 'Admin' };
|
||||
next();
|
||||
},
|
||||
requireGroup: () => (req, res, next) => next(),
|
||||
}));
|
||||
|
||||
jest.mock('../helpers/auditLog', () => jest.fn());
|
||||
|
||||
const UPLOAD_ID = 9999;
|
||||
|
||||
let activeRows = [];
|
||||
const recorded = { updates: [], inserts: [], resolves: [] };
|
||||
|
||||
function resetRecorder() {
|
||||
recorded.updates = [];
|
||||
recorded.inserts = [];
|
||||
recorded.resolves = [];
|
||||
}
|
||||
|
||||
// Programmable pg pool. The pool-level query serves the active-set projection;
|
||||
// the client-level query records the mutations persistUpload emits.
|
||||
const mockPool = {
|
||||
query: jest.fn((text) => {
|
||||
if (/FROM compliance_items WHERE status = 'active'/.test(text)) {
|
||||
return Promise.resolve({ rows: activeRows, rowCount: activeRows.length });
|
||||
}
|
||||
return Promise.resolve({ rows: [], rowCount: 0 });
|
||||
}),
|
||||
connect: jest.fn(() => Promise.resolve({
|
||||
query: jest.fn((text, params) => {
|
||||
if (/INSERT INTO compliance_uploads/.test(text)) {
|
||||
return Promise.resolve({ rows: [{ id: UPLOAD_ID }], rowCount: 1 });
|
||||
}
|
||||
if (/UPDATE compliance_items SET status = 'resolved'/.test(text)) {
|
||||
recorded.resolves.push({ uploadId: params[0], id: params[1] });
|
||||
} else if (/UPDATE compliance_items/.test(text) && /SET upload_id/.test(text)) {
|
||||
recorded.updates.push({ text, params });
|
||||
} else if (/INSERT INTO compliance_items/.test(text)) {
|
||||
recorded.inserts.push({ text, params });
|
||||
}
|
||||
return Promise.resolve({ rows: [], rowCount: 0 });
|
||||
}),
|
||||
release: jest.fn(),
|
||||
})),
|
||||
};
|
||||
|
||||
jest.mock('../db', () => mockPool);
|
||||
|
||||
const { persistUpload, deriveVertical } = require('../routes/compliance');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Fixtures and generators
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Canonical fixture modelled on the upload 91 incident: NTS_AEO items plus
|
||||
* open findings belonging to twelve other verticals.
|
||||
*/
|
||||
function fixtureUpload91Shape() {
|
||||
const rows = [];
|
||||
let id = 1;
|
||||
rows.push({ id: id++, hostname: 'AEO-KEEP', metric_id: 'Missing_AppID', seen_count: 3, first_seen_upload_id: 10, vertical: 'NTS_AEO' });
|
||||
rows.push({ id: id++, hostname: 'AEO-GONE', metric_id: 'Missing_AppID', seen_count: 2, first_seen_upload_id: 10, vertical: 'NTS_AEO' });
|
||||
for (const v of ['NTS_AVVOC', 'SBNOE', 'SDIT_EDIS', 'SDIT_CSD', 'NTS_NEO', 'NTS_CPE',
|
||||
'NTS_WTS', 'PRDCT_VSO', 'TSI', 'SR', 'SDIT_CISO', 'SDIT_IT']) {
|
||||
rows.push({ id: id++, hostname: `${v}-HOST`, metric_id: 'Missing_AppID', seen_count: 4, first_seen_upload_id: 20, vertical: v });
|
||||
}
|
||||
return rows;
|
||||
}
|
||||
|
||||
const verticalArb = fc.constantFrom(
|
||||
'NTS_AEO', 'NTS_AVVOC', 'SBNOE', 'SDIT_EDIS', 'SDIT_CSD',
|
||||
'NTS_NEO', 'NTS_CPE', 'TSI', 'SR', 'SDIT_IT',
|
||||
);
|
||||
|
||||
const activeRowArb = fc.record({
|
||||
hostname: fc.stringMatching(/^[A-Z0-9-]{3,12}$/),
|
||||
metric_id: fc.constantFrom('Missing_AppID', 'Missing_OS', '5.5.4i', '2.3.3i'),
|
||||
seen_count: fc.integer({ min: 1, max: 40 }),
|
||||
vertical: verticalArb,
|
||||
});
|
||||
|
||||
/** Deduplicate on hostname|||metric_id, since activeMap is keyed on it. */
|
||||
function dedupe(rows) {
|
||||
const seen = new Set();
|
||||
const out = [];
|
||||
rows.forEach((r, i) => {
|
||||
const k = `${r.hostname}|||${r.metric_id}`;
|
||||
if (seen.has(k)) return;
|
||||
seen.add(k);
|
||||
out.push({ ...r, id: i + 1, first_seen_upload_id: 100 });
|
||||
});
|
||||
return out;
|
||||
}
|
||||
|
||||
function itemFrom(row, overrides = {}) {
|
||||
return {
|
||||
hostname: row.hostname,
|
||||
metric_id: row.metric_id,
|
||||
ip_address: '10.0.0.1',
|
||||
device_type: 'SERVER VM',
|
||||
team: 'STEAM',
|
||||
metric_desc: 'desc',
|
||||
category: 'Asset Data Quality',
|
||||
extra_json: {},
|
||||
...overrides,
|
||||
};
|
||||
}
|
||||
|
||||
async function runUpload(items, vertical) {
|
||||
resetRecorder();
|
||||
return persistUpload({
|
||||
items,
|
||||
summary: { entries: [], overall_scores: {} },
|
||||
reportDate: '2026-07-20',
|
||||
filename: `${vertical}_2026_07_20.xlsx`,
|
||||
userId: 1,
|
||||
vertical,
|
||||
});
|
||||
}
|
||||
|
||||
/** Map recorded resolve calls back to the rows they targeted. */
|
||||
function resolvedRows(rows) {
|
||||
const byId = new Map(rows.map(r => [r.id, r]));
|
||||
return recorded.resolves.map(r => byId.get(r.id)).filter(Boolean);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Property 1 — Sweep isolation
|
||||
// **Validates: Requirements 2.1, 2.2**
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('Property 1 — resolve sweep touches only the upload\'s vertical', () => {
|
||||
it('canonical upload-91 fixture: an NTS_AEO sheet resolves no foreign-vertical rows', async () => {
|
||||
activeRows = fixtureUpload91Shape();
|
||||
// The sheet contains AEO-KEEP but not AEO-GONE, and nothing foreign.
|
||||
await runUpload([itemFrom({ hostname: 'AEO-KEEP', metric_id: 'Missing_AppID' })], 'NTS_AEO');
|
||||
|
||||
const hit = resolvedRows(activeRows);
|
||||
expect(hit.map(r => r.hostname)).toEqual(['AEO-GONE']);
|
||||
// The twelve foreign verticals are untouched — this is the regression guard.
|
||||
expect(hit.filter(r => r.vertical !== 'NTS_AEO')).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('property: no row outside the upload\'s vertical ever changes status', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(activeRowArb, { minLength: 1, maxLength: 30 }),
|
||||
verticalArb,
|
||||
fc.integer({ min: 0, max: 100 }),
|
||||
async (rawRows, uploadVertical, keepPct) => {
|
||||
activeRows = dedupe(rawRows);
|
||||
|
||||
// Include an arbitrary subset of the upload's own rows.
|
||||
const own = activeRows.filter(r => r.vertical === uploadVertical);
|
||||
const items = own
|
||||
.filter((_, i) => (i * 37) % 100 < keepPct)
|
||||
.map(r => itemFrom(r));
|
||||
|
||||
await runUpload(items, uploadVertical);
|
||||
|
||||
const hit = resolvedRows(activeRows);
|
||||
// Every resolved row belongs to the upload's vertical.
|
||||
expect(hit.every(r => r.vertical === uploadVertical)).toBe(true);
|
||||
},
|
||||
),
|
||||
{ numRuns: 200 },
|
||||
);
|
||||
});
|
||||
|
||||
it('property: an item absent from another vertical\'s sheet is left alone', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(activeRowArb, verticalArb, async (row, otherVertical) => {
|
||||
fc.pre(row.vertical !== otherVertical);
|
||||
activeRows = dedupe([row]);
|
||||
|
||||
// A sheet for a different vertical that mentions nothing.
|
||||
await runUpload([], otherVertical);
|
||||
|
||||
expect(recorded.resolves).toHaveLength(0);
|
||||
}),
|
||||
{ numRuns: 200 },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Property 2 — Resolved-count accuracy
|
||||
// **Validates: Requirements 2.3**
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('Property 2 — resolved_count counts only the upload\'s own vertical', () => {
|
||||
it('canonical fixture: resolved_count is 1, not 13', async () => {
|
||||
activeRows = fixtureUpload91Shape();
|
||||
const result = await runUpload(
|
||||
[itemFrom({ hostname: 'AEO-KEEP', metric_id: 'Missing_AppID' })], 'NTS_AEO');
|
||||
expect(result.resolvedCount).toBe(1);
|
||||
});
|
||||
|
||||
it('property: resolved_count equals own-vertical actives absent from the sheet', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(activeRowArb, { minLength: 1, maxLength: 25 }),
|
||||
verticalArb,
|
||||
async (rawRows, uploadVertical) => {
|
||||
activeRows = dedupe(rawRows);
|
||||
const own = activeRows.filter(r => r.vertical === uploadVertical);
|
||||
// Keep every other own row in the sheet.
|
||||
const kept = own.filter((_, i) => i % 2 === 0);
|
||||
const items = kept.map(r => itemFrom(r));
|
||||
|
||||
const result = await runUpload(items, uploadVertical);
|
||||
|
||||
expect(result.resolvedCount).toBe(own.length - kept.length);
|
||||
},
|
||||
),
|
||||
{ numRuns: 200 },
|
||||
);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Property 3 — Ownership transfer preserves history
|
||||
// **Validates: Requirements 2.4, 2.5, 2.6, 4.2, 4.4**
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('Property 3 — ownership transfer updates in place and preserves history', () => {
|
||||
it('the nine stranded CLIENTSIDEVM rows: SDIT_CSD row is claimed by NTS_AEO, not duplicated', async () => {
|
||||
activeRows = [{
|
||||
id: 42, hostname: 'CLIENTSIDEVM1', metric_id: 'Missing_AppID',
|
||||
seen_count: 6, first_seen_upload_id: 30, vertical: 'SDIT_CSD',
|
||||
}];
|
||||
|
||||
await runUpload(
|
||||
[itemFrom({ hostname: 'CLIENTSIDEVM1', metric_id: 'Missing_AppID' }, { team: 'STEAM' })],
|
||||
'NTS_AEO',
|
||||
);
|
||||
|
||||
// Claimed in place — no second row, no resolution.
|
||||
expect(recorded.inserts).toHaveLength(0);
|
||||
expect(recorded.resolves).toHaveLength(0);
|
||||
expect(recorded.updates).toHaveLength(1);
|
||||
|
||||
const p = recorded.updates[0].params;
|
||||
expect(p[1]).toBe(7); // seen_count incremented by exactly one
|
||||
expect(p[5]).toBe('STEAM'); // team refreshed
|
||||
expect(p[6]).toBe('NTS_AEO'); // vertical claimed
|
||||
expect(p[9]).toBe(42); // same row id
|
||||
|
||||
// first_seen_upload_id, resolution_date and remediation_plan are absent
|
||||
// from the SET clause, so they cannot be clobbered.
|
||||
expect(recorded.updates[0].text).not.toMatch(/first_seen_upload_id\s*=/);
|
||||
expect(recorded.updates[0].text).not.toMatch(/resolution_date\s*=/);
|
||||
expect(recorded.updates[0].text).not.toMatch(/remediation_plan\s*=/);
|
||||
});
|
||||
|
||||
it('property: a matched row is updated in place with seen_count + 1 and refreshed labels', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(activeRowArb, verticalArb, fc.stringMatching(/^[A-Z-]{3,10}$/),
|
||||
async (row, uploadVertical, team) => {
|
||||
activeRows = dedupe([row]);
|
||||
const stored = activeRows[0];
|
||||
|
||||
await runUpload(
|
||||
[itemFrom(stored, { team, category: 'Vulnerability Management' })],
|
||||
uploadVertical,
|
||||
);
|
||||
|
||||
expect(recorded.inserts).toHaveLength(0);
|
||||
expect(recorded.resolves).toHaveLength(0);
|
||||
expect(recorded.updates).toHaveLength(1);
|
||||
|
||||
const p = recorded.updates[0].params;
|
||||
expect(p[1]).toBe(stored.seen_count + 1);
|
||||
expect(p[5]).toBe(team);
|
||||
expect(p[6]).toBe(uploadVertical);
|
||||
expect(p[7]).toBe('Vulnerability Management');
|
||||
},
|
||||
),
|
||||
{ numRuns: 200 },
|
||||
);
|
||||
});
|
||||
|
||||
it('a new item records the upload\'s vertical on insert', async () => {
|
||||
activeRows = [];
|
||||
await runUpload([itemFrom({ hostname: 'BRAND-NEW', metric_id: 'Missing_OS' })], 'NTS_AEO');
|
||||
|
||||
expect(recorded.inserts).toHaveLength(1);
|
||||
expect(recorded.inserts[0].text).toMatch(/vertical/);
|
||||
expect(recorded.inserts[0].params).toContain('NTS_AEO');
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Property 4 — Single-vertical equivalence (preservation)
|
||||
// **Validates: Requirements 4.1**
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('Property 4 — single-vertical behaviour is unchanged from pre-fix', () => {
|
||||
it('property: when every row shares the upload\'s vertical, all absentees resolve', async () => {
|
||||
await fc.assert(
|
||||
fc.asyncProperty(
|
||||
fc.array(activeRowArb, { minLength: 1, maxLength: 25 }),
|
||||
verticalArb,
|
||||
async (rawRows, uploadVertical) => {
|
||||
// Force a single-vertical world — the pre-fix global sweep and
|
||||
// the scoped sweep must coincide exactly here.
|
||||
activeRows = dedupe(rawRows).map(r => ({ ...r, vertical: uploadVertical }));
|
||||
const kept = activeRows.filter((_, i) => i % 3 === 0);
|
||||
|
||||
const result = await runUpload(kept.map(r => itemFrom(r)), uploadVertical);
|
||||
|
||||
expect(result.resolvedCount).toBe(activeRows.length - kept.length);
|
||||
expect(resolvedRows(activeRows)).toHaveLength(activeRows.length - kept.length);
|
||||
},
|
||||
),
|
||||
{ numRuns: 200 },
|
||||
);
|
||||
});
|
||||
|
||||
it('legacy null-vertical uploads scope to null-vertical rows', async () => {
|
||||
activeRows = [
|
||||
{ id: 1, hostname: 'LEGACY-A', metric_id: 'Missing_OS', seen_count: 1, first_seen_upload_id: 1, vertical: null },
|
||||
{ id: 2, hostname: 'SCOPED-B', metric_id: 'Missing_OS', seen_count: 1, first_seen_upload_id: 1, vertical: 'NTS_AEO' },
|
||||
];
|
||||
await runUpload([], null);
|
||||
|
||||
const hit = resolvedRows(activeRows);
|
||||
expect(hit.map(r => r.hostname)).toEqual(['LEGACY-A']);
|
||||
});
|
||||
});
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Contract guards
|
||||
// **Validates: Requirements 2.8, 1.9**
|
||||
// ---------------------------------------------------------------------------
|
||||
describe('persistUpload requires a vertical', () => {
|
||||
it('throws when vertical is omitted entirely', async () => {
|
||||
activeRows = [];
|
||||
await expect(persistUpload({
|
||||
items: [], summary: {}, reportDate: '2026-07-20',
|
||||
filename: 'NTS_AEO_2026_07_20.xlsx', userId: 1,
|
||||
})).rejects.toThrow(/requires a vertical/);
|
||||
});
|
||||
|
||||
it('accepts an explicit null for the legacy path', async () => {
|
||||
activeRows = [];
|
||||
await expect(persistUpload({
|
||||
items: [], summary: { entries: [] }, reportDate: '2026-07-20',
|
||||
filename: 'NTS_AEO_2026_07_20.xlsx', userId: 1, vertical: null,
|
||||
})).resolves.toBeDefined();
|
||||
});
|
||||
});
|
||||
|
||||
describe('deriveVertical', () => {
|
||||
it.each([
|
||||
['NTS_AEO_2026_07_20.xlsx', 'NTS_AEO'],
|
||||
['SDIT_CSD_2026_05_18.xlsx', 'SDIT_CSD'],
|
||||
['TSI_2026_05_18.xlsx', 'TSI'],
|
||||
['PRDCT_VSO_2026_05_18.xlsx', 'PRDCT_VSO'],
|
||||
])('derives %s -> %s', (filename, expected) => {
|
||||
expect(deriveVertical(filename)).toBe(expected);
|
||||
});
|
||||
|
||||
it.each([
|
||||
['no-date.xlsx'],
|
||||
['2026_07_20.xlsx'],
|
||||
[''],
|
||||
[null],
|
||||
[undefined],
|
||||
])('returns null rather than guessing for %p', (filename) => {
|
||||
expect(deriveVertical(filename)).toBeNull();
|
||||
});
|
||||
});
|
||||
188
backend/__tests__/compliance-drift-metric-coverage.test.js
Normal file
188
backend/__tests__/compliance-drift-metric-coverage.test.js
Normal file
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* Drift Checker — metric category coverage
|
||||
*
|
||||
* Spec: .kiro/specs/compliance-cross-vertical-resolve-sweep/ (task 11)
|
||||
*
|
||||
* BACKGROUND
|
||||
* `Vulns_Aging` produced 46,087 rows categorised as "Other" without anyone
|
||||
* noticing. Detection was not the gap — compareSchemaToDrift already flagged
|
||||
* it as both an unknown metric and an unknown sheet. The gap was that
|
||||
* reconcileConfig "resolves" an unknown metric by adding it as "Other", which
|
||||
* satisfies both of those rules while leaving every ingested row
|
||||
* uncategorised. The warning disappears; the problem does not.
|
||||
*
|
||||
* `5.5.2` sat in the shipped config as "Other" with 11,712 uncategorised rows,
|
||||
* which is what that path produces.
|
||||
*
|
||||
* These tests pin the behaviour that closes the loop.
|
||||
*/
|
||||
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { compareSchemaToDrift, reconcileConfig, loadConfig } = require('../helpers/driftChecker');
|
||||
|
||||
const VALID_CATEGORIES = [
|
||||
'Vulnerability Management', 'Access & MFA', 'Logging & Monitoring',
|
||||
'End-of-Life OS', 'Decommissioned Assets', 'Asset Data Quality',
|
||||
'Application Security', 'Disaster Recovery', 'Endpoint Protection',
|
||||
];
|
||||
|
||||
const DETAIL_COLS = ['Preferred - Hostname', 'Team', 'Compliant'];
|
||||
|
||||
function makeSchema(metricNames) {
|
||||
return {
|
||||
sheets: [
|
||||
{ name: 'Summary', columns: ['Metric'], metric_values: metricNames },
|
||||
...metricNames.map(n => ({ name: n, columns: [...DETAIL_COLS] })),
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
function makeConfig(metricCategories, opts = {}) {
|
||||
const cfg = {
|
||||
metric_categories: metricCategories,
|
||||
core_cols: [...DETAIL_COLS],
|
||||
skip_sheets: ['Summary'],
|
||||
};
|
||||
if (opts.withValidCategories !== false) cfg.valid_categories = [...VALID_CATEGORIES];
|
||||
return cfg;
|
||||
}
|
||||
|
||||
function uncategorisedFindings(report) {
|
||||
return report.silent_miss.filter(f =>
|
||||
f.message.includes('rows will be ingested without a usable category'));
|
||||
}
|
||||
|
||||
describe('Drift: uncategorised metric detection', () => {
|
||||
it('flags a metric mapped to "Other" that has a detail sheet', () => {
|
||||
const report = compareSchemaToDrift(
|
||||
makeSchema(['2.3.4i', '5.5.2']),
|
||||
makeConfig({ '2.3.4i': 'Vulnerability Management', '5.5.2': 'Other' }),
|
||||
);
|
||||
const found = uncategorisedFindings(report);
|
||||
expect(found).toHaveLength(1);
|
||||
expect(found[0].value).toBe('5.5.2');
|
||||
expect(found[0].severity).toBe('silent_miss');
|
||||
// The message must name the acceptable categories so it is actionable.
|
||||
for (const c of VALID_CATEGORIES) expect(found[0].message).toContain(c);
|
||||
});
|
||||
|
||||
it('reports nothing when every metric has a real category', () => {
|
||||
const report = compareSchemaToDrift(
|
||||
makeSchema(['2.3.4i', 'Vulns_Aging', '5.5.2']),
|
||||
makeConfig({
|
||||
'2.3.4i': 'Vulnerability Management',
|
||||
'Vulns_Aging': 'Vulnerability Management',
|
||||
'5.5.2': 'End-of-Life OS',
|
||||
}),
|
||||
);
|
||||
expect(report.silent_miss).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('flags a typo\'d or renamed category, not just "Other"', () => {
|
||||
const report = compareSchemaToDrift(
|
||||
makeSchema(['2.3.4i']),
|
||||
makeConfig({ '2.3.4i': 'Vulnerabilty Mgmt' }), // misspelled
|
||||
);
|
||||
expect(uncategorisedFindings(report)).toHaveLength(1);
|
||||
});
|
||||
|
||||
it('does not double-report a metric that is absent from metric_categories', () => {
|
||||
// That case is already covered by the unknown-sheet / unknown-metric rules.
|
||||
const report = compareSchemaToDrift(
|
||||
makeSchema(['2.3.4i', 'BrandNew']),
|
||||
makeConfig({ '2.3.4i': 'Vulnerability Management' }),
|
||||
);
|
||||
expect(uncategorisedFindings(report)).toHaveLength(0);
|
||||
expect(report.silent_miss.filter(f => f.value === 'BrandNew')).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('skips the rule entirely when valid_categories is absent (backwards compatible)', () => {
|
||||
const report = compareSchemaToDrift(
|
||||
makeSchema(['2.3.4i', '5.5.2']),
|
||||
makeConfig({ '2.3.4i': 'Vulnerability Management', '5.5.2': 'Other' },
|
||||
{ withValidCategories: false }),
|
||||
);
|
||||
expect(uncategorisedFindings(report)).toHaveLength(0);
|
||||
});
|
||||
|
||||
it('does not flag a metric whose sheet is skipped', () => {
|
||||
const schema = makeSchema(['2.3.4i']);
|
||||
schema.sheets.push({ name: 'Aging Dashboard', columns: [...DETAIL_COLS] });
|
||||
const cfg = makeConfig({ '2.3.4i': 'Vulnerability Management', 'Aging Dashboard': 'Other' });
|
||||
cfg.skip_sheets = ['Summary', 'Aging Dashboard'];
|
||||
expect(uncategorisedFindings(compareSchemaToDrift(schema, cfg))).toHaveLength(0);
|
||||
});
|
||||
});
|
||||
|
||||
describe('Reconcile: the "Other" placeholder can no longer silence the warning', () => {
|
||||
let cfgPath;
|
||||
|
||||
beforeEach(() => {
|
||||
cfgPath = path.join(fs.mkdtempSync(path.join(os.tmpdir(), 'drift-')), 'compliance_config.json');
|
||||
});
|
||||
|
||||
it('adding a new metric as "Other" leaves it still flagged on the next check', () => {
|
||||
fs.writeFileSync(cfgPath, JSON.stringify(
|
||||
makeConfig({ '2.3.4i': 'Vulnerability Management' }), null, 2));
|
||||
const schema = makeSchema(['2.3.4i', 'Vulns_Aging']);
|
||||
|
||||
// Pass 1 — reconcile adds the placeholder.
|
||||
const pass1 = reconcileConfig(cfgPath,
|
||||
compareSchemaToDrift(schema, loadConfig(cfgPath)), schema);
|
||||
const added = pass1.changes.find(c => c.action === 'added' && c.value === 'Vulns_Aging');
|
||||
expect(added).toBeDefined();
|
||||
expect(added.detail).toMatch(/PLACEHOLDER ONLY/);
|
||||
expect(loadConfig(cfgPath).metric_categories['Vulns_Aging']).toBe('Other');
|
||||
|
||||
// Pass 2 — this is the regression. Before the fix this state was silent.
|
||||
const report2 = compareSchemaToDrift(schema, loadConfig(cfgPath));
|
||||
expect(uncategorisedFindings(report2).map(f => f.value)).toContain('Vulns_Aging');
|
||||
});
|
||||
|
||||
it('never auto-resolves an uncategorised metric — reports needs_review instead', () => {
|
||||
fs.writeFileSync(cfgPath, JSON.stringify(
|
||||
makeConfig({ '2.3.4i': 'Vulnerability Management', '5.5.2': 'Other' }), null, 2));
|
||||
const schema = makeSchema(['2.3.4i', '5.5.2']);
|
||||
|
||||
const { changes } = reconcileConfig(cfgPath,
|
||||
compareSchemaToDrift(schema, loadConfig(cfgPath)), schema);
|
||||
|
||||
const review = changes.find(c => c.action === 'needs_review' && c.value === '5.5.2');
|
||||
expect(review).toBeDefined();
|
||||
// The category must be left exactly as it was — no guessing.
|
||||
expect(loadConfig(cfgPath).metric_categories['5.5.2']).toBe('Other');
|
||||
});
|
||||
|
||||
it('preserves valid_categories when the config is rewritten', () => {
|
||||
fs.writeFileSync(cfgPath, JSON.stringify(
|
||||
makeConfig({ '2.3.4i': 'Vulnerability Management' }), null, 2));
|
||||
const schema = makeSchema(['2.3.4i', 'BrandNew']);
|
||||
|
||||
reconcileConfig(cfgPath, compareSchemaToDrift(schema, loadConfig(cfgPath)), schema);
|
||||
|
||||
expect(loadConfig(cfgPath).valid_categories).toEqual(VALID_CATEGORIES);
|
||||
});
|
||||
});
|
||||
|
||||
describe('The shipped config is fully categorised', () => {
|
||||
it('every metric maps to a category listed in valid_categories', () => {
|
||||
const cfg = loadConfig(path.join(__dirname, '..', 'scripts', 'compliance_config.json'));
|
||||
expect(Array.isArray(cfg.valid_categories)).toBe(true);
|
||||
|
||||
const valid = new Set(cfg.valid_categories);
|
||||
const offenders = Object.entries(cfg.metric_categories)
|
||||
.filter(([, category]) => !valid.has(category))
|
||||
.map(([metric, category]) => `${metric} -> ${category}`);
|
||||
|
||||
expect(offenders).toEqual([]);
|
||||
});
|
||||
|
||||
it('stays in sync with the frontend category map', () => {
|
||||
const backend = loadConfig(path.join(__dirname, '..', 'scripts', 'compliance_config.json')).metric_categories;
|
||||
const frontend = JSON.parse(fs.readFileSync(
|
||||
path.join(__dirname, '..', '..', 'frontend', 'src', 'data', 'complianceCategories.json'), 'utf8'));
|
||||
expect(frontend).toEqual(backend);
|
||||
});
|
||||
});
|
||||
@@ -1675,6 +1675,10 @@ describe('Preservation 2.E — persistUpload() snapshot is unchanged for single-
|
||||
reportDate: '2025-04-01',
|
||||
filename: 'nts-aeo-only-2025-04-01.xlsx',
|
||||
userId: 1,
|
||||
// Legacy AEO path: vertical is now passed explicitly rather than
|
||||
// defaulted, so the snapshot query still filters on
|
||||
// `vertical IS NOT DISTINCT FROM null`.
|
||||
vertical: null,
|
||||
});
|
||||
|
||||
// (1) Upload commit succeeded with the expected uploadId.
|
||||
@@ -1790,6 +1794,7 @@ describe('Preservation 2.F — persistUpload() commits the upload when snapshot
|
||||
reportDate: '2025-04-01',
|
||||
filename: 'preserve-error-2025-04-01.xlsx',
|
||||
userId: 1,
|
||||
vertical: null,
|
||||
});
|
||||
} catch (err) {
|
||||
thrown = err;
|
||||
|
||||
@@ -164,6 +164,33 @@ function compareSchemaToDrift(schema, config) {
|
||||
}
|
||||
}
|
||||
|
||||
// Uncategorised metric: the metric IS in metric_categories but maps to a
|
||||
// category the UI cannot render (anything outside valid_categories, notably
|
||||
// "Other"). Without this rule a metric can be silenced without being fixed:
|
||||
// reconcileConfig resolves an unknown metric by adding it as "Other", which
|
||||
// satisfies the two rules above while leaving every ingested row
|
||||
// uncategorised. This rule keeps reporting it until a real category is set.
|
||||
//
|
||||
// valid_categories is optional for backwards compatibility — when absent the
|
||||
// rule is skipped entirely rather than guessing.
|
||||
if (Array.isArray(config.valid_categories) && config.valid_categories.length > 0) {
|
||||
const validCategories = new Set(config.valid_categories);
|
||||
for (const sheet of detailSheets) {
|
||||
if (!metricCategoryKeys.has(sheet.name)) continue; // already flagged as unknown sheet
|
||||
const assigned = config.metric_categories[sheet.name];
|
||||
if (!validCategories.has(assigned)) {
|
||||
silent_miss.push({
|
||||
severity: 'silent_miss',
|
||||
message: `Metric "${sheet.name}" has a detail sheet but is categorised as "${assigned}" — `
|
||||
+ 'rows will be ingested without a usable category. Assign one of: '
|
||||
+ config.valid_categories.join(', '),
|
||||
value: sheet.name,
|
||||
sheet: sheet.name
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// --- Cosmetic rules ---
|
||||
|
||||
// New column in detail sheet: a detail sheet has columns not in core_cols
|
||||
@@ -210,7 +237,15 @@ function compareSchemaToDrift(schema, config) {
|
||||
* have a completely different column structure and shouldn't cause removal).
|
||||
*
|
||||
* Silent-miss — "unknown metric":
|
||||
* A metric value in the Summary is not in metric_categories. Add it as 'Other'.
|
||||
* A metric value in the Summary is not in metric_categories. Add it as 'Other'
|
||||
* as a PLACEHOLDER so the metric is tracked. This does not resolve the finding:
|
||||
* the "uncategorised metric" rule continues to report it until a real category
|
||||
* is assigned, so this can no longer silence the warning.
|
||||
*
|
||||
* Silent-miss — "uncategorised metric":
|
||||
* The metric is mapped but to a category the UI cannot render (e.g. 'Other').
|
||||
* Reported as 'needs_review' and never auto-resolved — picking the right
|
||||
* category requires knowing what the metric measures.
|
||||
*
|
||||
* Silent-miss — "unknown sheet":
|
||||
* Left as a warning. Auto-adding unknown sheets creates a reconcile loop.
|
||||
@@ -306,14 +341,34 @@ function reconcileConfig(configPath, driftReport, schema) {
|
||||
// --- Resolve silent-miss findings ---
|
||||
|
||||
for (const finding of (driftReport.silent_miss || [])) {
|
||||
// Unknown metric in Summary: add to metric_categories as 'Other'
|
||||
// Uncategorised metric: NOT auto-resolvable. Choosing the right category
|
||||
// requires knowing what the metric measures, so this is surfaced for a human
|
||||
// rather than silently defaulted. Skipped explicitly so the branch below
|
||||
// cannot claim to have handled it.
|
||||
if (finding.message.includes('rows will be ingested without a usable category')) {
|
||||
changes.push({
|
||||
action: 'needs_review',
|
||||
key: 'metric_categories',
|
||||
value: finding.value,
|
||||
detail: `Metric "${finding.value}" needs a real category — currently `
|
||||
+ `"${config.metric_categories[finding.value]}". Assign one manually; `
|
||||
+ 'leaving it uncategorised means its rows carry no usable category.'
|
||||
});
|
||||
continue;
|
||||
}
|
||||
|
||||
// Unknown metric in Summary: add to metric_categories as 'Other' so the
|
||||
// metric is at least tracked. This is a placeholder, NOT a resolution — the
|
||||
// "uncategorised metric" rule will keep reporting it until a real category
|
||||
// is assigned, so adding it here can no longer silence the warning.
|
||||
if (finding.message.includes('not in metric_categories') && !(finding.value in config.metric_categories)) {
|
||||
config.metric_categories[finding.value] = 'Other';
|
||||
changes.push({
|
||||
action: 'added',
|
||||
key: 'metric_categories',
|
||||
value: finding.value,
|
||||
detail: `Added new metric "${finding.value}" to metric_categories as "Other"`
|
||||
detail: `Added new metric "${finding.value}" as "Other" — PLACEHOLDER ONLY. `
|
||||
+ 'Assign a real category; until then its rows have no usable category.'
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
56
backend/migrations/add_supplemental_metadata.js
Normal file
56
backend/migrations/add_supplemental_metadata.js
Normal file
@@ -0,0 +1,56 @@
|
||||
const pool = require('../db');
|
||||
|
||||
async function run() {
|
||||
console.log('Starting supplemental metadata migration...');
|
||||
try {
|
||||
// Add resolution_date and remediation_plan to supplemental_items
|
||||
const { rows: cols } = await pool.query(`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'supplemental_items' AND column_name = 'resolution_date'
|
||||
`);
|
||||
if (cols.length === 0) {
|
||||
await pool.query(`ALTER TABLE supplemental_items ADD COLUMN resolution_date TEXT`);
|
||||
console.log('✓ resolution_date column added to supplemental_items');
|
||||
} else {
|
||||
console.log('✓ resolution_date column already exists (skipped)');
|
||||
}
|
||||
|
||||
const { rows: cols2 } = await pool.query(`
|
||||
SELECT column_name FROM information_schema.columns
|
||||
WHERE table_name = 'supplemental_items' AND column_name = 'remediation_plan'
|
||||
`);
|
||||
if (cols2.length === 0) {
|
||||
await pool.query(`ALTER TABLE supplemental_items ADD COLUMN remediation_plan TEXT`);
|
||||
console.log('✓ remediation_plan column added to supplemental_items');
|
||||
} else {
|
||||
console.log('✓ remediation_plan column already exists (skipped)');
|
||||
}
|
||||
|
||||
// Create supplemental_notes table
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS supplemental_notes (
|
||||
id SERIAL PRIMARY KEY,
|
||||
hostname TEXT NOT NULL,
|
||||
sheet_name TEXT,
|
||||
note TEXT NOT NULL,
|
||||
created_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
console.log('✓ supplemental_notes table created');
|
||||
|
||||
await pool.query(`CREATE INDEX IF NOT EXISTS idx_supplemental_notes_hostname ON supplemental_notes(hostname)`);
|
||||
console.log('✓ idx_supplemental_notes_hostname index created');
|
||||
|
||||
console.log('Migration complete.');
|
||||
} catch (err) {
|
||||
console.error('Migration failed:', err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
|
||||
if (require.main === module) {
|
||||
run().then(() => process.exit(0)).catch(() => process.exit(1));
|
||||
}
|
||||
71
backend/migrations/add_supplemental_tables.js
Normal file
71
backend/migrations/add_supplemental_tables.js
Normal file
@@ -0,0 +1,71 @@
|
||||
const pool = require('../db');
|
||||
|
||||
async function run() {
|
||||
console.log('Starting supplemental tables migration...');
|
||||
try {
|
||||
// Upload records for supplemental workbooks
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS supplemental_uploads (
|
||||
id SERIAL PRIMARY KEY,
|
||||
filename TEXT NOT NULL,
|
||||
report_date TEXT,
|
||||
uploaded_by INTEGER REFERENCES users(id) ON DELETE SET NULL,
|
||||
uploaded_at TIMESTAMPTZ DEFAULT NOW(),
|
||||
counts_json JSONB DEFAULT '{}'
|
||||
)
|
||||
`);
|
||||
console.log('✓ supplemental_uploads table created');
|
||||
|
||||
// One row per finding (asset × sheet)
|
||||
await pool.query(`
|
||||
CREATE TABLE IF NOT EXISTS supplemental_items (
|
||||
id SERIAL PRIMARY KEY,
|
||||
upload_id INTEGER NOT NULL REFERENCES supplemental_uploads(id) ON DELETE CASCADE,
|
||||
first_seen_upload_id INTEGER REFERENCES supplemental_uploads(id) ON DELETE SET NULL,
|
||||
resolved_upload_id INTEGER REFERENCES supplemental_uploads(id) ON DELETE SET NULL,
|
||||
sheet_name TEXT NOT NULL,
|
||||
hostname TEXT NOT NULL,
|
||||
ip_address TEXT,
|
||||
ipv6_address TEXT,
|
||||
device_function TEXT,
|
||||
device_type TEXT,
|
||||
model TEXT,
|
||||
vendor TEXT,
|
||||
equip_inst_id TEXT,
|
||||
responsible_team TEXT,
|
||||
team TEXT,
|
||||
application_id TEXT,
|
||||
application_ref_id TEXT,
|
||||
extra_json JSONB DEFAULT '{}',
|
||||
status TEXT NOT NULL DEFAULT 'active' CHECK(status IN ('active', 'resolved')),
|
||||
seen_count INTEGER DEFAULT 1,
|
||||
created_at TIMESTAMPTZ DEFAULT NOW()
|
||||
)
|
||||
`);
|
||||
console.log('✓ supplemental_items table created');
|
||||
|
||||
await pool.query(`CREATE INDEX IF NOT EXISTS idx_supplemental_items_upload ON supplemental_items(upload_id)`);
|
||||
console.log('✓ idx_supplemental_items_upload index created');
|
||||
|
||||
await pool.query(`CREATE INDEX IF NOT EXISTS idx_supplemental_items_identity ON supplemental_items(hostname, sheet_name)`);
|
||||
console.log('✓ idx_supplemental_items_identity index created');
|
||||
|
||||
await pool.query(`CREATE INDEX IF NOT EXISTS idx_supplemental_items_team_status ON supplemental_items(team, status)`);
|
||||
console.log('✓ idx_supplemental_items_team_status index created');
|
||||
|
||||
await pool.query(`CREATE INDEX IF NOT EXISTS idx_supplemental_items_sheet_status ON supplemental_items(sheet_name, status)`);
|
||||
console.log('✓ idx_supplemental_items_sheet_status index created');
|
||||
|
||||
console.log('Migration complete.');
|
||||
} catch (err) {
|
||||
console.error('Migration failed:', err.message);
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { run };
|
||||
|
||||
// Self-execute when run directly
|
||||
if (require.main === module) {
|
||||
run().then(() => process.exit(0)).catch(() => process.exit(1));
|
||||
}
|
||||
@@ -34,6 +34,8 @@ const POSTGRES_MIGRATIONS = [
|
||||
'add_user_ivanti_identity.js',
|
||||
'add_atlas_known_column.js',
|
||||
'add_session_impersonation.js',
|
||||
'add_supplemental_tables.js',
|
||||
'add_supplemental_metadata.js',
|
||||
];
|
||||
|
||||
async function runAll() {
|
||||
|
||||
@@ -59,19 +59,47 @@ function isSafeTempPath(filePath) {
|
||||
return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Derive the vertical from an upload filename.
|
||||
//
|
||||
// Compliance publishes one file per vertical named <VERTICAL>_YYYY_MM_DD.xlsx
|
||||
// (e.g. NTS_AEO_2026_07_20.xlsx -> NTS_AEO). The date suffix convention matches
|
||||
// extract_report_date() in scripts/parse_compliance_xlsx.py.
|
||||
//
|
||||
// Returns null when the filename does not match, so callers can reject rather
|
||||
// than invent a vertical.
|
||||
// ---------------------------------------------------------------------------
|
||||
function deriveVertical(filename) {
|
||||
if (!filename || typeof filename !== 'string') return null;
|
||||
const stem = path.basename(filename, path.extname(filename));
|
||||
const m = stem.match(/^(.+?)_(\d{4})_(\d{2})_(\d{2})$/);
|
||||
if (!m) return null;
|
||||
const vertical = m[1].trim();
|
||||
return vertical.length > 0 && vertical.length <= 64 ? vertical : null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compute diff: new / recurring / resolved
|
||||
// ---------------------------------------------------------------------------
|
||||
async function computeDiff(incomingItems) {
|
||||
// `vertical` mirrors the scoping persistUpload applies at commit time so the
|
||||
// preview predicts what committing will actually do. Matching stays global
|
||||
// (an asset may be claimed from another vertical) while the resolved count is
|
||||
// scoped to the incoming vertical. Passing undefined keeps the legacy global
|
||||
// count for callers that have no vertical to offer.
|
||||
async function computeDiff(incomingItems, vertical) {
|
||||
const { rows: activeRows } = await pool.query(
|
||||
`SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'`
|
||||
`SELECT hostname, metric_id, vertical FROM compliance_items WHERE status = 'active'`
|
||||
);
|
||||
const activeKeys = new Set(activeRows.map(r => `${r.hostname}|||${r.metric_id}`));
|
||||
const newKeys = new Set(incomingItems.map(i => `${i.hostname}|||${i.metric_id}`));
|
||||
|
||||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||||
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
|
||||
for (const k of activeKeys) { if (!newKeys.has(k)) resolvedCount++; }
|
||||
for (const k of newKeys) { if (activeKeys.has(k)) recurringCount++; else newCount++; }
|
||||
|
||||
for (const row of activeRows) {
|
||||
if (vertical !== undefined && row.vertical !== vertical) continue;
|
||||
if (!newKeys.has(`${row.hostname}|||${row.metric_id}`)) resolvedCount++;
|
||||
}
|
||||
|
||||
return { newCount, recurringCount, resolvedCount };
|
||||
}
|
||||
@@ -79,15 +107,36 @@ async function computeDiff(incomingItems) {
|
||||
// ---------------------------------------------------------------------------
|
||||
// Write a parsed upload to the DB (within a transaction)
|
||||
//
|
||||
// `vertical` defaults to null for legacy AEO uploads (the /commit route).
|
||||
// When threaded through from a multi-vertical caller it filters the
|
||||
// compliance_snapshots aggregation so the snapshot reflects only the
|
||||
// snapshotted vertical's items — this prevents cross-vertical
|
||||
// contamination on dates where multiple verticals share a report_date.
|
||||
// `vertical` identifies which vertical's spreadsheet this upload represents and
|
||||
// is required. A compliance spreadsheet covers exactly one vertical and makes no
|
||||
// assertion about any other vertical's assets, so its authority to resolve
|
||||
// findings is limited to that vertical.
|
||||
//
|
||||
// Two scopes are deliberately different:
|
||||
//
|
||||
// - The MATCH set is global. Matching on hostname|||metric_id across verticals
|
||||
// lets an asset whose ownership moved between reporting cycles update its
|
||||
// existing row in place, preserving seen_count, first_seen_upload_id and
|
||||
// remediation fields instead of fragmenting into a second row.
|
||||
//
|
||||
// - The SWEEP set is scoped to `vertical`. Only findings this spreadsheet has
|
||||
// authority over may be resolved. Sweeping globally marks every other
|
||||
// vertical's open findings as resolved (see
|
||||
// .kiro/specs/compliance-cross-vertical-resolve-sweep).
|
||||
//
|
||||
// `vertical` additionally filters the compliance_snapshots aggregation so the
|
||||
// snapshot reflects only the snapshotted vertical's items.
|
||||
// ---------------------------------------------------------------------------
|
||||
async function persistUpload({ items, summary, reportDate, filename, userId, vertical = null }) {
|
||||
async function persistUpload({ items, summary, reportDate, filename, userId, vertical }) {
|
||||
if (vertical === undefined) {
|
||||
throw new Error('persistUpload requires a vertical');
|
||||
}
|
||||
|
||||
// Global projection — serves the match set. `vertical` is selected so the
|
||||
// sweep can be filtered in application code without a second query.
|
||||
const { rows: activeRows } = await pool.query(
|
||||
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id FROM compliance_items WHERE status = 'active'`
|
||||
`SELECT id, hostname, metric_id, seen_count, first_seen_upload_id, vertical
|
||||
FROM compliance_items WHERE status = 'active'`
|
||||
);
|
||||
const activeMap = {};
|
||||
activeRows.forEach(r => { activeMap[`${r.hostname}|||${r.metric_id}`] = r; });
|
||||
@@ -100,10 +149,10 @@ async function persistUpload({ items, summary, reportDate, filename, userId, ver
|
||||
|
||||
// 1. Insert the upload record
|
||||
const uploadResult = await client.query(
|
||||
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json)
|
||||
VALUES ($1, $2, $3, NOW(), $4)
|
||||
`INSERT INTO compliance_uploads (filename, report_date, uploaded_by, uploaded_at, summary_json, vertical)
|
||||
VALUES ($1, $2, $3, NOW(), $4, $5)
|
||||
RETURNING id`,
|
||||
[filename, reportDate || null, userId || null, JSON.stringify(summary)]
|
||||
[filename, reportDate || null, userId || null, JSON.stringify(summary), vertical]
|
||||
);
|
||||
const uploadId = uploadResult.rows[0].id;
|
||||
|
||||
@@ -116,28 +165,40 @@ async function persistUpload({ items, summary, reportDate, filename, userId, ver
|
||||
const extraStr = JSON.stringify(item.extra_json || {});
|
||||
|
||||
if (existing) {
|
||||
// Refresh ownership (team, vertical) and derived labels
|
||||
// (category, metric_desc) from the incoming spreadsheet, which is
|
||||
// the authority. seen_count, first_seen_upload_id,
|
||||
// resolution_date and remediation_plan are deliberately preserved
|
||||
// so an ownership transfer keeps the asset's history.
|
||||
await client.query(
|
||||
`UPDATE compliance_items
|
||||
SET upload_id = $1, seen_count = $2, ip_address = $3, device_type = $4, extra_json = $5
|
||||
WHERE id = $6`,
|
||||
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr, existing.id]
|
||||
SET upload_id = $1, seen_count = $2, ip_address = $3, device_type = $4, extra_json = $5,
|
||||
team = $6, vertical = $7, category = $8, metric_desc = $9
|
||||
WHERE id = $10`,
|
||||
[uploadId, existing.seen_count + 1, item.ip_address, item.device_type, extraStr,
|
||||
item.team, vertical, item.category, item.metric_desc, existing.id]
|
||||
);
|
||||
recurringCount++;
|
||||
} else {
|
||||
await client.query(
|
||||
`INSERT INTO compliance_items
|
||||
(upload_id, hostname, ip_address, device_type, team, metric_id, metric_desc,
|
||||
category, extra_json, status, first_seen_upload_id, seen_count)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, 1)`,
|
||||
category, extra_json, status, first_seen_upload_id, seen_count, vertical)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, 'active', $10, 1, $11)`,
|
||||
[uploadId, item.hostname, item.ip_address, item.device_type, item.team,
|
||||
item.metric_id, item.metric_desc, item.category, extraStr, uploadId]
|
||||
item.metric_id, item.metric_desc, item.category, extraStr, uploadId, vertical]
|
||||
);
|
||||
newCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// 3. Mark items not present in this upload as resolved
|
||||
// 3. Mark items not present in this upload as resolved — SCOPED to this
|
||||
// upload's vertical. A spreadsheet covers one vertical and says nothing
|
||||
// about any other, so absence from it is not evidence of resolution for
|
||||
// assets owned elsewhere. Rows claimed by the upsert above already
|
||||
// carry this vertical and are excluded by the newKeys check.
|
||||
for (const [key, row] of Object.entries(activeMap)) {
|
||||
if (row.vertical !== vertical) continue;
|
||||
if (!newKeys.has(key)) {
|
||||
await client.query(
|
||||
`UPDATE compliance_items SET status = 'resolved', resolved_upload_id = $1 WHERE id = $2`,
|
||||
@@ -336,7 +397,10 @@ function createComplianceRouter(upload) {
|
||||
return res.status(422).json({ error: parsed.error });
|
||||
}
|
||||
|
||||
const diff = await computeDiff(parsed.items);
|
||||
// Scope the predicted resolve count to this file's vertical so
|
||||
// the preview matches what /commit will do.
|
||||
const previewVertical = deriveVertical(req.file.originalname);
|
||||
const diff = await computeDiff(parsed.items, previewVertical ?? undefined);
|
||||
|
||||
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||||
const tempFilename = `compliance_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
|
||||
@@ -398,13 +462,18 @@ function createComplianceRouter(upload) {
|
||||
* Commits a previously previewed compliance upload to the database. Resolves items no longer
|
||||
* present, upserts recurring/new items, and creates a compliance snapshot for the current month.
|
||||
*
|
||||
* @body { tempFile: string, filename?: string, report_date?: string }
|
||||
* @body { tempFile: string, filename?: string, report_date?: string, vertical?: string }
|
||||
* @response 200 { upload: { id, filename, report_date, uploaded_at, new_count, resolved_count, recurring_count } }
|
||||
* @response 400 { error } — missing/invalid tempFile or expired preview session
|
||||
* @response 400 { error, filename } — vertical could not be determined from the body or filename
|
||||
* @response 500 { error } — commit failure
|
||||
*
|
||||
* `vertical` scopes which findings this upload may resolve. When omitted it
|
||||
* is derived from the filename (<VERTICAL>_YYYY_MM_DD.xlsx); if neither is
|
||||
* available the request is rejected rather than sweeping every vertical.
|
||||
*/
|
||||
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { tempFile, filename, report_date } = req.body;
|
||||
const { tempFile, filename, report_date, vertical } = req.body;
|
||||
if (!tempFile || typeof tempFile !== 'string') return res.status(400).json({ error: 'tempFile is required' });
|
||||
// Reconstruct full path from basename only — never trust a client-supplied absolute path
|
||||
const resolvedTempFile = path.join(TEMP_DIR, path.basename(tempFile));
|
||||
@@ -415,12 +484,34 @@ function createComplianceRouter(upload) {
|
||||
try { parsed = JSON.parse(fs.readFileSync(resolvedTempFile, 'utf8')); }
|
||||
catch { return res.status(400).json({ error: 'Could not read preview data — please upload again' }); }
|
||||
|
||||
// A spreadsheet's authority to resolve findings is limited to one
|
||||
// vertical, so an upload cannot be committed without knowing which.
|
||||
// Prefer an explicit value, otherwise derive it from the filename.
|
||||
const effectiveFilename = filename || parsed.filename;
|
||||
let resolvedVertical = null;
|
||||
if (vertical !== undefined && vertical !== null && vertical !== '') {
|
||||
if (typeof vertical !== 'string' || vertical.length > 64) {
|
||||
return res.status(400).json({ error: 'Invalid vertical' });
|
||||
}
|
||||
resolvedVertical = vertical.trim();
|
||||
} else {
|
||||
resolvedVertical = deriveVertical(effectiveFilename);
|
||||
}
|
||||
if (!resolvedVertical) {
|
||||
return res.status(400).json({
|
||||
error: 'Could not determine the vertical for this upload. Supply "vertical" in the request body, '
|
||||
+ 'or name the file <VERTICAL>_YYYY_MM_DD.xlsx (e.g. NTS_AEO_2026_07_20.xlsx).',
|
||||
filename: effectiveFilename || null,
|
||||
});
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await persistUpload({
|
||||
items: parsed.items, summary: parsed.summary,
|
||||
reportDate: report_date || parsed.report_date,
|
||||
filename: filename || parsed.filename,
|
||||
filename: effectiveFilename,
|
||||
userId: req.user?.id || null,
|
||||
vertical: resolvedVertical,
|
||||
});
|
||||
fs.unlink(resolvedTempFile, () => {});
|
||||
|
||||
@@ -526,6 +617,7 @@ function createComplianceRouter(upload) {
|
||||
* @query team — optional, one of STEAM | ACCESS-ENG | ACCESS-OPS | INTELDEV
|
||||
* @response 200 { entries: Array, overall_scores: object, upload: { id, report_date, uploaded_at } | null, multi_vertical_uploads?: Array<{ id, vertical, uploaded_at }> }
|
||||
* @response 400 { error } — invalid team
|
||||
* @response 403 { error, code: 'TEAM_ACCESS_DENIED', requested, allowed } — requested team outside the caller's team scope
|
||||
* @response 500 { error } — database error
|
||||
*
|
||||
* When two or more uploads share the latest `report_date` (multi-vertical
|
||||
@@ -603,6 +695,7 @@ function createComplianceRouter(upload) {
|
||||
* @query status — optional, "active" (default) or "resolved"
|
||||
* @response 200 { devices: Array<{ hostname, ip_address, device_type, team, status, failing_metrics, seen_count, first_seen, last_seen, resolved_on, has_notes }>, team, status }
|
||||
* @response 400 { error } — missing/invalid team or invalid status
|
||||
* @response 403 { error, code: 'TEAM_ACCESS_DENIED', requested, allowed } — requested team outside the caller's team scope
|
||||
* @response 500 { error } — database error
|
||||
*/
|
||||
router.get('/items', async (req, res) => {
|
||||
@@ -1766,4 +1859,4 @@ function createComplianceRouter(upload) {
|
||||
return router;
|
||||
}
|
||||
|
||||
module.exports = { createComplianceRouter, bucketAgingItems, computeWaterfall, persistUpload, groupByHostname };
|
||||
module.exports = { createComplianceRouter, bucketAgingItems, computeWaterfall, persistUpload, groupByHostname, deriveVertical };
|
||||
|
||||
843
backend/routes/supplemental.js
Normal file
843
backend/routes/supplemental.js
Normal file
@@ -0,0 +1,843 @@
|
||||
// Supplemental (Granite Hygiene) Routes
|
||||
// Handles supplemental workbook upload/parse, per-sheet finding tracking, and trends.
|
||||
|
||||
const express = require('express');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const { spawn } = require('child_process');
|
||||
const pool = require('../db');
|
||||
const { requireAuth, requireGroup, requireTeam } = require('../middleware/auth');
|
||||
const logAudit = require('../helpers/auditLog');
|
||||
|
||||
const PARSER_SCRIPT = path.join(__dirname, '../scripts/parse_supplemental_xlsx.py');
|
||||
const PYTHON_BIN = process.env.PYTHON_BIN || 'python3';
|
||||
const TEMP_DIR = path.join(process.cwd(), 'uploads', 'temp');
|
||||
const ALLOWED_TEAMS = new Set(['STEAM', 'ACCESS-ENG', 'ACCESS-OPS', 'INTELDEV']);
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Run Python parser
|
||||
// ---------------------------------------------------------------------------
|
||||
function parseSupplementalXlsx(filePath) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const py = spawn(PYTHON_BIN, [PARSER_SCRIPT, filePath]);
|
||||
let out = '';
|
||||
let err = '';
|
||||
py.stdout.on('data', d => { out += d; });
|
||||
py.stderr.on('data', d => { err += d; });
|
||||
py.on('close', code => {
|
||||
if (code !== 0) return reject(new Error(err || `Parser exited with code ${code}`));
|
||||
try { resolve(JSON.parse(out)); }
|
||||
catch (e) { reject(new Error('Parser returned invalid JSON')); }
|
||||
});
|
||||
py.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function isSafeTempPath(filePath) {
|
||||
const resolved = path.resolve(filePath);
|
||||
return resolved.startsWith(TEMP_DIR + path.sep) && path.extname(resolved) === '.json';
|
||||
}
|
||||
|
||||
function extractReportDate(filename) {
|
||||
if (!filename) return null;
|
||||
const stem = path.basename(filename, path.extname(filename));
|
||||
const m = stem.match(/(\d{4})_(\d{2})_(\d{2})/);
|
||||
return m ? `${m[1]}-${m[2]}-${m[3]}` : null;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Compute per-sheet diff against current active items
|
||||
// ---------------------------------------------------------------------------
|
||||
async function computeSheetDiffs(sheets) {
|
||||
const { rows: activeRows } = await pool.query(
|
||||
`SELECT id, hostname, sheet_name FROM supplemental_items WHERE status = 'active'`
|
||||
);
|
||||
|
||||
// Build map: sheet_name -> Set of hostnames
|
||||
const activeBySheet = {};
|
||||
for (const row of activeRows) {
|
||||
if (!activeBySheet[row.sheet_name]) activeBySheet[row.sheet_name] = new Set();
|
||||
activeBySheet[row.sheet_name].add(row.hostname);
|
||||
}
|
||||
|
||||
const diffs = {};
|
||||
for (const [sheetName, items] of Object.entries(sheets)) {
|
||||
const activeSet = activeBySheet[sheetName] || new Set();
|
||||
const incomingSet = new Set(items.map(i => i.hostname));
|
||||
|
||||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||||
for (const h of incomingSet) {
|
||||
if (activeSet.has(h)) recurringCount++;
|
||||
else newCount++;
|
||||
}
|
||||
for (const h of activeSet) {
|
||||
if (!incomingSet.has(h)) resolvedCount++;
|
||||
}
|
||||
|
||||
diffs[sheetName] = {
|
||||
count: items.length,
|
||||
new: newCount,
|
||||
recurring: recurringCount,
|
||||
resolved: resolvedCount,
|
||||
};
|
||||
}
|
||||
|
||||
return diffs;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Persist upload in a transaction
|
||||
// ---------------------------------------------------------------------------
|
||||
async function persistSupplementalUpload({ sheets, reportDate, filename, userId }) {
|
||||
// Load all active items globally (across all sheets)
|
||||
const { rows: activeRows } = await pool.query(
|
||||
`SELECT id, hostname, sheet_name, seen_count, first_seen_upload_id
|
||||
FROM supplemental_items WHERE status = 'active'`
|
||||
);
|
||||
|
||||
// Build map: "hostname|||sheet_name" -> row
|
||||
const activeMap = {};
|
||||
for (const row of activeRows) {
|
||||
activeMap[`${row.hostname}|||${row.sheet_name}`] = row;
|
||||
}
|
||||
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
// Insert upload record
|
||||
const uploadResult = await client.query(
|
||||
`INSERT INTO supplemental_uploads (filename, report_date, uploaded_by, uploaded_at)
|
||||
VALUES ($1, $2, $3, NOW()) RETURNING id`,
|
||||
[filename, reportDate || null, userId || null]
|
||||
);
|
||||
const uploadId = uploadResult.rows[0].id;
|
||||
|
||||
const countsJson = {};
|
||||
|
||||
for (const [sheetName, items] of Object.entries(sheets)) {
|
||||
let newCount = 0, recurringCount = 0, resolvedCount = 0;
|
||||
const incomingHostnames = new Set();
|
||||
|
||||
// Upsert each item
|
||||
for (const item of items) {
|
||||
const key = `${item.hostname}|||${sheetName}`;
|
||||
incomingHostnames.add(item.hostname);
|
||||
const existing = activeMap[key];
|
||||
const extraStr = JSON.stringify(item.extra_json || {});
|
||||
|
||||
if (existing) {
|
||||
// Update existing — increment seen_count, refresh metadata
|
||||
await client.query(
|
||||
`UPDATE supplemental_items
|
||||
SET upload_id = $1, seen_count = $2, ip_address = $3, ipv6_address = $4,
|
||||
device_function = $5, device_type = $6, model = $7, vendor = $8,
|
||||
equip_inst_id = $9, responsible_team = $10, team = $11,
|
||||
application_id = $12, application_ref_id = $13, extra_json = $14
|
||||
WHERE id = $15`,
|
||||
[uploadId, existing.seen_count + 1, item.ip_address, item.ipv6_address,
|
||||
item.device_function, item.device_type, item.model, item.vendor,
|
||||
item.equip_inst_id, item.responsible_team, item.team,
|
||||
item.application_id, item.application_ref_id, extraStr, existing.id]
|
||||
);
|
||||
recurringCount++;
|
||||
} else {
|
||||
// New item
|
||||
await client.query(
|
||||
`INSERT INTO supplemental_items
|
||||
(upload_id, first_seen_upload_id, sheet_name, hostname, ip_address, ipv6_address,
|
||||
device_function, device_type, model, vendor, equip_inst_id, responsible_team,
|
||||
team, application_id, application_ref_id, extra_json, status, seen_count)
|
||||
VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, 'active', 1)`,
|
||||
[uploadId, uploadId, sheetName, item.hostname, item.ip_address, item.ipv6_address,
|
||||
item.device_function, item.device_type, item.model, item.vendor,
|
||||
item.equip_inst_id, item.responsible_team, item.team,
|
||||
item.application_id, item.application_ref_id, extraStr]
|
||||
);
|
||||
newCount++;
|
||||
}
|
||||
}
|
||||
|
||||
// Resolve items for THIS sheet that are no longer present
|
||||
for (const [key, row] of Object.entries(activeMap)) {
|
||||
if (row.sheet_name !== sheetName) continue;
|
||||
if (!incomingHostnames.has(row.hostname)) {
|
||||
await client.query(
|
||||
`UPDATE supplemental_items SET status = 'resolved', resolved_upload_id = $1 WHERE id = $2`,
|
||||
[uploadId, row.id]
|
||||
);
|
||||
resolvedCount++;
|
||||
}
|
||||
}
|
||||
|
||||
countsJson[sheetName] = { new: newCount, recurring: recurringCount, resolved: resolvedCount };
|
||||
}
|
||||
|
||||
// Update upload record with counts
|
||||
await client.query(
|
||||
`UPDATE supplemental_uploads SET counts_json = $1 WHERE id = $2`,
|
||||
[JSON.stringify(countsJson), uploadId]
|
||||
);
|
||||
|
||||
await client.query('COMMIT');
|
||||
return { uploadId, counts: countsJson };
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Router factory
|
||||
// ---------------------------------------------------------------------------
|
||||
function createSupplementalRouter(upload) {
|
||||
const router = express.Router();
|
||||
|
||||
router.use(requireAuth());
|
||||
router.use(requireTeam());
|
||||
|
||||
/**
|
||||
* POST /preview
|
||||
*
|
||||
* Upload a supplemental xlsx workbook, parse it via the Python parser,
|
||||
* compute per-sheet diffs against current active items, and save parsed
|
||||
* data to a temp file for subsequent commit.
|
||||
*
|
||||
* @body {file} file - Multipart .xlsx file upload (field name: "file")
|
||||
* @response 200 - { sheets: { [sheetName]: { count, new, recurring, resolved } }, teams: { [team]: count }, report_date: string|null, tempFile: string, filename: string, total: number }
|
||||
* @response 400 - { error: string } — no file, wrong extension, or multer error
|
||||
* @response 422 - { error: string } — parser returned a structured error
|
||||
* @response 500 - { error: string } — parser crash or unexpected failure
|
||||
*/
|
||||
router.post('/preview', requireGroup('Admin', 'Standard_User'), (req, res) => {
|
||||
upload.single('file')(req, res, async (uploadErr) => {
|
||||
if (uploadErr) return res.status(400).json({ error: uploadErr.message });
|
||||
if (!req.file) return res.status(400).json({ error: 'No file uploaded' });
|
||||
if (path.extname(req.file.originalname).toLowerCase() !== '.xlsx') {
|
||||
fs.unlink(req.file.path, () => {});
|
||||
return res.status(400).json({ error: 'File must be an .xlsx spreadsheet' });
|
||||
}
|
||||
|
||||
try {
|
||||
const parsed = await parseSupplementalXlsx(req.file.path);
|
||||
if (parsed.error) {
|
||||
fs.unlink(req.file.path, () => {});
|
||||
return res.status(422).json({ error: parsed.error });
|
||||
}
|
||||
|
||||
const diffs = await computeSheetDiffs(parsed.sheets);
|
||||
|
||||
// Compute team breakdown
|
||||
const teams = {};
|
||||
for (const items of Object.values(parsed.sheets)) {
|
||||
for (const item of items) {
|
||||
if (item.team) {
|
||||
teams[item.team] = (teams[item.team] || 0) + 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Save to temp file
|
||||
if (!fs.existsSync(TEMP_DIR)) fs.mkdirSync(TEMP_DIR, { recursive: true });
|
||||
const tempFilename = `supp_preview_${Date.now()}_${Math.random().toString(36).slice(2)}.json`;
|
||||
const tempFilePath = path.join(TEMP_DIR, tempFilename);
|
||||
|
||||
fs.writeFileSync(tempFilePath, JSON.stringify({
|
||||
sheets: parsed.sheets,
|
||||
report_date: parsed.report_date,
|
||||
filename: req.file.originalname.replace(/[^\w.\-() ]/g, '_'),
|
||||
}));
|
||||
|
||||
fs.unlink(req.file.path, () => {});
|
||||
|
||||
res.json({
|
||||
sheets: diffs,
|
||||
teams,
|
||||
report_date: parsed.report_date || extractReportDate(req.file.originalname),
|
||||
tempFile: tempFilename,
|
||||
filename: req.file.originalname,
|
||||
total: parsed.total,
|
||||
});
|
||||
} catch (err) {
|
||||
fs.unlink(req.file.path, () => {});
|
||||
console.error('[Supplemental] Preview error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to parse file: ' + err.message });
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /commit
|
||||
*
|
||||
* Commit a previously previewed supplemental upload. Reads the temp file
|
||||
* saved during /preview, persists all sheet items to the database (upsert
|
||||
* new/recurring, resolve removed), and returns the created upload record.
|
||||
*
|
||||
* @body {string} tempFile - Filename of the preview temp file (from /preview response)
|
||||
* @response 200 - { upload: { id, filename, report_date, uploaded_at, counts_json } }
|
||||
* @response 400 - { error: string } — missing tempFile, invalid path, expired session, or unreadable data
|
||||
* @response 500 - { error: string } — database transaction failure
|
||||
*/
|
||||
router.post('/commit', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { tempFile } = req.body;
|
||||
if (!tempFile || typeof tempFile !== 'string') return res.status(400).json({ error: 'tempFile is required' });
|
||||
|
||||
const resolvedTempFile = path.join(TEMP_DIR, path.basename(tempFile));
|
||||
if (!isSafeTempPath(resolvedTempFile)) return res.status(400).json({ error: 'Invalid tempFile path' });
|
||||
if (!fs.existsSync(resolvedTempFile)) return res.status(400).json({ error: 'Preview session expired — please upload again' });
|
||||
|
||||
let parsed;
|
||||
try { parsed = JSON.parse(fs.readFileSync(resolvedTempFile, 'utf8')); }
|
||||
catch { return res.status(400).json({ error: 'Could not read preview data — please upload again' }); }
|
||||
|
||||
try {
|
||||
const result = await persistSupplementalUpload({
|
||||
sheets: parsed.sheets,
|
||||
reportDate: parsed.report_date,
|
||||
filename: parsed.filename,
|
||||
userId: req.user?.id || null,
|
||||
});
|
||||
|
||||
fs.unlink(resolvedTempFile, () => {});
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id, username: req.user.username,
|
||||
action: 'supplemental_upload_commit', entityType: 'supplemental_upload',
|
||||
entityId: String(result.uploadId),
|
||||
details: { filename: parsed.filename, counts: result.counts },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
const { rows } = await pool.query(
|
||||
`SELECT id, filename, report_date, uploaded_at, counts_json FROM supplemental_uploads WHERE id = $1`,
|
||||
[result.uploadId]
|
||||
);
|
||||
res.json({ upload: rows[0] });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] Commit error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to commit upload: ' + err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /summary
|
||||
*
|
||||
* Returns per-sheet active item counts with team breakdown and last upload info.
|
||||
*
|
||||
* @query {string} [team] - Filter by team (STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV)
|
||||
* @response 200 - { sheets: [{ sheet_name, label, total_active, teams: { [team]: count } }], last_upload: { id, filename, report_date, uploaded_at }|null, total_active: number }
|
||||
* @response 400 - { error: string } — invalid team value
|
||||
* @response 403 - { error: string, code: 'TEAM_ACCESS_DENIED' } — team not in user's scope
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.get('/summary', async (req, res) => {
|
||||
const team = req.query.team;
|
||||
if (team && !ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
||||
if (team && req.teamScope && !req.teamScope.short.includes(team)) {
|
||||
return res.status(403).json({ error: 'Access denied', code: 'TEAM_ACCESS_DENIED' });
|
||||
}
|
||||
|
||||
try {
|
||||
// Get per-sheet counts with team breakdown
|
||||
let query = `SELECT sheet_name, team, COUNT(*)::int AS count
|
||||
FROM supplemental_items WHERE status = 'active'`;
|
||||
const params = [];
|
||||
if (team) {
|
||||
query += ` AND team = $1`;
|
||||
params.push(team);
|
||||
} else if (req.teamScope) {
|
||||
query += ` AND team = ANY($1)`;
|
||||
params.push(req.teamScope.short);
|
||||
}
|
||||
query += ` GROUP BY sheet_name, team ORDER BY sheet_name, team`;
|
||||
|
||||
const { rows } = await pool.query(query, params);
|
||||
|
||||
// Aggregate into per-sheet summary
|
||||
const sheetMap = {};
|
||||
for (const row of rows) {
|
||||
if (!sheetMap[row.sheet_name]) {
|
||||
sheetMap[row.sheet_name] = { sheet_name: row.sheet_name, total_active: 0, teams: {} };
|
||||
}
|
||||
sheetMap[row.sheet_name].total_active += row.count;
|
||||
sheetMap[row.sheet_name].teams[row.team] = row.count;
|
||||
}
|
||||
|
||||
const SHEET_LABELS = {
|
||||
'Missing_AppID': 'Missing App ID',
|
||||
'Missing_DF': 'Missing Device Function',
|
||||
'Missing_OS_Granite': 'Missing OS (Granite)',
|
||||
'Retired_AppID': 'Retired App ID',
|
||||
};
|
||||
|
||||
const sheets = Object.values(sheetMap).map(s => ({
|
||||
...s,
|
||||
label: SHEET_LABELS[s.sheet_name] || s.sheet_name,
|
||||
}));
|
||||
|
||||
// Get last upload info
|
||||
const { rows: uploadRows } = await pool.query(
|
||||
`SELECT id, filename, report_date, uploaded_at FROM supplemental_uploads ORDER BY id DESC LIMIT 1`
|
||||
);
|
||||
|
||||
const totalActive = sheets.reduce((sum, s) => sum + s.total_active, 0);
|
||||
|
||||
res.json({
|
||||
sheets,
|
||||
last_upload: uploadRows[0] || null,
|
||||
total_active: totalActive,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] GET /summary error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /items
|
||||
*
|
||||
* Paginated device list for a specific sheet, optionally filtered by team and hostname search.
|
||||
*
|
||||
* @query {string} sheet - Sheet name to query (required)
|
||||
* @query {string} [team] - Filter by team (STEAM, ACCESS-ENG, ACCESS-OPS, INTELDEV)
|
||||
* @query {number} [page=1] - Page number (1-indexed)
|
||||
* @query {number} [page_size=50] - Results per page (max 200)
|
||||
* @query {string} [search] - Hostname substring filter (ILIKE match)
|
||||
* @response 200 - { devices: [{ hostname, ip_address, ipv6_address, device_function, device_type, model, vendor, equip_inst_id, responsible_team, team, application_id, application_ref_id, extra_json, seen_count, first_seen, last_seen }], total: number, page: number, page_size: number }
|
||||
* @response 400 - { error: string } — missing sheet or invalid team
|
||||
* @response 403 - { error: string, code: 'TEAM_ACCESS_DENIED' } — team not in user's scope
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.get('/items', async (req, res) => {
|
||||
const { sheet, team, page = '1', page_size = '50', search } = req.query;
|
||||
if (!sheet) return res.status(400).json({ error: 'sheet is required' });
|
||||
if (team && !ALLOWED_TEAMS.has(team)) return res.status(400).json({ error: 'Invalid team' });
|
||||
if (team && req.teamScope && !req.teamScope.short.includes(team)) {
|
||||
return res.status(403).json({ error: 'Access denied', code: 'TEAM_ACCESS_DENIED' });
|
||||
}
|
||||
|
||||
const pageNum = Math.max(1, parseInt(page, 10) || 1);
|
||||
const size = Math.min(200, Math.max(1, parseInt(page_size, 10) || 50));
|
||||
const offset = (pageNum - 1) * size;
|
||||
|
||||
try {
|
||||
let whereConditions = [`si.sheet_name = $1`, `si.status = 'active'`];
|
||||
const params = [sheet];
|
||||
let paramIdx = 2;
|
||||
|
||||
if (team) {
|
||||
whereConditions.push(`si.team = $${paramIdx++}`);
|
||||
params.push(team);
|
||||
} else if (req.teamScope) {
|
||||
whereConditions.push(`si.team = ANY($${paramIdx++})`);
|
||||
params.push(req.teamScope.short);
|
||||
}
|
||||
|
||||
if (search && search.trim()) {
|
||||
whereConditions.push(`si.hostname ILIKE $${paramIdx++}`);
|
||||
params.push(`%${search.trim()}%`);
|
||||
}
|
||||
|
||||
const whereClause = whereConditions.join(' AND ');
|
||||
|
||||
// Get total count
|
||||
const { rows: countRows } = await pool.query(
|
||||
`SELECT COUNT(*)::int AS total FROM supplemental_items si WHERE ${whereClause}`, params
|
||||
);
|
||||
const total = countRows[0].total;
|
||||
|
||||
// Get paginated items
|
||||
const { rows } = await pool.query(
|
||||
`SELECT si.hostname, si.ip_address, si.ipv6_address, si.device_function, si.device_type,
|
||||
si.model, si.vendor, si.equip_inst_id, si.responsible_team, si.team,
|
||||
si.application_id, si.application_ref_id, si.extra_json, si.seen_count,
|
||||
fu.report_date AS first_seen, lu.report_date AS last_seen
|
||||
FROM supplemental_items si
|
||||
LEFT JOIN supplemental_uploads fu ON si.first_seen_upload_id = fu.id
|
||||
LEFT JOIN supplemental_uploads lu ON si.upload_id = lu.id
|
||||
WHERE ${whereClause}
|
||||
ORDER BY si.hostname ASC
|
||||
LIMIT $${paramIdx++} OFFSET $${paramIdx++}`,
|
||||
[...params, size, offset]
|
||||
);
|
||||
|
||||
res.json({ devices: rows, total, page: pageNum, page_size: size });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] GET /items error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /trends
|
||||
*
|
||||
* Returns historical cumulative active counts per sheet, computed from
|
||||
* upload-level new/resolved deltas. One data point per upload date.
|
||||
*
|
||||
* @response 200 - { trends: [{ report_date: string, [sheetName]: number, ... }] }
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.get('/trends', async (req, res) => {
|
||||
try {
|
||||
const { rows } = await pool.query(
|
||||
`SELECT id, report_date, counts_json FROM supplemental_uploads ORDER BY report_date ASC`
|
||||
);
|
||||
|
||||
// Build cumulative active counts per date based on new/resolved deltas
|
||||
// Simpler: just query active item counts per sheet as of each upload
|
||||
// Actually, counts_json has the new/recurring/resolved — we can derive active totals
|
||||
const trends = [];
|
||||
const runningTotals = {};
|
||||
|
||||
for (const upload of rows) {
|
||||
const counts = upload.counts_json || {};
|
||||
const point = { report_date: upload.report_date };
|
||||
|
||||
for (const [sheetName, sheetCounts] of Object.entries(counts)) {
|
||||
if (!runningTotals[sheetName]) runningTotals[sheetName] = 0;
|
||||
runningTotals[sheetName] += (sheetCounts.new || 0) - (sheetCounts.resolved || 0);
|
||||
point[sheetName] = Math.max(0, runningTotals[sheetName]);
|
||||
}
|
||||
|
||||
trends.push(point);
|
||||
}
|
||||
|
||||
res.json({ trends });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] GET /trends error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /uploads
|
||||
*
|
||||
* Returns the full supplemental upload history, most recent first.
|
||||
*
|
||||
* @response 200 - { uploads: [{ id, filename, report_date, uploaded_at, counts_json }] }
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.get('/uploads', async (req, res) => {
|
||||
try {
|
||||
const { rows } = await pool.query(
|
||||
`SELECT id, filename, report_date, uploaded_at, counts_json
|
||||
FROM supplemental_uploads ORDER BY id DESC`
|
||||
);
|
||||
res.json({ uploads: rows });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] GET /uploads error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /rollback/:uploadId
|
||||
*
|
||||
* Rolls back the most recent supplemental upload. Deletes items first
|
||||
* introduced by that upload, reactivates items resolved by it, and
|
||||
* decrements seen_count for recurring items. Admin only.
|
||||
*
|
||||
* @param {string} uploadId - ID of the upload to roll back (must be the most recent)
|
||||
* @response 200 - { message: string, rolled_back: { upload_id, filename, items_deleted, items_reactivated } }
|
||||
* @response 400 - { error: string } — invalid ID or not the most recent upload
|
||||
* @response 404 - { error: string } — upload not found
|
||||
* @response 500 - { error: string } — rollback transaction failure
|
||||
*/
|
||||
router.post('/rollback/:uploadId', requireGroup('Admin'), async (req, res) => {
|
||||
const uploadId = parseInt(req.params.uploadId, 10);
|
||||
if (isNaN(uploadId)) return res.status(400).json({ error: 'Invalid upload ID' });
|
||||
|
||||
try {
|
||||
const { rows: uploadRows } = await pool.query(
|
||||
`SELECT id, filename, report_date FROM supplemental_uploads WHERE id = $1`, [uploadId]
|
||||
);
|
||||
if (!uploadRows[0]) return res.status(404).json({ error: 'Upload not found' });
|
||||
|
||||
const { rows: latestRows } = await pool.query(
|
||||
`SELECT id FROM supplemental_uploads ORDER BY id DESC LIMIT 1`
|
||||
);
|
||||
if (latestRows[0].id !== uploadId) {
|
||||
return res.status(400).json({ error: 'Only the most recent upload can be rolled back', latest_upload_id: latestRows[0].id });
|
||||
}
|
||||
|
||||
const client = await pool.connect();
|
||||
try {
|
||||
await client.query('BEGIN');
|
||||
|
||||
// Delete items that were first introduced by this upload
|
||||
const deleteNew = await client.query(
|
||||
`DELETE FROM supplemental_items WHERE first_seen_upload_id = $1 AND upload_id = $1`, [uploadId]
|
||||
);
|
||||
|
||||
// Reactivate items that were resolved by this upload
|
||||
const reactivate = await client.query(
|
||||
`UPDATE supplemental_items SET status = 'active', resolved_upload_id = NULL WHERE resolved_upload_id = $1`, [uploadId]
|
||||
);
|
||||
|
||||
// For recurring items (seen_count was incremented), point them back to previous upload
|
||||
const { rows: prevRows } = await pool.query(
|
||||
`SELECT id FROM supplemental_uploads WHERE id < $1 ORDER BY id DESC LIMIT 1`, [uploadId]
|
||||
);
|
||||
if (prevRows[0]) {
|
||||
await client.query(
|
||||
`UPDATE supplemental_items SET upload_id = $1, seen_count = GREATEST(seen_count - 1, 1) WHERE upload_id = $2 AND first_seen_upload_id != $2`,
|
||||
[prevRows[0].id, uploadId]
|
||||
);
|
||||
}
|
||||
|
||||
// Delete upload record
|
||||
await client.query(`DELETE FROM supplemental_uploads WHERE id = $1`, [uploadId]);
|
||||
|
||||
await client.query('COMMIT');
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id, username: req.user.username,
|
||||
action: 'supplemental_upload_rollback', entityType: 'supplemental_upload',
|
||||
entityId: String(uploadId),
|
||||
details: { filename: uploadRows[0].filename, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.json({
|
||||
message: `Rolled back supplemental upload "${uploadRows[0].filename}"`,
|
||||
rolled_back: { upload_id: uploadId, filename: uploadRows[0].filename, items_deleted: deleteNew.rowCount, items_reactivated: reactivate.rowCount },
|
||||
});
|
||||
} catch (err) {
|
||||
await client.query('ROLLBACK');
|
||||
throw err;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] Rollback error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to rollback: ' + err.message });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* GET /items/:hostname
|
||||
*
|
||||
* Returns detailed info for a supplemental device including all sheets it
|
||||
* appears on (active and resolved), along with any associated notes.
|
||||
*
|
||||
* @param {string} hostname - Device hostname to look up
|
||||
* @response 200 - { hostname, ip_address, ipv6_address, device_function, device_type, model, vendor, equip_inst_id, responsible_team, team, findings: [{ sheet_name, status, seen_count, first_seen, last_seen, resolution_date, remediation_plan, extra_json, application_id, application_ref_id }], notes: [{ id, sheet_name, note, created_at, created_by }] }
|
||||
* @response 400 - { error: string } — invalid or missing hostname
|
||||
* @response 404 - { error: string } — device not found in any sheet
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.get('/items/:hostname', async (req, res) => {
|
||||
const hostname = req.params.hostname;
|
||||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||||
|
||||
try {
|
||||
const { rows } = await pool.query(
|
||||
`SELECT si.sheet_name, si.hostname, si.ip_address, si.ipv6_address, si.device_function,
|
||||
si.device_type, si.model, si.vendor, si.equip_inst_id, si.responsible_team,
|
||||
si.team, si.application_id, si.application_ref_id, si.extra_json,
|
||||
si.status, si.seen_count, si.resolution_date, si.remediation_plan,
|
||||
fu.report_date AS first_seen, lu.report_date AS last_seen
|
||||
FROM supplemental_items si
|
||||
LEFT JOIN supplemental_uploads fu ON si.first_seen_upload_id = fu.id
|
||||
LEFT JOIN supplemental_uploads lu ON si.upload_id = lu.id
|
||||
WHERE si.hostname = $1
|
||||
ORDER BY si.status ASC, si.sheet_name ASC`,
|
||||
[hostname]
|
||||
);
|
||||
|
||||
if (rows.length === 0) return res.status(404).json({ error: 'Device not found' });
|
||||
|
||||
const { rows: noteRows } = await pool.query(
|
||||
`SELECT id, sheet_name, note, created_at, created_by FROM supplemental_notes WHERE hostname = $1 ORDER BY created_at DESC`,
|
||||
[hostname]
|
||||
).catch(() => ({ rows: [] }));
|
||||
|
||||
const identity = rows[0];
|
||||
res.json({
|
||||
hostname: identity.hostname,
|
||||
ip_address: identity.ip_address,
|
||||
ipv6_address: identity.ipv6_address,
|
||||
device_function: identity.device_function,
|
||||
device_type: identity.device_type,
|
||||
model: identity.model,
|
||||
vendor: identity.vendor,
|
||||
equip_inst_id: identity.equip_inst_id,
|
||||
responsible_team: identity.responsible_team,
|
||||
team: identity.team,
|
||||
findings: rows.map(r => ({
|
||||
sheet_name: r.sheet_name,
|
||||
status: r.status,
|
||||
seen_count: r.seen_count,
|
||||
first_seen: r.first_seen,
|
||||
last_seen: r.last_seen,
|
||||
resolution_date: r.resolution_date,
|
||||
remediation_plan: r.remediation_plan,
|
||||
extra_json: r.extra_json,
|
||||
application_id: r.application_id,
|
||||
application_ref_id: r.application_ref_id,
|
||||
})),
|
||||
notes: noteRows,
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] GET /items/:hostname error:', err.message);
|
||||
res.status(500).json({ error: 'Database error' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* PATCH /items/:hostname/metadata
|
||||
*
|
||||
* Updates resolution_date and/or remediation_plan for active supplemental
|
||||
* items. Optionally scoped to specific sheet(s) via sheet_name or sheet_names.
|
||||
*
|
||||
* @param {string} hostname - Device hostname to update
|
||||
* @body {string} [resolution_date] - Target resolution date (YYYY-MM-DD or null to clear)
|
||||
* @body {string} [remediation_plan] - Free-text remediation plan (or null to clear)
|
||||
* @body {string} [sheet_name] - Scope update to a single sheet
|
||||
* @body {string[]} [sheet_names] - Scope update to multiple sheets (overrides sheet_name)
|
||||
* @response 200 - { updated: number }
|
||||
* @response 400 - { error: string } — invalid hostname or no fields provided
|
||||
* @response 404 - { error: string } — no active items found for this device
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.patch('/items/:hostname/metadata', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const hostname = req.params.hostname;
|
||||
if (!hostname || hostname.length > 300) return res.status(400).json({ error: 'Invalid hostname' });
|
||||
|
||||
const { resolution_date, remediation_plan, sheet_name, sheet_names } = req.body;
|
||||
|
||||
// Resolve sheet scoping
|
||||
let targetSheets = null;
|
||||
if (sheet_names && Array.isArray(sheet_names) && sheet_names.length > 0) {
|
||||
targetSheets = sheet_names;
|
||||
} else if (sheet_name && typeof sheet_name === 'string') {
|
||||
targetSheets = [sheet_name];
|
||||
}
|
||||
|
||||
const setClauses = [];
|
||||
const values = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
if (resolution_date !== undefined) {
|
||||
setClauses.push(`resolution_date = $${paramIdx++}`);
|
||||
values.push(resolution_date || null);
|
||||
}
|
||||
if (remediation_plan !== undefined) {
|
||||
setClauses.push(`remediation_plan = $${paramIdx++}`);
|
||||
values.push(remediation_plan || null);
|
||||
}
|
||||
if (setClauses.length === 0) return res.status(400).json({ error: 'No fields to update' });
|
||||
|
||||
try {
|
||||
let whereClause = `hostname = $${paramIdx++} AND status = 'active'`;
|
||||
values.push(hostname);
|
||||
|
||||
if (targetSheets) {
|
||||
whereClause += ` AND sheet_name = ANY($${paramIdx++})`;
|
||||
values.push(targetSheets);
|
||||
}
|
||||
|
||||
const result = await pool.query(
|
||||
`UPDATE supplemental_items SET ${setClauses.join(', ')} WHERE ${whereClause}`,
|
||||
values
|
||||
);
|
||||
|
||||
if (result.rowCount === 0) return res.status(404).json({ error: 'No active items found for this device' });
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id, username: req.user.username,
|
||||
action: 'supplemental_metadata_update', entityType: 'supplemental_item',
|
||||
entityId: hostname,
|
||||
details: { resolution_date, remediation_plan, sheet_names: targetSheets },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.json({ updated: result.rowCount });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] PATCH /items/:hostname/metadata error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to update metadata' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* POST /notes
|
||||
*
|
||||
* Add a note to a supplemental device, optionally scoped to a specific sheet.
|
||||
* Notes are limited to 1000 characters.
|
||||
*
|
||||
* @body {string} hostname - Device hostname to attach the note to (required, max 300 chars)
|
||||
* @body {string} [sheet_name] - Optional sheet name to scope the note to
|
||||
* @body {string} note - Note text (required, max 1000 chars after trim)
|
||||
* @response 201 - { note: { id, hostname, sheet_name, note, created_at } }
|
||||
* @response 400 - { error: string } — invalid hostname or empty note
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.post('/notes', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const { hostname, sheet_name, note } = req.body;
|
||||
if (!hostname || typeof hostname !== 'string' || hostname.length > 300) {
|
||||
return res.status(400).json({ error: 'Invalid hostname' });
|
||||
}
|
||||
const noteText = String(note || '').trim().slice(0, 1000);
|
||||
if (!noteText) return res.status(400).json({ error: 'Note cannot be empty' });
|
||||
|
||||
try {
|
||||
const { rows } = await pool.query(
|
||||
`INSERT INTO supplemental_notes (hostname, sheet_name, note, created_by, created_at)
|
||||
VALUES ($1, $2, $3, $4, NOW()) RETURNING id, hostname, sheet_name, note, created_at`,
|
||||
[hostname, sheet_name || null, noteText, req.user?.id || null]
|
||||
);
|
||||
|
||||
logAudit({
|
||||
userId: req.user.id, username: req.user.username,
|
||||
action: 'supplemental_note_add', entityType: 'supplemental_note',
|
||||
entityId: String(rows[0].id),
|
||||
details: { hostname, sheet_name },
|
||||
ipAddress: req.ip,
|
||||
});
|
||||
|
||||
res.status(201).json({ note: rows[0] });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] POST /notes error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to save note' });
|
||||
}
|
||||
});
|
||||
|
||||
/**
|
||||
* DELETE /notes/:id
|
||||
*
|
||||
* Delete a supplemental note. Authors can delete their own notes;
|
||||
* Admin users can delete any note.
|
||||
*
|
||||
* @param {string} id - Note ID to delete
|
||||
* @response 200 - { deleted: 1 }
|
||||
* @response 400 - { error: string } — invalid note ID
|
||||
* @response 403 - { error: string } — non-author, non-Admin attempting deletion
|
||||
* @response 404 - { error: string } — note not found
|
||||
* @response 500 - { error: string } — database error
|
||||
*/
|
||||
router.delete('/notes/:id', requireGroup('Admin', 'Standard_User'), async (req, res) => {
|
||||
const noteId = parseInt(req.params.id, 10);
|
||||
if (isNaN(noteId)) return res.status(400).json({ error: 'Invalid note ID' });
|
||||
|
||||
try {
|
||||
const { rows } = await pool.query(`SELECT id, created_by FROM supplemental_notes WHERE id = $1`, [noteId]);
|
||||
if (!rows[0]) return res.status(404).json({ error: 'Note not found' });
|
||||
|
||||
const isAuthor = req.user && String(req.user.id) === String(rows[0].created_by);
|
||||
const isAdminUser = req.user && req.user.group === 'Admin';
|
||||
if (!isAuthor && !isAdminUser) return res.status(403).json({ error: 'You can only delete your own notes' });
|
||||
|
||||
await pool.query(`DELETE FROM supplemental_notes WHERE id = $1`, [noteId]);
|
||||
res.json({ deleted: 1 });
|
||||
} catch (err) {
|
||||
console.error('[Supplemental] DELETE /notes error:', err.message);
|
||||
res.status(500).json({ error: 'Failed to delete note' });
|
||||
}
|
||||
});
|
||||
|
||||
return router;
|
||||
}
|
||||
|
||||
module.exports = { createSupplementalRouter };
|
||||
@@ -1,29 +1,65 @@
|
||||
{
|
||||
"metric_categories": {
|
||||
"1.1.1": "Logging & Monitoring",
|
||||
"1.1.3": "Logging & Monitoring",
|
||||
"1.4.1": "Logging & Monitoring",
|
||||
"1.1.1": "Asset Data Quality",
|
||||
"1.1.2": "Asset Data Quality",
|
||||
"1.1.3": "Disaster Recovery",
|
||||
"1.2.2": "Logging & Monitoring",
|
||||
"1.2.4": "End-of-Life OS",
|
||||
"1.2.5": "Endpoint Protection",
|
||||
"1.2.5All": "Endpoint Protection",
|
||||
"1.4.1": "Disaster Recovery",
|
||||
"1.4.2": "Disaster Recovery",
|
||||
"1.5.1B": "Application Security",
|
||||
"1.5.2": "Application Security",
|
||||
"2.3.3i": "Vulnerability Management",
|
||||
"2.3.4i": "Vulnerability Management",
|
||||
"2.3.5i": "Vulnerability Management",
|
||||
"2.3.6i": "Vulnerability Management",
|
||||
"2.3.7i": "Vulnerability Management",
|
||||
"2.3.8i": "Vulnerability Management",
|
||||
"2.3.9i": "Vulnerability Management",
|
||||
"5.2.3": "Access & MFA",
|
||||
"5.2.4": "Access & MFA",
|
||||
"5.2.5": "Access & MFA",
|
||||
"5.2.6": "Access & MFA",
|
||||
"5.2.7": "Access & MFA",
|
||||
"5.2.8": "Access & MFA",
|
||||
"5.3.4": "Endpoint Protection",
|
||||
"5.4.2": "Endpoint Protection",
|
||||
"5.4.3": "Endpoint Protection",
|
||||
"5.4.6": "Vulnerability Management",
|
||||
"5.4.6i": "Vulnerability Management",
|
||||
"5.5.2": "End-of-Life OS",
|
||||
"5.5.4": "Vulnerability Management",
|
||||
"5.5.4i": "Vulnerability Management",
|
||||
"5.5.5": "Decommissioned Assets",
|
||||
"5.6.2A": "Vulnerability Management",
|
||||
"5.6.3": "Asset Data Quality",
|
||||
"5.6.3B": "Access & MFA",
|
||||
"5.7.1": "Logging & Monitoring",
|
||||
"5.8.1": "Application Security",
|
||||
"7.1.1": "Logging & Monitoring",
|
||||
"7.1.4": "Logging & Monitoring",
|
||||
"7.1.4": "Asset Data Quality",
|
||||
"7.6.13": "Disaster Recovery",
|
||||
"7.6.15": "Disaster Recovery",
|
||||
"7.6.16": "Disaster Recovery",
|
||||
"Missing_AppID": "Asset Data Quality",
|
||||
"Missing_DF": "Asset Data Quality",
|
||||
"Missing_EOS": "End-of-Life OS",
|
||||
"Missing_OS": "Asset Data Quality",
|
||||
"5.5.2": "Other"
|
||||
"Vulns_Aging": "Vulnerability Management"
|
||||
},
|
||||
"valid_categories": [
|
||||
"Vulnerability Management",
|
||||
"Access & MFA",
|
||||
"Logging & Monitoring",
|
||||
"End-of-Life OS",
|
||||
"Decommissioned Assets",
|
||||
"Asset Data Quality",
|
||||
"Application Security",
|
||||
"Disaster Recovery",
|
||||
"Endpoint Protection"
|
||||
],
|
||||
"core_cols": [
|
||||
"Preferred - Hostname",
|
||||
"GRANITE - IPv4_Address",
|
||||
|
||||
191
backend/scripts/parse_supplemental_xlsx.py
Normal file
191
backend/scripts/parse_supplemental_xlsx.py
Normal file
@@ -0,0 +1,191 @@
|
||||
#!/usr/bin/env python3
|
||||
"""
|
||||
Parse NTS_AEO supplemental (Granite hygiene) xlsx and write JSON to stdout.
|
||||
Usage: python3 parse_supplemental_xlsx.py <path_to_xlsx>
|
||||
|
||||
Output:
|
||||
{
|
||||
"sheets": { "Missing_AppID": [...], "Missing_OS_Granite": [...], ... },
|
||||
"report_date": "YYYY-MM-DD" | null,
|
||||
"total": int
|
||||
}
|
||||
"""
|
||||
import sys
|
||||
import os
|
||||
import json
|
||||
import re
|
||||
import pandas as pd
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
# Recognized sheet names (matched case-insensitively)
|
||||
RECOGNIZED_SHEETS = {
|
||||
'missing_appid': 'Missing_AppID',
|
||||
'missing_df': 'Missing_DF',
|
||||
'missing_os_granite': 'Missing_OS_Granite',
|
||||
'retired_appid': 'Retired_AppID',
|
||||
}
|
||||
|
||||
# Core column mappings: output_field -> list of possible source column names (lowercase)
|
||||
CORE_COLUMNS = {
|
||||
'hostname': ['preferred - hostname', 'granite - hostname'],
|
||||
'ip_address': ['granite - ipv4_address'],
|
||||
'ipv6_address': ['granite - ipv6_address'],
|
||||
'device_function': ['granite - device_function'],
|
||||
'device_type': ['granite - type'],
|
||||
'model': ['granite - model'],
|
||||
'vendor': ['granite - vendor'],
|
||||
'equip_inst_id': ['granite - equip_inst_id'],
|
||||
'responsible_team': ['granite - responsible_team'],
|
||||
'team': ['team'],
|
||||
'application_id': ['application_id'],
|
||||
'application_ref_id': ['application_reference_id'],
|
||||
}
|
||||
|
||||
# Columns to exclude from extra_json (already extracted as core fields)
|
||||
EXCLUDE_FROM_EXTRA = set()
|
||||
for candidates in CORE_COLUMNS.values():
|
||||
for c in candidates:
|
||||
EXCLUDE_FROM_EXTRA.add(c)
|
||||
# Also exclude vertical — it's metadata not per-device data
|
||||
EXCLUDE_FROM_EXTRA.add('vertical')
|
||||
|
||||
|
||||
def safe_str(val):
|
||||
"""Convert to string, return empty string for NaN/None."""
|
||||
if val is None:
|
||||
return ''
|
||||
s = str(val).strip()
|
||||
return '' if s.lower() == 'nan' else s
|
||||
|
||||
|
||||
def find_column(df_columns_lower, candidates):
|
||||
"""Find the first matching column from candidates list."""
|
||||
for candidate in candidates:
|
||||
if candidate in df_columns_lower:
|
||||
return df_columns_lower[candidate]
|
||||
return None
|
||||
|
||||
|
||||
def parse_sheet(xl, sheet_name):
|
||||
"""Parse a single sheet into a list of item dicts."""
|
||||
try:
|
||||
df = pd.read_excel(xl, sheet_name=sheet_name, header=0)
|
||||
except Exception:
|
||||
return []
|
||||
|
||||
if df.empty:
|
||||
return []
|
||||
|
||||
# Normalize column names for matching
|
||||
df.columns = [str(c).strip() for c in df.columns]
|
||||
# Build lowercase -> original mapping
|
||||
col_lower_map = {c.lower(): c for c in df.columns}
|
||||
|
||||
# Resolve core column positions
|
||||
resolved = {}
|
||||
for field, candidates in CORE_COLUMNS.items():
|
||||
orig_col = find_column(col_lower_map, candidates)
|
||||
resolved[field] = orig_col # None if not found
|
||||
|
||||
# Determine which original columns are "extra"
|
||||
core_orig_cols = set(c for c in resolved.values() if c is not None)
|
||||
# Also exclude vertical by checking lowercase
|
||||
extra_cols = [
|
||||
c for c in df.columns
|
||||
if c.lower() not in EXCLUDE_FROM_EXTRA and c not in core_orig_cols
|
||||
]
|
||||
|
||||
items = []
|
||||
for _, row in df.iterrows():
|
||||
# Extract hostname — required field
|
||||
hostname_col = resolved.get('hostname')
|
||||
if not hostname_col:
|
||||
continue
|
||||
hostname = safe_str(row.get(hostname_col, ''))
|
||||
if not hostname:
|
||||
continue
|
||||
|
||||
# Extract all core fields
|
||||
item = {'hostname': hostname}
|
||||
for field, orig_col in resolved.items():
|
||||
if field == 'hostname':
|
||||
continue
|
||||
if orig_col:
|
||||
val = row.get(orig_col)
|
||||
if pd.isna(val) if not isinstance(val, str) else False:
|
||||
item[field] = None
|
||||
else:
|
||||
s = safe_str(val)
|
||||
item[field] = s if s else None
|
||||
else:
|
||||
item[field] = None
|
||||
|
||||
# Build extra_json from remaining columns
|
||||
extra = {}
|
||||
for col in extra_cols:
|
||||
val = row.get(col)
|
||||
if pd.isna(val) if not isinstance(val, str) else False:
|
||||
continue
|
||||
s = safe_str(val)
|
||||
if s:
|
||||
extra[col] = val.isoformat() if hasattr(val, 'isoformat') else s
|
||||
item['extra_json'] = extra
|
||||
|
||||
items.append(item)
|
||||
|
||||
return items
|
||||
|
||||
|
||||
def extract_report_date(filepath):
|
||||
"""Try to pull YYYY-MM-DD from the filename."""
|
||||
stem = Path(filepath).stem
|
||||
m = re.search(r'(\d{4})_(\d{2})_(\d{2})', stem)
|
||||
if m:
|
||||
return f"{m.group(1)}-{m.group(2)}-{m.group(3)}"
|
||||
return None
|
||||
|
||||
|
||||
def main():
|
||||
if len(sys.argv) < 2:
|
||||
print(json.dumps({'error': 'No file path provided'}))
|
||||
sys.exit(1)
|
||||
|
||||
filepath = sys.argv[1]
|
||||
|
||||
try:
|
||||
xl = pd.ExcelFile(filepath)
|
||||
except Exception as e:
|
||||
print(json.dumps({'error': f'Cannot open file: {str(e)}'}))
|
||||
sys.exit(1)
|
||||
|
||||
# Match sheet names case-insensitively
|
||||
sheets_output = {}
|
||||
matched_count = 0
|
||||
|
||||
for actual_name in xl.sheet_names:
|
||||
normalized = actual_name.lower().strip()
|
||||
if normalized in RECOGNIZED_SHEETS:
|
||||
canonical_name = RECOGNIZED_SHEETS[normalized]
|
||||
items = parse_sheet(xl, actual_name)
|
||||
sheets_output[canonical_name] = items
|
||||
matched_count += 1
|
||||
|
||||
if matched_count == 0:
|
||||
print(json.dumps({
|
||||
'error': f'No recognized sheets found. Expected one of: {list(RECOGNIZED_SHEETS.values())}. '
|
||||
f'Found: {xl.sheet_names}'
|
||||
}))
|
||||
sys.exit(1)
|
||||
|
||||
total = sum(len(items) for items in sheets_output.values())
|
||||
|
||||
print(json.dumps({
|
||||
'sheets': sheets_output,
|
||||
'report_date': extract_report_date(filepath),
|
||||
'total': total,
|
||||
}))
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
407
backend/scripts/repair_upload_91_sweep.js
Normal file
407
backend/scripts/repair_upload_91_sweep.js
Normal file
@@ -0,0 +1,407 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Repair the cross-vertical resolve sweep caused by upload 91.
|
||||
*
|
||||
* Spec: .kiro/specs/compliance-cross-vertical-resolve-sweep/
|
||||
*
|
||||
* Upload 91 (NTS_AEO_2026_07_20.xlsx) was committed through the legacy /commit
|
||||
* route, which resolved items using a globally-scoped active set. It marked
|
||||
* 123,444 items resolved, of which only 899 belonged to NTS_AEO. The remaining
|
||||
* 122,545 belong to twelve other verticals and are misreported as compliance
|
||||
* wins.
|
||||
*
|
||||
* This is a one-off repair, not a migration — it targets a specific historical
|
||||
* upload rather than the schema, so it must NOT be added to migrations/run-all.js.
|
||||
*
|
||||
* Usage:
|
||||
* node scripts/repair_upload_91_sweep.js --dry-run # report only, rolls back
|
||||
* node scripts/repair_upload_91_sweep.js --execute # commit the repair
|
||||
*
|
||||
* The dry run performs every write inside a transaction and then rolls back, so
|
||||
* the reported counts are exact rather than estimated.
|
||||
*
|
||||
* Idempotent: every step is guarded, so a second --execute run is a no-op.
|
||||
*/
|
||||
|
||||
const pool = require('../db');
|
||||
|
||||
const UPLOAD_ID = 91;
|
||||
const OWN_VERTICAL = 'NTS_AEO';
|
||||
// Verticals whose rows upload 91 captured via its UPDATE branch. Granite's
|
||||
// RESPONSIBLE_TEAM places these assets under NTS-AEO-*, so the spreadsheet's
|
||||
// ownership claim is authoritative (see design.md, repair steps 3 and 4).
|
||||
const CAPTURED_VERTICALS = ['SDIT_CSD', 'NTS_NEO'];
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const DRY_RUN = args.includes('--dry-run');
|
||||
const EXECUTE = args.includes('--execute');
|
||||
|
||||
// Optional. Path to the parsed preview JSON for upload 91. When supplied, the
|
||||
// authoritative `team` for the captured rows is read from it, so those rows
|
||||
// become visible under the correct team immediately rather than waiting for the
|
||||
// next upload to refresh them. Without it, team is left alone and self-corrects
|
||||
// on the next cycle now that persistUpload refreshes it.
|
||||
const previewFlag = args.indexOf('--from-preview');
|
||||
const PREVIEW_PATH = previewFlag !== -1 ? args[previewFlag + 1] : null;
|
||||
|
||||
if (DRY_RUN === EXECUTE) {
|
||||
console.error('Specify exactly one of --dry-run or --execute');
|
||||
console.error('Optional: --from-preview <path-to-preview.json> to also correct stale team values');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Load hostname|||metric_id -> team from the spreadsheet, which is the authority.
|
||||
function loadSpreadsheetTeams() {
|
||||
if (!PREVIEW_PATH) return null;
|
||||
const fs = require('fs');
|
||||
if (!fs.existsSync(PREVIEW_PATH)) {
|
||||
console.error(`Preview file not found: ${PREVIEW_PATH}`);
|
||||
process.exit(1);
|
||||
}
|
||||
const parsed = JSON.parse(fs.readFileSync(PREVIEW_PATH, 'utf8'));
|
||||
const map = new Map();
|
||||
for (const i of parsed.items || []) {
|
||||
if (i.team) map.set(`${i.hostname}|||${i.metric_id}`, i.team);
|
||||
}
|
||||
return map;
|
||||
}
|
||||
|
||||
function heading(text) {
|
||||
console.log(`\n${text}`);
|
||||
console.log('-'.repeat(text.length));
|
||||
}
|
||||
|
||||
async function reportState(client, label) {
|
||||
heading(label);
|
||||
|
||||
const { rows: sweep } = await client.query(
|
||||
`SELECT COALESCE(vertical, '(null)') AS vertical,
|
||||
COUNT(*) FILTER (WHERE status = 'resolved') AS resolved,
|
||||
COUNT(*) FILTER (WHERE status = 'active') AS active
|
||||
FROM compliance_items
|
||||
WHERE resolved_upload_id = $1
|
||||
GROUP BY vertical ORDER BY 2 DESC`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
if (sweep.length === 0) {
|
||||
console.log(` no rows carry resolved_upload_id = ${UPLOAD_ID}`);
|
||||
} else {
|
||||
console.log(' rows attributed to the sweep:');
|
||||
for (const r of sweep) {
|
||||
console.log(` ${r.vertical.padEnd(20)} resolved=${String(r.resolved).padStart(7)} active=${r.active}`);
|
||||
}
|
||||
}
|
||||
|
||||
const { rows: labels } = await client.query(
|
||||
`SELECT COALESCE(vertical, '(null)') AS vertical, COUNT(*) AS rows
|
||||
FROM compliance_items WHERE upload_id = $1 GROUP BY vertical ORDER BY 2 DESC`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
console.log(` rows whose upload_id = ${UPLOAD_ID}:`);
|
||||
for (const r of labels) {
|
||||
console.log(` ${r.vertical.padEnd(20)} ${r.rows}`);
|
||||
}
|
||||
|
||||
const { rows: [u] } = await client.query(
|
||||
`SELECT resolved_count, new_count, recurring_count, vertical
|
||||
FROM compliance_uploads WHERE id = $1`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
console.log(` upload record: vertical=${u.vertical ?? '(null)'} resolved_count=${u.resolved_count} `
|
||||
+ `new_count=${u.new_count} recurring_count=${u.recurring_count}`);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Invariant checks — run before and after so the repair can be shown safe
|
||||
// ---------------------------------------------------------------------------
|
||||
async function checkInvariants(client, label) {
|
||||
heading(label);
|
||||
|
||||
// Informational, NOT a pass/fail gate. Upload 91's activeMap was keyed on
|
||||
// hostname|||metric_id, so where a key had several active rows only one
|
||||
// entered the map and only that one was swept. The sweep therefore
|
||||
// deduplicated those keys as a side effect. Reactivating restores the
|
||||
// duplication that legitimately existed beforehand, so this count RISING is
|
||||
// expected and correct. The table already tolerates duplicate identity rows
|
||||
// — every read path applies DISTINCT ON.
|
||||
const { rows: [dupes] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n FROM (
|
||||
SELECT hostname, metric_id FROM compliance_items WHERE status = 'active'
|
||||
GROUP BY hostname, metric_id HAVING COUNT(*) > 1
|
||||
) x`
|
||||
);
|
||||
console.log(` (hostname, metric_id) pairs with more than one ACTIVE row: ${dupes.n} [informational]`);
|
||||
|
||||
const { rows: [cross] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n
|
||||
FROM compliance_items ci
|
||||
JOIN compliance_uploads u ON ci.resolved_upload_id = u.id
|
||||
WHERE ci.vertical IS DISTINCT FROM u.vertical`
|
||||
);
|
||||
console.log(` rows resolved by an upload from a DIFFERENT vertical: ${cross.n}`);
|
||||
|
||||
const { rows: [nulls] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n FROM compliance_items WHERE vertical IS NULL`
|
||||
);
|
||||
console.log(` compliance_items rows with vertical IS NULL: ${nulls.n}`);
|
||||
|
||||
const { rows: [steam] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n FROM compliance_items
|
||||
WHERE team = 'STEAM' AND metric_id = 'Missing_AppID' AND status = 'active'
|
||||
AND (vertical IS NULL OR vertical = 'NTS_AEO')`
|
||||
);
|
||||
console.log(` STEAM Missing_AppID active and visible to /items: ${steam.n} (spreadsheet says 160)`);
|
||||
|
||||
return { dupes: dupes.n, cross: cross.n, nulls: nulls.n, steam: steam.n };
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// The repair
|
||||
// ---------------------------------------------------------------------------
|
||||
async function runRepair(client) {
|
||||
heading('Applying repair steps');
|
||||
|
||||
// Step 1 — reactivate findings swept from other verticals. Exactly
|
||||
// invertible: the sweep mutated only status and resolved_upload_id.
|
||||
const step1 = await client.query(
|
||||
`UPDATE compliance_items
|
||||
SET status = 'active', resolved_upload_id = NULL
|
||||
WHERE resolved_upload_id = $1
|
||||
AND vertical IS NOT NULL
|
||||
AND vertical <> $2`,
|
||||
[UPLOAD_ID, OWN_VERTICAL]
|
||||
);
|
||||
console.log(` 1. reactivated foreign-vertical findings ${step1.rowCount}`);
|
||||
|
||||
// Step 2 is intentionally a no-op: rows with vertical = NTS_AEO were
|
||||
// legitimately resolved, because upload 91 was an NTS_AEO spreadsheet.
|
||||
const { rows: [keep] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n FROM compliance_items
|
||||
WHERE resolved_upload_id = $1 AND vertical = $2`,
|
||||
[UPLOAD_ID, OWN_VERTICAL]
|
||||
);
|
||||
console.log(` 2. left legitimately resolved (NTS_AEO, untouched) ${keep.n}`);
|
||||
|
||||
// Step 3 — label the rows upload 91 inserted. persistUpload omitted the
|
||||
// vertical column entirely, so these landed as NULL.
|
||||
const { rows: step3ids } = await client.query(
|
||||
`SELECT id FROM compliance_items WHERE upload_id = $1 AND vertical IS NULL ORDER BY id`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
const step3 = await client.query(
|
||||
`UPDATE compliance_items SET vertical = $2 WHERE upload_id = $1 AND vertical IS NULL`,
|
||||
[UPLOAD_ID, OWN_VERTICAL]
|
||||
);
|
||||
console.log(` 3. labelled inserted rows as ${OWN_VERTICAL} ${step3.rowCount}`);
|
||||
|
||||
// Step 4 — the rows upload 91 captured via its UPDATE branch while they
|
||||
// still carried a foreign vertical. These are the nine stranded
|
||||
// CLIENTSIDEVM Missing_AppID rows plus one NTS_NEO row. Recorded
|
||||
// individually so the ownership call can be revisited.
|
||||
const { rows: step4rows } = await client.query(
|
||||
`SELECT id, hostname, metric_id, team, vertical FROM compliance_items
|
||||
WHERE upload_id = $1 AND status = 'active' AND vertical = ANY($2)
|
||||
ORDER BY vertical, hostname`,
|
||||
[UPLOAD_ID, CAPTURED_VERTICALS]
|
||||
);
|
||||
const step4 = await client.query(
|
||||
`UPDATE compliance_items SET vertical = $3
|
||||
WHERE upload_id = $1 AND status = 'active' AND vertical = ANY($2)`,
|
||||
[UPLOAD_ID, CAPTURED_VERTICALS, OWN_VERTICAL]
|
||||
);
|
||||
console.log(` 4. reassigned captured rows to ${OWN_VERTICAL} ${step4.rowCount}`);
|
||||
|
||||
// 4b — correct stale team on those rows from the spreadsheet, if provided.
|
||||
// Upload 91's UPDATE branch did not refresh team, so these rows kept the
|
||||
// owner they had when first inserted months earlier.
|
||||
const teams = loadSpreadsheetTeams();
|
||||
let teamFixes = 0;
|
||||
for (const r of step4rows) {
|
||||
let note = `team=${r.team}`;
|
||||
if (teams) {
|
||||
const authoritative = teams.get(`${r.hostname}|||${r.metric_id}`);
|
||||
if (authoritative && authoritative !== r.team) {
|
||||
await client.query(`UPDATE compliance_items SET team = $2 WHERE id = $1`,
|
||||
[r.id, authoritative]);
|
||||
note = `team ${r.team} -> ${authoritative}`;
|
||||
teamFixes++;
|
||||
}
|
||||
}
|
||||
console.log(` id=${String(r.id).padStart(8)} ${r.vertical} -> ${OWN_VERTICAL} ${r.hostname} (${r.metric_id}, ${note})`);
|
||||
}
|
||||
if (teams) {
|
||||
console.log(` 4b. corrected stale team values ${teamFixes}`);
|
||||
} else {
|
||||
console.log(' 4b. team correction SKIPPED (no --from-preview given)');
|
||||
console.log(' stale team values will self-correct on the next upload');
|
||||
}
|
||||
|
||||
// Step 5 — correct the upload's recorded resolved_count, and give the
|
||||
// upload row the vertical it should always have carried.
|
||||
const { rows: [actual] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n FROM compliance_items WHERE resolved_upload_id = $1`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
await client.query(
|
||||
`UPDATE compliance_uploads SET resolved_count = $2, vertical = $3 WHERE id = $1`,
|
||||
[UPLOAD_ID, actual.n, OWN_VERTICAL]
|
||||
);
|
||||
console.log(` 5. corrected upload resolved_count to ${actual.n}`);
|
||||
|
||||
// Step 6 — recompute the snapshot for the affected month. Upload 91 is now
|
||||
// an NTS_AEO upload, so the aggregation is scoped to that vertical rather
|
||||
// than to the NULL vertical it previously matched.
|
||||
const { rows: [dates] } = await client.query(
|
||||
`SELECT report_date, to_char(uploaded_at, 'YYYY-MM') AS upload_month
|
||||
FROM compliance_uploads WHERE id = $1`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
const month = dates.upload_month;
|
||||
const { rows: before } = await client.query(
|
||||
`SELECT vertical, total_devices, compliant, non_compliant, compliance_pct
|
||||
FROM compliance_snapshots WHERE snapshot_month = $1 ORDER BY vertical`,
|
||||
[month]
|
||||
);
|
||||
console.log(` 6. recomputing compliance_snapshots for ${month}`);
|
||||
for (const r of before) {
|
||||
console.log(` before ${r.vertical.padEnd(12)} total=${r.total_devices} compliant=${r.compliant} non_compliant=${r.non_compliant} pct=${r.compliance_pct}`);
|
||||
}
|
||||
|
||||
const { rows: stats } = await client.query(
|
||||
`WITH hostname_status AS (
|
||||
SELECT team, hostname, MIN(status) AS status
|
||||
FROM compliance_items
|
||||
WHERE team IS NOT NULL AND vertical IS NOT DISTINCT FROM $1
|
||||
GROUP BY team, hostname
|
||||
)
|
||||
SELECT team,
|
||||
COUNT(*)::int AS total_devices,
|
||||
COUNT(*) FILTER (WHERE status = 'resolved')::int AS compliant,
|
||||
COUNT(*) FILTER (WHERE status = 'active')::int AS non_compliant
|
||||
FROM hostname_status GROUP BY team ORDER BY team`,
|
||||
[OWN_VERTICAL]
|
||||
);
|
||||
for (const s of stats) {
|
||||
const pct = s.total_devices > 0
|
||||
? Math.round((s.compliant / s.total_devices) * 100 * 100) / 100
|
||||
: 0;
|
||||
await client.query(
|
||||
`INSERT INTO compliance_snapshots
|
||||
(snapshot_month, vertical, total_devices, compliant, non_compliant, compliance_pct)
|
||||
VALUES ($1, $2, $3, $4, $5, $6)
|
||||
ON CONFLICT (snapshot_month, vertical)
|
||||
DO UPDATE SET total_devices = $3, compliant = $4, non_compliant = $5, compliance_pct = $6`,
|
||||
[month, s.team, s.total_devices, s.compliant, s.non_compliant, pct]
|
||||
);
|
||||
console.log(` after ${s.team.padEnd(12)} total=${s.total_devices} compliant=${s.compliant} non_compliant=${s.non_compliant} pct=${pct}`);
|
||||
}
|
||||
|
||||
return {
|
||||
reactivated: step1.rowCount,
|
||||
keptResolved: keep.n,
|
||||
labelled: step3.rowCount,
|
||||
reassigned: step4.rowCount,
|
||||
resolvedCount: actual.n,
|
||||
step3ids: step3ids.map(r => r.id),
|
||||
step4rows,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Driver
|
||||
// ---------------------------------------------------------------------------
|
||||
async function main() {
|
||||
console.log('='.repeat(72));
|
||||
console.log(`Upload ${UPLOAD_ID} cross-vertical sweep repair — ${DRY_RUN ? 'DRY RUN' : 'EXECUTE'}`);
|
||||
console.log('='.repeat(72));
|
||||
|
||||
const client = await pool.connect();
|
||||
let failed = false;
|
||||
|
||||
try {
|
||||
const { rows: [upload] } = await client.query(
|
||||
`SELECT id, filename, report_date FROM compliance_uploads WHERE id = $1`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
if (!upload) {
|
||||
console.error(`\nUpload ${UPLOAD_ID} not found — nothing to repair.`);
|
||||
return;
|
||||
}
|
||||
console.log(`\nTarget: ${upload.filename} (report_date ${upload.report_date})`);
|
||||
|
||||
await client.query('BEGIN');
|
||||
|
||||
await reportState(client, 'BEFORE — current state');
|
||||
const invBefore = await checkInvariants(client, 'BEFORE — invariants');
|
||||
|
||||
const result = await runRepair(client);
|
||||
|
||||
await reportState(client, 'AFTER — resulting state');
|
||||
const invAfter = await checkInvariants(client, 'AFTER — invariants');
|
||||
|
||||
heading('Summary');
|
||||
console.log(` reactivated (foreign verticals) ${result.reactivated}`);
|
||||
console.log(` left resolved (NTS_AEO) ${result.keptResolved}`);
|
||||
console.log(` labelled NULL -> ${OWN_VERTICAL} ${result.labelled}`);
|
||||
console.log(` reassigned captured rows ${result.reassigned}`);
|
||||
console.log(` upload resolved_count ${result.resolvedCount}`);
|
||||
|
||||
console.log(` duplicate active pairs ${invBefore.dupes} -> ${invAfter.dupes} `
|
||||
+ `(+${invAfter.dupes - invBefore.dupes} restored, pre-existing condition)`);
|
||||
|
||||
// Guard: the reactivation must be an EXACT inversion of the sweep. The
|
||||
// sweep mutated only status and resolved_upload_id, so every row we
|
||||
// touched must have carried resolved_upload_id = 91 and must now be
|
||||
// active with a null resolved_upload_id. Nothing else may have changed.
|
||||
const { rows: [leaked] } = await client.query(
|
||||
`SELECT COUNT(*)::int AS n FROM compliance_items
|
||||
WHERE resolved_upload_id = $1 AND status = 'active'`,
|
||||
[UPLOAD_ID]
|
||||
);
|
||||
if (leaked.n !== 0) {
|
||||
console.error(`\nABORT: ${leaked.n} rows are active but still point at upload ${UPLOAD_ID}.`);
|
||||
failed = true;
|
||||
}
|
||||
const expectedReactivated = 122545;
|
||||
if (result.reactivated !== expectedReactivated && invBefore.cross === 123444) {
|
||||
console.error(`\nABORT: expected to reactivate ${expectedReactivated} rows, got ${result.reactivated}.`);
|
||||
failed = true;
|
||||
}
|
||||
if (invAfter.cross !== 0) {
|
||||
console.error(`\nABORT: ${invAfter.cross} rows still resolved by a foreign vertical's upload.`);
|
||||
failed = true;
|
||||
}
|
||||
if (invAfter.nulls !== 0) {
|
||||
console.error(`\nABORT: ${invAfter.nulls} rows still have vertical IS NULL.`);
|
||||
failed = true;
|
||||
}
|
||||
|
||||
if (failed) {
|
||||
await client.query('ROLLBACK');
|
||||
console.error('\nRolled back — no changes were written.');
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
if (DRY_RUN) {
|
||||
await client.query('ROLLBACK');
|
||||
console.log('\nDRY RUN complete — transaction rolled back, no changes written.');
|
||||
console.log('Re-run with --execute to apply.');
|
||||
} else {
|
||||
await client.query('COMMIT');
|
||||
console.log('\nCOMMITTED.');
|
||||
console.log('\nTo reverse step 1, re-resolve the reactivated rows. Steps 3 and 4 changed');
|
||||
console.log(`vertical on these ids: ${result.step3ids.length} inserted rows plus`);
|
||||
console.log(`${result.step4rows.length} captured rows (ids listed above).`);
|
||||
}
|
||||
} catch (err) {
|
||||
try { await client.query('ROLLBACK'); } catch { /* connection may be gone */ }
|
||||
console.error('\nRepair failed, transaction rolled back:', err.message);
|
||||
process.exitCode = 1;
|
||||
} finally {
|
||||
client.release();
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -34,6 +34,7 @@ const createIvantiArchiveRouter = require('./routes/ivantiArchive');
|
||||
const createIvantiFpWorkflowRouter = require('./routes/ivantiFpWorkflow');
|
||||
const { createComplianceRouter } = require('./routes/compliance');
|
||||
const { createVCLMultiVerticalRouter } = require('./routes/vclMultiVertical');
|
||||
const { createSupplementalRouter } = require('./routes/supplemental');
|
||||
const createAtlasRouter = require('./routes/atlas');
|
||||
const createJiraTicketsRouter = require('./routes/jiraTickets');
|
||||
const createCardApiRouter = require('./routes/cardApi');
|
||||
@@ -264,6 +265,9 @@ app.use('/api/ivanti/fp-workflow', createIvantiFpWorkflowRouter());
|
||||
// Must be mounted BEFORE the general compliance router since both share the /api/compliance prefix
|
||||
app.use('/api/compliance/vcl-multi', createVCLMultiVerticalRouter(complianceUpload));
|
||||
|
||||
// Supplemental (Granite hygiene) routes — separate ingest for supplemental workbook
|
||||
app.use('/api/compliance/supplemental', createSupplementalRouter(complianceUpload));
|
||||
|
||||
// AEO compliance routes — xlsx upload, non-compliant item tracking, notes
|
||||
app.use('/api/compliance', createComplianceRouter(complianceUpload));
|
||||
|
||||
|
||||
@@ -1,9 +1,11 @@
|
||||
import React, { useState, useEffect, useCallback, useRef } from 'react';
|
||||
import { Upload, MessageSquare, RefreshCw, AlertCircle, Loader, RotateCcw, Info, FileSpreadsheet } from 'lucide-react';
|
||||
import { Upload, MessageSquare, RefreshCw, AlertCircle, Loader, RotateCcw, Info, FileSpreadsheet, Database } from 'lucide-react';
|
||||
import { useAuth } from '../../contexts/AuthContext';
|
||||
import ComplianceUploadModal from './ComplianceUploadModal';
|
||||
import SupplementalUploadModal from './SupplementalUploadModal';
|
||||
import ComplianceDetailPanel from './ComplianceDetailPanel';
|
||||
import ComplianceChartsPanel from './ComplianceChartsPanel';
|
||||
import GraniteHygieneSection from './GraniteHygieneSection';
|
||||
import MetricInfoPanel from './MetricInfoPanel';
|
||||
import VCLReportPage from './VCLReportPage';
|
||||
import LoaderModal from '../LoaderModal';
|
||||
@@ -357,6 +359,8 @@ export default function CompliancePage({ onNavigate }) {
|
||||
const [error, setError] = useState(null);
|
||||
const [selectedHost, setSelectedHost] = useState(null);
|
||||
const [showUpload, setShowUpload] = useState(false);
|
||||
const [showSuppUpload, setShowSuppUpload] = useState(false);
|
||||
const [suppRefreshKey, setSuppRefreshKey] = useState(0);
|
||||
const [rollbackConfirm, setRollbackConfirm] = useState(false);
|
||||
const [rollbackLoading, setRollbackLoading] = useState(false);
|
||||
const [rollbackResult, setRollbackResult] = useState(null);
|
||||
@@ -576,6 +580,20 @@ export default function CompliancePage({ onNavigate }) {
|
||||
>
|
||||
VCL Report
|
||||
</button>
|
||||
{canWrite() && (
|
||||
<button onClick={() => setShowSuppUpload(true)}
|
||||
style={{
|
||||
background: 'rgba(139,92,246,0.1)', border: '1px solid rgba(139,92,246,0.5)',
|
||||
color: '#A78BFA', padding: '0.5rem 1rem',
|
||||
display: 'flex', alignItems: 'center', gap: '0.4rem',
|
||||
fontFamily: 'monospace', fontSize: '0.75rem', fontWeight: '600',
|
||||
textTransform: 'uppercase', letterSpacing: '0.05em', cursor: 'pointer',
|
||||
borderRadius: '0.375rem',
|
||||
}}>
|
||||
<Database style={{ width: '14px', height: '14px' }} />
|
||||
Supplemental
|
||||
</button>
|
||||
)}
|
||||
{canWrite() && (
|
||||
<button onClick={() => setShowUpload(true)}
|
||||
className="intel-button"
|
||||
@@ -760,6 +778,15 @@ export default function CompliancePage({ onNavigate }) {
|
||||
{/* ── Historical trend charts ──────────────────────────────── */}
|
||||
{!vclView && <ComplianceChartsPanel />}
|
||||
|
||||
{/* ── Granite Hygiene (supplemental workbook findings) ──────── */}
|
||||
{!vclView && (
|
||||
<GraniteHygieneSection
|
||||
activeTeam={activeTeam}
|
||||
teamScope={null}
|
||||
refreshKey={suppRefreshKey}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* ── Device table ─────────────────────────────────────────── */}
|
||||
{!vclView && <div style={{
|
||||
background: 'linear-gradient(135deg,rgba(30,41,59,0.95) 0%,rgba(15,23,42,0.98) 100%)',
|
||||
@@ -970,6 +997,14 @@ export default function CompliancePage({ onNavigate }) {
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* ── Supplemental upload modal ────────────────────────────── */}
|
||||
{showSuppUpload && (
|
||||
<SupplementalUploadModal
|
||||
onClose={() => setShowSuppUpload(false)}
|
||||
onUploadComplete={() => { setShowSuppUpload(false); setSuppRefreshKey(k => k + 1); }}
|
||||
/>
|
||||
)}
|
||||
|
||||
{/* ── Metric info panel ───────────────────────────────────── */}
|
||||
{infoMetric && (
|
||||
<MetricInfoPanel
|
||||
|
||||
329
frontend/src/components/pages/GraniteHygieneSection.js
Normal file
329
frontend/src/components/pages/GraniteHygieneSection.js
Normal file
@@ -0,0 +1,329 @@
|
||||
import React, { useState, useEffect, useCallback } from 'react';
|
||||
import { ChevronDown, ChevronRight, Search, Database } from 'lucide-react';
|
||||
import SupplementalDetailPanel from './SupplementalDetailPanel';
|
||||
|
||||
// ⚠️ CONVENTION: Use relative API path — fallback should be '/api', not 'http://localhost:3001/api'
|
||||
const API_BASE = process.env.REACT_APP_API_BASE || 'http://localhost:3001/api';
|
||||
|
||||
const SHEET_COLORS = {
|
||||
'Missing_AppID': '#F59E0B',
|
||||
'Missing_DF': '#8B5CF6',
|
||||
'Missing_OS_Granite': '#EF4444',
|
||||
'Retired_AppID': '#64748B',
|
||||
};
|
||||
|
||||
const SHEET_ORDER = ['Missing_OS_Granite', 'Missing_AppID', 'Missing_DF', 'Retired_AppID'];
|
||||
|
||||
export default function GraniteHygieneSection({ activeTeam, teamScope: _teamScope, refreshKey }) {
|
||||
const [summary, setSummary] = useState(null);
|
||||
const [expanded, setExpanded] = useState(true);
|
||||
const [activeSheet, setActiveSheet] = useState(null);
|
||||
const [devices, setDevices] = useState([]);
|
||||
const [devicesTotal, setDevicesTotal] = useState(0);
|
||||
const [devicesPage, setDevicesPage] = useState(1);
|
||||
const [devicesSearch, setDevicesSearch] = useState('');
|
||||
const [loading, setLoading] = useState(false);
|
||||
const [devicesLoading, setDevicesLoading] = useState(false);
|
||||
const [selectedHost, setSelectedHost] = useState(null);
|
||||
|
||||
const fetchSummary = useCallback(async () => {
|
||||
setLoading(true);
|
||||
try {
|
||||
const teamParam = activeTeam ? `?team=${activeTeam}` : '';
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/summary${teamParam}`, { credentials: 'include' });
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
setSummary(data);
|
||||
}
|
||||
} catch (_err) { /* silent */ }
|
||||
setLoading(false);
|
||||
}, [activeTeam]);
|
||||
|
||||
useEffect(() => { fetchSummary(); }, [fetchSummary, refreshKey]);
|
||||
|
||||
const fetchDevices = useCallback(async (sheet, page, search) => {
|
||||
setDevicesLoading(true);
|
||||
try {
|
||||
const params = new URLSearchParams({ sheet, page: String(page), page_size: '50' });
|
||||
if (activeTeam) params.set('team', activeTeam);
|
||||
if (search) params.set('search', search);
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/items?${params}`, { credentials: 'include' });
|
||||
if (res.ok) {
|
||||
const data = await res.json();
|
||||
setDevices(data.devices || []);
|
||||
setDevicesTotal(data.total || 0);
|
||||
}
|
||||
} catch (_err) { /* silent */ }
|
||||
setDevicesLoading(false);
|
||||
}, [activeTeam]);
|
||||
|
||||
const handleCardClick = (sheetName) => {
|
||||
if (activeSheet === sheetName) {
|
||||
setActiveSheet(null);
|
||||
setDevices([]);
|
||||
} else {
|
||||
setActiveSheet(sheetName);
|
||||
setDevicesPage(1);
|
||||
setDevicesSearch('');
|
||||
fetchDevices(sheetName, 1, '');
|
||||
}
|
||||
};
|
||||
|
||||
const handlePageChange = (newPage) => {
|
||||
setDevicesPage(newPage);
|
||||
fetchDevices(activeSheet, newPage, devicesSearch);
|
||||
};
|
||||
|
||||
const handleSearch = (val) => {
|
||||
setDevicesSearch(val);
|
||||
setDevicesPage(1);
|
||||
fetchDevices(activeSheet, 1, val);
|
||||
};
|
||||
|
||||
// Don't render if no data
|
||||
if (!summary || (summary.sheets && summary.sheets.length === 0)) {
|
||||
if (loading) return null;
|
||||
return null;
|
||||
}
|
||||
|
||||
const sortedSheets = [...(summary.sheets || [])].sort((a, b) => {
|
||||
const ai = SHEET_ORDER.indexOf(a.sheet_name);
|
||||
const bi = SHEET_ORDER.indexOf(b.sheet_name);
|
||||
return (ai === -1 ? 99 : ai) - (bi === -1 ? 99 : bi);
|
||||
});
|
||||
|
||||
const totalPages = Math.max(1, Math.ceil(devicesTotal / 50));
|
||||
|
||||
return (
|
||||
<div style={styles.section}>
|
||||
{/* Section header */}
|
||||
<button
|
||||
onClick={() => setExpanded(!expanded)}
|
||||
style={styles.sectionHeader}
|
||||
>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>
|
||||
{expanded
|
||||
? <ChevronDown style={{ width: '14px', height: '14px', color: '#64748B' }} />
|
||||
: <ChevronRight style={{ width: '14px', height: '14px', color: '#64748B' }} />
|
||||
}
|
||||
<Database style={{ width: '14px', height: '14px', color: '#14B8A6' }} />
|
||||
<span style={{ fontSize: '0.8rem', fontWeight: '600', color: '#E2E8F0' }}>Granite Hygiene</span>
|
||||
<span style={styles.totalBadge}>{summary.total_active?.toLocaleString()}</span>
|
||||
</div>
|
||||
{summary.last_upload && (
|
||||
<span style={{ fontSize: '0.65rem', color: '#475569', fontFamily: "'JetBrains Mono', monospace" }}>
|
||||
Last: {summary.last_upload.report_date}
|
||||
</span>
|
||||
)}
|
||||
</button>
|
||||
|
||||
{expanded && (
|
||||
<div style={styles.content}>
|
||||
{/* Sheet cards */}
|
||||
<div style={styles.cardGrid}>
|
||||
{sortedSheets.map(sheet => (
|
||||
<button
|
||||
key={sheet.sheet_name}
|
||||
onClick={() => handleCardClick(sheet.sheet_name)}
|
||||
style={{
|
||||
...styles.card,
|
||||
borderColor: activeSheet === sheet.sheet_name
|
||||
? SHEET_COLORS[sheet.sheet_name] || '#334155'
|
||||
: '#334155',
|
||||
}}
|
||||
>
|
||||
<div style={styles.cardTop}>
|
||||
<div style={{
|
||||
width: '4px', height: '100%', borderRadius: '2px',
|
||||
background: SHEET_COLORS[sheet.sheet_name] || '#475569',
|
||||
position: 'absolute', left: 0, top: 0,
|
||||
}} />
|
||||
<span style={styles.cardLabel}>{sheet.label}</span>
|
||||
<span style={styles.cardCount}>{sheet.total_active.toLocaleString()}</span>
|
||||
</div>
|
||||
{sheet.teams && Object.keys(sheet.teams).length > 0 && (
|
||||
<div style={styles.cardTeams}>
|
||||
{Object.entries(sheet.teams).sort((a, b) => b[1] - a[1]).map(([team, count]) => (
|
||||
<span key={team} style={styles.cardTeamBadge}>
|
||||
{team}: {count.toLocaleString()}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</button>
|
||||
))}
|
||||
</div>
|
||||
|
||||
{/* Device list panel */}
|
||||
{activeSheet && (
|
||||
<div style={styles.devicePanel}>
|
||||
<div style={styles.deviceHeader}>
|
||||
<span style={styles.deviceTitle}>
|
||||
{sortedSheets.find(s => s.sheet_name === activeSheet)?.label || activeSheet}
|
||||
<span style={{ color: '#64748B', fontWeight: '400', marginLeft: '0.5rem' }}>
|
||||
({devicesTotal.toLocaleString()} devices)
|
||||
</span>
|
||||
</span>
|
||||
<div style={styles.searchBox}>
|
||||
<Search style={{ width: '12px', height: '12px', color: '#475569' }} />
|
||||
<input
|
||||
type="text"
|
||||
placeholder="Search hostname..."
|
||||
value={devicesSearch}
|
||||
onChange={e => handleSearch(e.target.value)}
|
||||
style={styles.searchInput}
|
||||
/>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{devicesLoading ? (
|
||||
<div style={{ padding: '1.5rem', textAlign: 'center', color: '#475569', fontSize: '0.8rem' }}>Loading...</div>
|
||||
) : devices.length === 0 ? (
|
||||
<div style={{ padding: '1.5rem', textAlign: 'center', color: '#475569', fontSize: '0.8rem' }}>No devices found</div>
|
||||
) : (
|
||||
<>
|
||||
<div style={{ overflowX: 'auto' }}>
|
||||
<table style={styles.table}>
|
||||
<thead>
|
||||
<tr>
|
||||
<th style={styles.th}>Hostname</th>
|
||||
<th style={styles.th}>IP</th>
|
||||
<th style={styles.th}>Function</th>
|
||||
<th style={styles.th}>Model</th>
|
||||
<th style={styles.th}>Vendor</th>
|
||||
<th style={styles.th}>Equip ID</th>
|
||||
<th style={{ ...styles.th, textAlign: 'center' }}>Seen</th>
|
||||
<th style={styles.th}>First Seen</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{devices.map((d, i) => (
|
||||
<tr key={`${d.hostname}-${i}`}
|
||||
onClick={() => setSelectedHost(d.hostname)}
|
||||
style={{ ...(i % 2 === 0 ? {} : { background: 'rgba(15,23,42,0.3)' }), cursor: 'pointer' }}
|
||||
onMouseEnter={e => e.currentTarget.style.background = 'rgba(20,184,166,0.05)'}
|
||||
onMouseLeave={e => e.currentTarget.style.background = i % 2 === 0 ? '' : 'rgba(15,23,42,0.3)'}
|
||||
>
|
||||
<td style={{ ...styles.td, color: '#E2E8F0', fontWeight: '500' }}>{d.hostname}</td>
|
||||
<td style={styles.td}>{d.ip_address || d.ipv6_address || '—'}</td>
|
||||
<td style={styles.td}>{d.device_function || '—'}</td>
|
||||
<td style={styles.td}>{d.model || '—'}</td>
|
||||
<td style={styles.td}>{d.vendor || '—'}</td>
|
||||
<td style={{ ...styles.td, fontFamily: "'JetBrains Mono', monospace" }}>{d.equip_inst_id || '—'}</td>
|
||||
<td style={{ ...styles.td, textAlign: 'center' }}>{d.seen_count}</td>
|
||||
<td style={styles.td}>{d.first_seen || '—'}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
</div>
|
||||
|
||||
{/* Pagination */}
|
||||
{totalPages > 1 && (
|
||||
<div style={styles.pagination}>
|
||||
<button
|
||||
onClick={() => handlePageChange(Math.max(1, devicesPage - 1))}
|
||||
disabled={devicesPage <= 1}
|
||||
style={styles.pageBtn}
|
||||
>
|
||||
← Prev
|
||||
</button>
|
||||
<span style={{ fontSize: '0.7rem', color: '#64748B', fontFamily: "'JetBrains Mono', monospace" }}>
|
||||
Page {devicesPage} of {totalPages}
|
||||
</span>
|
||||
<button
|
||||
onClick={() => handlePageChange(Math.min(totalPages, devicesPage + 1))}
|
||||
disabled={devicesPage >= totalPages}
|
||||
style={styles.pageBtn}
|
||||
>
|
||||
Next →
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Detail panel */}
|
||||
{selectedHost && (
|
||||
<SupplementalDetailPanel
|
||||
hostname={selectedHost}
|
||||
onClose={() => setSelectedHost(null)}
|
||||
onSaved={() => { fetchDevices(activeSheet, devicesPage, devicesSearch); fetchSummary(); }}
|
||||
/>
|
||||
)}
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = {
|
||||
section: {
|
||||
marginTop: '1.5rem', borderRadius: '0.5rem', border: '1px solid #1E293B',
|
||||
background: '#0F172A', overflow: 'hidden',
|
||||
},
|
||||
sectionHeader: {
|
||||
display: 'flex', justifyContent: 'space-between', alignItems: 'center', width: '100%',
|
||||
padding: '0.75rem 1rem', background: 'none', border: 'none', cursor: 'pointer',
|
||||
borderBottom: '1px solid #1E293B',
|
||||
},
|
||||
totalBadge: {
|
||||
fontSize: '0.7rem', fontWeight: '700', color: '#14B8A6',
|
||||
background: 'rgba(20,184,166,0.1)', border: '1px solid rgba(20,184,166,0.3)',
|
||||
borderRadius: '0.25rem', padding: '0.1rem 0.4rem', marginLeft: '0.5rem',
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
},
|
||||
content: { padding: '1rem' },
|
||||
cardGrid: { display: 'grid', gridTemplateColumns: 'repeat(auto-fit, minmax(220px, 1fr))', gap: '0.75rem' },
|
||||
card: {
|
||||
position: 'relative', background: '#1E293B', border: '1px solid #334155',
|
||||
borderRadius: '0.5rem', padding: '0.75rem 0.75rem 0.75rem 1rem', cursor: 'pointer',
|
||||
textAlign: 'left', transition: 'border-color 0.15s',
|
||||
},
|
||||
cardTop: { display: 'flex', justifyContent: 'space-between', alignItems: 'center' },
|
||||
cardLabel: { fontSize: '0.75rem', color: '#CBD5E1', fontWeight: '500' },
|
||||
cardCount: {
|
||||
fontSize: '1.1rem', fontWeight: '700', color: '#E2E8F0',
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
},
|
||||
cardTeams: { display: 'flex', gap: '0.375rem', flexWrap: 'wrap', marginTop: '0.5rem' },
|
||||
cardTeamBadge: {
|
||||
fontSize: '0.6rem', color: '#64748B', fontFamily: "'JetBrains Mono', monospace",
|
||||
background: '#0F172A', borderRadius: '0.2rem', padding: '0.1rem 0.35rem',
|
||||
},
|
||||
devicePanel: {
|
||||
marginTop: '1rem', border: '1px solid #334155', borderRadius: '0.5rem',
|
||||
background: '#1E293B', overflow: 'hidden',
|
||||
},
|
||||
deviceHeader: {
|
||||
display: 'flex', justifyContent: 'space-between', alignItems: 'center',
|
||||
padding: '0.75rem 1rem', borderBottom: '1px solid #334155',
|
||||
},
|
||||
deviceTitle: { fontSize: '0.8rem', fontWeight: '600', color: '#E2E8F0' },
|
||||
searchBox: {
|
||||
display: 'flex', alignItems: 'center', gap: '0.375rem', background: '#0F172A',
|
||||
border: '1px solid #334155', borderRadius: '0.375rem', padding: '0.3rem 0.5rem',
|
||||
},
|
||||
searchInput: {
|
||||
background: 'none', border: 'none', color: '#E2E8F0', fontSize: '0.75rem',
|
||||
outline: 'none', width: '140px', fontFamily: "'JetBrains Mono', monospace",
|
||||
},
|
||||
table: { width: '100%', borderCollapse: 'collapse', fontSize: '0.75rem' },
|
||||
th: {
|
||||
textAlign: 'left', padding: '0.5rem 0.625rem', color: '#475569', fontWeight: '600',
|
||||
fontSize: '0.65rem', borderBottom: '1px solid #334155', textTransform: 'uppercase',
|
||||
letterSpacing: '0.05em', fontFamily: "'JetBrains Mono', monospace", whiteSpace: 'nowrap',
|
||||
},
|
||||
td: { padding: '0.4rem 0.625rem', color: '#94A3B8', whiteSpace: 'nowrap' },
|
||||
pagination: {
|
||||
display: 'flex', justifyContent: 'center', alignItems: 'center', gap: '1rem',
|
||||
padding: '0.75rem', borderTop: '1px solid #334155',
|
||||
},
|
||||
pageBtn: {
|
||||
background: 'none', border: '1px solid #334155', color: '#94A3B8', borderRadius: '0.25rem',
|
||||
padding: '0.25rem 0.625rem', fontSize: '0.7rem', cursor: 'pointer',
|
||||
fontFamily: "'JetBrains Mono', monospace",
|
||||
},
|
||||
};
|
||||
370
frontend/src/components/pages/SupplementalDetailPanel.js
Normal file
370
frontend/src/components/pages/SupplementalDetailPanel.js
Normal file
@@ -0,0 +1,370 @@
|
||||
import React, { useState, useEffect, useCallback } from 'react';
|
||||
import { X, Send, Loader, Calendar, FileText, Save, Trash2 } from 'lucide-react';
|
||||
import { useAuth } from '../../contexts/AuthContext';
|
||||
|
||||
// ⚠️ CONVENTION: Fallback should be relative '/api', not an absolute URL. Other components use REACT_APP_API_BASE which is '/api' in production.
|
||||
const API_BASE = process.env.REACT_APP_API_BASE || 'http://localhost:3001/api';
|
||||
const TEAL = '#14B8A6';
|
||||
|
||||
const SHEET_COLORS = {
|
||||
'Missing_AppID': '#F59E0B',
|
||||
'Missing_DF': '#8B5CF6',
|
||||
'Missing_OS_Granite': '#EF4444',
|
||||
'Retired_AppID': '#64748B',
|
||||
};
|
||||
|
||||
const SHEET_LABELS = {
|
||||
'Missing_AppID': 'Missing App ID',
|
||||
'Missing_DF': 'Missing Device Function',
|
||||
'Missing_OS_Granite': 'Missing OS (Granite)',
|
||||
'Retired_AppID': 'Retired App ID',
|
||||
};
|
||||
|
||||
export default function SupplementalDetailPanel({ hostname, onClose, onSaved }) {
|
||||
const { canWrite } = useAuth();
|
||||
const [detail, setDetail] = useState(null);
|
||||
const [loading, setLoading] = useState(true);
|
||||
const [error, setError] = useState(null);
|
||||
|
||||
// Metadata editing
|
||||
const [resolutionDate, setResolutionDate] = useState('');
|
||||
const [remediationPlan, setRemediationPlan] = useState('');
|
||||
const [selectedSheets, setSelectedSheets] = useState([]);
|
||||
const [metaSaving, setMetaSaving] = useState(false);
|
||||
const [metaError, setMetaError] = useState(null);
|
||||
|
||||
// Notes
|
||||
const [noteText, setNoteText] = useState('');
|
||||
const [noteSubmitting, setNoteSubmitting] = useState(false);
|
||||
|
||||
const fetchDetail = useCallback(async () => {
|
||||
setLoading(true);
|
||||
setError(null);
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/items/${encodeURIComponent(hostname)}`, { credentials: 'include' });
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(data.error || 'Failed to load device');
|
||||
setDetail(data);
|
||||
|
||||
// Default: select all active findings
|
||||
const activeSheets = (data.findings || []).filter(f => f.status === 'active').map(f => f.sheet_name);
|
||||
setSelectedSheets(activeSheets);
|
||||
|
||||
// Populate metadata from first active finding
|
||||
const firstActive = (data.findings || []).find(f => f.status === 'active');
|
||||
if (firstActive) {
|
||||
setResolutionDate(firstActive.resolution_date || '');
|
||||
setRemediationPlan(firstActive.remediation_plan || '');
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err.message);
|
||||
} finally {
|
||||
setLoading(false);
|
||||
}
|
||||
}, [hostname]);
|
||||
|
||||
useEffect(() => { fetchDetail(); }, [fetchDetail]);
|
||||
|
||||
const handleSaveMetadata = async () => {
|
||||
if (selectedSheets.length === 0) return;
|
||||
setMetaSaving(true);
|
||||
setMetaError(null);
|
||||
try {
|
||||
const body = {
|
||||
resolution_date: resolutionDate || null,
|
||||
remediation_plan: remediationPlan || null,
|
||||
sheet_names: selectedSheets,
|
||||
};
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/items/${encodeURIComponent(hostname)}/metadata`, {
|
||||
method: 'PATCH', credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(data.error || 'Failed to save');
|
||||
await fetchDetail();
|
||||
if (onSaved) onSaved();
|
||||
} catch (err) {
|
||||
setMetaError(err.message);
|
||||
} finally {
|
||||
setMetaSaving(false);
|
||||
}
|
||||
};
|
||||
|
||||
const handleAddNote = async () => {
|
||||
const text = noteText.trim();
|
||||
if (!text) return;
|
||||
setNoteSubmitting(true);
|
||||
try {
|
||||
const body = { hostname, note: text, sheet_name: selectedSheets[0] || null };
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/notes`, {
|
||||
method: 'POST', credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (!res.ok) { const d = await res.json(); throw new Error(d.error || 'Failed'); }
|
||||
setNoteText('');
|
||||
await fetchDetail();
|
||||
} catch (_err) { /* silent */ }
|
||||
setNoteSubmitting(false);
|
||||
};
|
||||
|
||||
const handleDeleteNote = async (noteId) => {
|
||||
try {
|
||||
await fetch(`${API_BASE}/compliance/supplemental/notes/${noteId}`, {
|
||||
method: 'DELETE', credentials: 'include',
|
||||
});
|
||||
await fetchDetail();
|
||||
} catch (_err) { /* silent */ }
|
||||
};
|
||||
|
||||
const toggleSheet = (sheet) => {
|
||||
setSelectedSheets(prev =>
|
||||
prev.includes(sheet) ? prev.filter(s => s !== sheet) : [...prev, sheet]
|
||||
);
|
||||
};
|
||||
|
||||
return (
|
||||
<div style={styles.overlay} onClick={onClose}>
|
||||
<div style={styles.panel} onClick={e => e.stopPropagation()}>
|
||||
{/* Header */}
|
||||
<div style={styles.header}>
|
||||
<div>
|
||||
<div style={{ fontSize: '0.95rem', fontWeight: '700', color: '#E2E8F0', fontFamily: "'JetBrains Mono', monospace" }}>
|
||||
{hostname}
|
||||
</div>
|
||||
{detail && (
|
||||
<div style={{ fontSize: '0.7rem', color: '#64748B', marginTop: '0.25rem' }}>
|
||||
{detail.ip_address || detail.ipv6_address || '—'} · {detail.device_function || '—'} · {detail.vendor} {detail.model}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
<button onClick={onClose} style={styles.closeBtn}>
|
||||
<X style={{ width: '16px', height: '16px' }} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Body */}
|
||||
<div style={styles.body}>
|
||||
{loading ? (
|
||||
<div style={{ padding: '2rem', textAlign: 'center' }}>
|
||||
<Loader style={{ width: '20px', height: '20px', color: TEAL, animation: 'spin 1s linear infinite' }} />
|
||||
</div>
|
||||
) : error ? (
|
||||
<div style={{ padding: '1.5rem', color: '#EF4444', fontSize: '0.8rem' }}>{error}</div>
|
||||
) : detail && (
|
||||
<>
|
||||
{/* Device identity */}
|
||||
<div style={styles.section}>
|
||||
<div style={styles.sectionLabel}>Device Info</div>
|
||||
<div style={styles.infoGrid}>
|
||||
<InfoRow label="Equip ID" value={detail.equip_inst_id} />
|
||||
<InfoRow label="Team" value={detail.team} />
|
||||
<InfoRow label="Responsible" value={detail.responsible_team} />
|
||||
<InfoRow label="Vendor" value={detail.vendor} />
|
||||
<InfoRow label="Model" value={detail.model} />
|
||||
<InfoRow label="Function" value={detail.device_function} />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Findings */}
|
||||
<div style={styles.section}>
|
||||
<div style={styles.sectionLabel}>Findings ({detail.findings.filter(f => f.status === 'active').length} active)</div>
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: '0.5rem' }}>
|
||||
{detail.findings.map((f, i) => {
|
||||
const color = SHEET_COLORS[f.sheet_name] || '#64748B';
|
||||
const isSelected = selectedSheets.includes(f.sheet_name);
|
||||
return (
|
||||
<button
|
||||
key={`${f.sheet_name}-${i}`}
|
||||
onClick={() => toggleSheet(f.sheet_name)}
|
||||
style={{
|
||||
...styles.findingCard,
|
||||
borderColor: isSelected ? color : '#334155',
|
||||
opacity: f.status === 'resolved' ? 0.5 : 1,
|
||||
}}
|
||||
>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'center' }}>
|
||||
<span style={{ fontSize: '0.75rem', fontWeight: '600', color }}>
|
||||
{SHEET_LABELS[f.sheet_name] || f.sheet_name}
|
||||
</span>
|
||||
<span style={{ fontSize: '0.65rem', color: '#64748B', fontFamily: "'JetBrains Mono', monospace" }}>
|
||||
{f.status === 'resolved' ? 'Resolved' : `Seen ${f.seen_count}×`}
|
||||
</span>
|
||||
</div>
|
||||
<div style={{ fontSize: '0.65rem', color: '#475569', marginTop: '0.25rem' }}>
|
||||
First: {f.first_seen || '—'} · Last: {f.last_seen || '—'}
|
||||
{f.resolution_date && <span> · Resolve by: <span style={{ color: '#F59E0B' }}>{f.resolution_date}</span></span>}
|
||||
</div>
|
||||
{f.extra_json && Object.keys(f.extra_json).length > 0 && (
|
||||
<div style={{ fontSize: '0.6rem', color: '#334155', marginTop: '0.25rem', fontFamily: "'JetBrains Mono', monospace" }}>
|
||||
{Object.entries(f.extra_json).slice(0, 3).map(([k, v]) => (
|
||||
<span key={k} style={{ marginRight: '0.75rem' }}>{k}: {String(v).slice(0, 30)}</span>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</button>
|
||||
);
|
||||
})}
|
||||
</div>
|
||||
</div>
|
||||
|
||||
{/* Metadata editing */}
|
||||
{canWrite() && selectedSheets.length > 0 && (
|
||||
<div style={styles.section}>
|
||||
<div style={styles.sectionLabel}>
|
||||
Remediation — {selectedSheets.length === 1 ? SHEET_LABELS[selectedSheets[0]] : `${selectedSheets.length} findings`}
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: '0.75rem' }}>
|
||||
<label style={styles.fieldLabel}>
|
||||
<Calendar style={{ width: '11px', height: '11px' }} /> Resolution Date
|
||||
</label>
|
||||
<input
|
||||
type="date"
|
||||
value={resolutionDate}
|
||||
onChange={e => setResolutionDate(e.target.value)}
|
||||
style={styles.input}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<div style={{ marginBottom: '0.75rem' }}>
|
||||
<label style={styles.fieldLabel}>
|
||||
<FileText style={{ width: '11px', height: '11px' }} /> Remediation Plan
|
||||
</label>
|
||||
<textarea
|
||||
value={remediationPlan}
|
||||
onChange={e => setRemediationPlan(e.target.value)}
|
||||
placeholder="Describe remediation steps..."
|
||||
rows={3}
|
||||
style={{ ...styles.input, resize: 'vertical', minHeight: '60px' }}
|
||||
/>
|
||||
</div>
|
||||
|
||||
<button onClick={handleSaveMetadata} disabled={metaSaving} style={styles.saveBtn}>
|
||||
{metaSaving
|
||||
? <><Loader style={{ width: '12px', height: '12px', animation: 'spin 1s linear infinite' }} /> Saving...</>
|
||||
: <><Save style={{ width: '12px', height: '12px' }} /> Save</>
|
||||
}
|
||||
</button>
|
||||
{metaError && <div style={{ color: '#EF4444', fontSize: '0.7rem', marginTop: '0.5rem' }}>{metaError}</div>}
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Notes */}
|
||||
<div style={styles.section}>
|
||||
<div style={styles.sectionLabel}>Notes</div>
|
||||
|
||||
{canWrite() && (
|
||||
<div style={{ display: 'flex', gap: '0.5rem', marginBottom: '0.75rem' }}>
|
||||
<input
|
||||
type="text"
|
||||
value={noteText}
|
||||
onChange={e => setNoteText(e.target.value)}
|
||||
onKeyDown={e => { if (e.key === 'Enter') handleAddNote(); }}
|
||||
placeholder="Add a note..."
|
||||
style={{ ...styles.input, flex: 1 }}
|
||||
/>
|
||||
<button onClick={handleAddNote} disabled={noteSubmitting || !noteText.trim()} style={styles.sendBtn}>
|
||||
<Send style={{ width: '12px', height: '12px' }} />
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{(detail.notes || []).length === 0 ? (
|
||||
<div style={{ fontSize: '0.7rem', color: '#334155' }}>No notes yet</div>
|
||||
) : (
|
||||
<div style={{ display: 'flex', flexDirection: 'column', gap: '0.375rem' }}>
|
||||
{detail.notes.map(n => (
|
||||
<div key={n.id} style={styles.noteCard}>
|
||||
<div style={{ display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start' }}>
|
||||
<div style={{ fontSize: '0.75rem', color: '#CBD5E1', lineHeight: 1.4 }}>{n.note}</div>
|
||||
{canWrite() && (
|
||||
<button onClick={() => handleDeleteNote(n.id)} style={styles.deleteNoteBtn}>
|
||||
<Trash2 style={{ width: '10px', height: '10px' }} />
|
||||
</button>
|
||||
)}
|
||||
</div>
|
||||
<div style={{ fontSize: '0.6rem', color: '#475569', marginTop: '0.25rem' }}>
|
||||
{n.sheet_name && <span style={{ color: SHEET_COLORS[n.sheet_name] || '#64748B' }}>{n.sheet_name}</span>}
|
||||
{n.sheet_name && ' · '}
|
||||
{n.created_at?.slice(0, 10)}
|
||||
</div>
|
||||
</div>
|
||||
))}
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function InfoRow({ label, value }) {
|
||||
return (
|
||||
<div style={{ display: 'flex', gap: '0.5rem', alignItems: 'baseline' }}>
|
||||
<span style={{ fontSize: '0.65rem', color: '#475569', fontFamily: "'JetBrains Mono', monospace", textTransform: 'uppercase', minWidth: '70px' }}>{label}</span>
|
||||
<span style={{ fontSize: '0.75rem', color: '#CBD5E1' }}>{value || '—'}</span>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = {
|
||||
overlay: {
|
||||
position: 'fixed', inset: 0, background: 'rgba(0,0,0,0.6)', zIndex: 8000,
|
||||
display: 'flex', justifyContent: 'flex-end',
|
||||
},
|
||||
panel: {
|
||||
width: '480px', maxWidth: '90vw', height: '100vh', background: '#1E293B',
|
||||
borderLeft: '1px solid #334155', display: 'flex', flexDirection: 'column',
|
||||
overflow: 'hidden',
|
||||
},
|
||||
header: {
|
||||
display: 'flex', justifyContent: 'space-between', alignItems: 'flex-start',
|
||||
padding: '1.25rem 1.25rem 1rem', borderBottom: '1px solid #334155', flexShrink: 0,
|
||||
},
|
||||
closeBtn: { background: 'none', border: 'none', color: '#64748B', cursor: 'pointer', padding: '0.25rem' },
|
||||
body: { flex: 1, overflow: 'auto', padding: '1rem 1.25rem' },
|
||||
section: { marginBottom: '1.5rem' },
|
||||
sectionLabel: {
|
||||
fontSize: '0.65rem', color: '#64748B', fontFamily: "'JetBrains Mono', monospace",
|
||||
textTransform: 'uppercase', letterSpacing: '0.06em', marginBottom: '0.5rem',
|
||||
},
|
||||
infoGrid: { display: 'flex', flexDirection: 'column', gap: '0.3rem' },
|
||||
findingCard: {
|
||||
width: '100%', textAlign: 'left', background: '#0F172A', border: '1px solid #334155',
|
||||
borderRadius: '0.375rem', padding: '0.625rem 0.75rem', cursor: 'pointer',
|
||||
transition: 'border-color 0.15s',
|
||||
},
|
||||
fieldLabel: {
|
||||
display: 'flex', alignItems: 'center', gap: '0.35rem',
|
||||
fontSize: '0.65rem', color: '#64748B', fontFamily: "'JetBrains Mono', monospace",
|
||||
textTransform: 'uppercase', marginBottom: '0.3rem',
|
||||
},
|
||||
input: {
|
||||
width: '100%', background: '#0F172A', border: '1px solid #334155', borderRadius: '0.375rem',
|
||||
padding: '0.4rem 0.625rem', color: '#E2E8F0', fontSize: '0.8rem',
|
||||
fontFamily: "'JetBrains Mono', monospace", outline: 'none',
|
||||
},
|
||||
saveBtn: {
|
||||
display: 'flex', alignItems: 'center', gap: '0.35rem',
|
||||
background: `${TEAL}18`, border: `1px solid ${TEAL}`, color: TEAL,
|
||||
borderRadius: '0.375rem', padding: '0.4rem 0.875rem', cursor: 'pointer',
|
||||
fontSize: '0.75rem', fontWeight: '600', fontFamily: "'JetBrains Mono', monospace",
|
||||
},
|
||||
sendBtn: {
|
||||
background: `${TEAL}18`, border: `1px solid ${TEAL}`, color: TEAL,
|
||||
borderRadius: '0.375rem', padding: '0.4rem 0.5rem', cursor: 'pointer',
|
||||
},
|
||||
noteCard: {
|
||||
background: '#0F172A', border: '1px solid #1E293B', borderRadius: '0.375rem',
|
||||
padding: '0.5rem 0.625rem',
|
||||
},
|
||||
deleteNoteBtn: {
|
||||
background: 'none', border: 'none', color: '#475569', cursor: 'pointer',
|
||||
padding: '0.15rem', borderRadius: '0.2rem', flexShrink: 0,
|
||||
},
|
||||
};
|
||||
280
frontend/src/components/pages/SupplementalUploadModal.js
Normal file
280
frontend/src/components/pages/SupplementalUploadModal.js
Normal file
@@ -0,0 +1,280 @@
|
||||
import React, { useState, useRef } from 'react';
|
||||
import { X, CheckCircle, AlertCircle, Loader, FileSpreadsheet } from 'lucide-react';
|
||||
|
||||
// ⚠️ CONVENTION: Fallback should be a relative path ('/api'), not an absolute URL — fetch uses credentials: 'include' with relative paths
|
||||
const API_BASE = process.env.REACT_APP_API_BASE || 'http://localhost:3001/api';
|
||||
|
||||
const SHEET_LABELS = {
|
||||
'Missing_AppID': 'Missing App ID',
|
||||
'Missing_DF': 'Missing Device Function',
|
||||
'Missing_OS_Granite': 'Missing OS (Granite)',
|
||||
'Retired_AppID': 'Retired App ID',
|
||||
};
|
||||
|
||||
// phase: idle → uploading → preview → committing → done | error
|
||||
export default function SupplementalUploadModal({ onClose, onUploadComplete }) {
|
||||
const [phase, setPhase] = useState('idle');
|
||||
const [previewData, setPreviewData] = useState(null);
|
||||
const [error, setError] = useState(null);
|
||||
const [dragOver, setDragOver] = useState(false);
|
||||
const fileInputRef = useRef(null);
|
||||
|
||||
const handleFile = async (file) => {
|
||||
if (!file) return;
|
||||
if (!file.name.toLowerCase().endsWith('.xlsx')) {
|
||||
setError('File must be an .xlsx spreadsheet');
|
||||
return;
|
||||
}
|
||||
|
||||
setPhase('uploading');
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const formData = new FormData();
|
||||
formData.append('file', file);
|
||||
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/preview`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
body: formData,
|
||||
});
|
||||
const data = await res.json();
|
||||
|
||||
if (!res.ok) throw new Error(data.error || 'Upload failed');
|
||||
|
||||
setPreviewData(data);
|
||||
setPhase('preview');
|
||||
} catch (err) {
|
||||
setError(err.message);
|
||||
setPhase('error');
|
||||
}
|
||||
};
|
||||
|
||||
const handleCommit = async () => {
|
||||
if (!previewData) return;
|
||||
setPhase('committing');
|
||||
setError(null);
|
||||
|
||||
try {
|
||||
const res = await fetch(`${API_BASE}/compliance/supplemental/commit`, {
|
||||
method: 'POST',
|
||||
credentials: 'include',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ tempFile: previewData.tempFile }),
|
||||
});
|
||||
const data = await res.json();
|
||||
if (!res.ok) throw new Error(data.error || 'Commit failed');
|
||||
|
||||
setPhase('done');
|
||||
setTimeout(onUploadComplete, 1200);
|
||||
} catch (err) {
|
||||
setError(err.message);
|
||||
setPhase('error');
|
||||
}
|
||||
};
|
||||
|
||||
const handleDrop = (e) => {
|
||||
e.preventDefault();
|
||||
setDragOver(false);
|
||||
const file = e.dataTransfer?.files?.[0];
|
||||
if (file) handleFile(file);
|
||||
};
|
||||
|
||||
const handleDragOver = (e) => { e.preventDefault(); setDragOver(true); };
|
||||
const handleDragLeave = () => setDragOver(false);
|
||||
const handleInputChange = (e) => { if (e.target.files?.[0]) handleFile(e.target.files[0]); };
|
||||
|
||||
return (
|
||||
<div style={styles.overlay} onClick={onClose}>
|
||||
<div style={styles.modal} onClick={e => e.stopPropagation()}>
|
||||
{/* Header */}
|
||||
<div style={styles.header}>
|
||||
<div style={{ display: 'flex', alignItems: 'center', gap: '0.5rem' }}>
|
||||
<FileSpreadsheet style={{ width: '18px', height: '18px', color: '#14B8A6' }} />
|
||||
<span style={{ fontSize: '1rem', fontWeight: '600', color: '#E2E8F0' }}>
|
||||
Upload Supplemental Workbook
|
||||
</span>
|
||||
</div>
|
||||
<button onClick={onClose} style={styles.closeBtn}>
|
||||
<X style={{ width: '16px', height: '16px' }} />
|
||||
</button>
|
||||
</div>
|
||||
|
||||
{/* Body */}
|
||||
<div style={styles.body}>
|
||||
{phase === 'idle' && (
|
||||
<div
|
||||
style={{ ...styles.dropZone, ...(dragOver ? styles.dropZoneActive : {}) }}
|
||||
onDrop={handleDrop}
|
||||
onDragOver={handleDragOver}
|
||||
onDragLeave={handleDragLeave}
|
||||
onClick={() => fileInputRef.current?.click()}
|
||||
>
|
||||
<FileSpreadsheet style={{ width: '32px', height: '32px', color: '#64748B', marginBottom: '0.75rem' }} />
|
||||
<p style={{ color: '#94A3B8', fontSize: '0.85rem', margin: 0 }}>
|
||||
Drop supplemental xlsx here or click to browse
|
||||
</p>
|
||||
<p style={{ color: '#475569', fontSize: '0.7rem', margin: '0.5rem 0 0' }}>
|
||||
Expected: NTS_AEO_(supp only)_YYYY_MM_DD.xlsx
|
||||
</p>
|
||||
<input
|
||||
ref={fileInputRef}
|
||||
type="file"
|
||||
accept=".xlsx"
|
||||
style={{ display: 'none' }}
|
||||
onChange={handleInputChange}
|
||||
/>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === 'uploading' && (
|
||||
<div style={styles.center}>
|
||||
<Loader style={{ width: '24px', height: '24px', color: '#14B8A6', animation: 'spin 1s linear infinite' }} />
|
||||
<p style={{ color: '#94A3B8', marginTop: '0.75rem' }}>Parsing workbook...</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === 'preview' && previewData && (
|
||||
<div>
|
||||
<div style={{ marginBottom: '1rem' }}>
|
||||
<span style={styles.label}>File:</span>
|
||||
<span style={{ color: '#E2E8F0', fontSize: '0.8rem' }}>{previewData.filename}</span>
|
||||
</div>
|
||||
<div style={{ marginBottom: '1rem' }}>
|
||||
<span style={styles.label}>Report Date:</span>
|
||||
<span style={{ color: '#E2E8F0', fontSize: '0.8rem' }}>{previewData.report_date || 'Unknown'}</span>
|
||||
</div>
|
||||
<div style={{ marginBottom: '1rem' }}>
|
||||
<span style={styles.label}>Total Findings:</span>
|
||||
<span style={{ color: '#14B8A6', fontSize: '0.8rem', fontWeight: '600' }}>{previewData.total?.toLocaleString()}</span>
|
||||
</div>
|
||||
|
||||
{/* Sheet breakdown table */}
|
||||
<table style={styles.table}>
|
||||
<thead>
|
||||
<tr>
|
||||
<th style={styles.th}>Sheet</th>
|
||||
<th style={{ ...styles.th, textAlign: 'right' }}>Total</th>
|
||||
<th style={{ ...styles.th, textAlign: 'right' }}>New</th>
|
||||
<th style={{ ...styles.th, textAlign: 'right' }}>Recurring</th>
|
||||
<th style={{ ...styles.th, textAlign: 'right' }}>Resolved</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody>
|
||||
{Object.entries(previewData.sheets).map(([sheet, counts]) => (
|
||||
<tr key={sheet}>
|
||||
<td style={styles.td}>{SHEET_LABELS[sheet] || sheet}</td>
|
||||
<td style={{ ...styles.td, textAlign: 'right' }}>{counts.count?.toLocaleString()}</td>
|
||||
<td style={{ ...styles.td, textAlign: 'right', color: '#10B981' }}>+{counts.new}</td>
|
||||
<td style={{ ...styles.td, textAlign: 'right', color: '#94A3B8' }}>{counts.recurring?.toLocaleString()}</td>
|
||||
<td style={{ ...styles.td, textAlign: 'right', color: '#F59E0B' }}>-{counts.resolved}</td>
|
||||
</tr>
|
||||
))}
|
||||
</tbody>
|
||||
</table>
|
||||
|
||||
{/* Team breakdown */}
|
||||
{previewData.teams && Object.keys(previewData.teams).length > 0 && (
|
||||
<div style={{ marginTop: '1rem' }}>
|
||||
<span style={styles.label}>Team breakdown:</span>
|
||||
<div style={{ display: 'flex', gap: '0.75rem', flexWrap: 'wrap', marginTop: '0.375rem' }}>
|
||||
{Object.entries(previewData.teams).sort((a, b) => b[1] - a[1]).map(([team, count]) => (
|
||||
<span key={team} style={styles.teamBadge}>
|
||||
{team}: {count.toLocaleString()}
|
||||
</span>
|
||||
))}
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{/* Confirm button */}
|
||||
<div style={{ marginTop: '1.5rem', display: 'flex', gap: '0.75rem', justifyContent: 'flex-end' }}>
|
||||
<button onClick={onClose} style={styles.cancelBtn}>Cancel</button>
|
||||
<button onClick={handleCommit} style={styles.confirmBtn}>Commit Upload</button>
|
||||
</div>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === 'committing' && (
|
||||
<div style={styles.center}>
|
||||
<Loader style={{ width: '24px', height: '24px', color: '#14B8A6', animation: 'spin 1s linear infinite' }} />
|
||||
<p style={{ color: '#94A3B8', marginTop: '0.75rem' }}>Committing to database...</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === 'done' && (
|
||||
<div style={styles.center}>
|
||||
<CheckCircle style={{ width: '32px', height: '32px', color: '#10B981' }} />
|
||||
<p style={{ color: '#10B981', marginTop: '0.75rem', fontWeight: '600' }}>Upload committed successfully</p>
|
||||
</div>
|
||||
)}
|
||||
|
||||
{phase === 'error' && (
|
||||
<div style={styles.center}>
|
||||
<AlertCircle style={{ width: '32px', height: '32px', color: '#EF4444' }} />
|
||||
<p style={{ color: '#EF4444', marginTop: '0.75rem', fontSize: '0.85rem' }}>{error}</p>
|
||||
<button onClick={() => { setPhase('idle'); setError(null); }} style={styles.retryBtn}>
|
||||
Try Again
|
||||
</button>
|
||||
</div>
|
||||
)}
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
const styles = {
|
||||
overlay: {
|
||||
position: 'fixed', inset: 0, background: 'rgba(0,0,0,0.7)', display: 'flex',
|
||||
alignItems: 'center', justifyContent: 'center', zIndex: 9000,
|
||||
},
|
||||
modal: {
|
||||
background: '#1E293B', borderRadius: '0.75rem', border: '1px solid #334155',
|
||||
width: '560px', maxWidth: '90vw', maxHeight: '85vh', overflow: 'auto',
|
||||
boxShadow: '0 25px 50px rgba(0,0,0,0.5)',
|
||||
},
|
||||
header: {
|
||||
display: 'flex', justifyContent: 'space-between', alignItems: 'center',
|
||||
padding: '1rem 1.25rem', borderBottom: '1px solid #334155',
|
||||
},
|
||||
closeBtn: {
|
||||
background: 'none', border: 'none', color: '#64748B', cursor: 'pointer',
|
||||
padding: '0.25rem', borderRadius: '0.25rem',
|
||||
},
|
||||
body: { padding: '1.25rem' },
|
||||
dropZone: {
|
||||
border: '2px dashed #334155', borderRadius: '0.5rem', padding: '2.5rem 1.5rem',
|
||||
textAlign: 'center', cursor: 'pointer', transition: 'border-color 0.15s',
|
||||
},
|
||||
dropZoneActive: { borderColor: '#14B8A6', background: 'rgba(20,184,166,0.05)' },
|
||||
center: { display: 'flex', flexDirection: 'column', alignItems: 'center', padding: '2rem 0' },
|
||||
label: {
|
||||
fontSize: '0.7rem', color: '#64748B', fontFamily: "'JetBrains Mono', monospace",
|
||||
textTransform: 'uppercase', letterSpacing: '0.05em', marginRight: '0.5rem',
|
||||
},
|
||||
table: { width: '100%', borderCollapse: 'collapse', fontSize: '0.8rem' },
|
||||
th: {
|
||||
textAlign: 'left', padding: '0.5rem 0.75rem', color: '#64748B', fontWeight: '600',
|
||||
fontSize: '0.7rem', borderBottom: '1px solid #334155', textTransform: 'uppercase',
|
||||
letterSpacing: '0.05em', fontFamily: "'JetBrains Mono', monospace",
|
||||
},
|
||||
td: { padding: '0.5rem 0.75rem', color: '#CBD5E1', borderBottom: '1px solid #1E293B' },
|
||||
teamBadge: {
|
||||
fontSize: '0.7rem', fontFamily: "'JetBrains Mono', monospace", color: '#94A3B8',
|
||||
background: '#0F172A', border: '1px solid #334155', borderRadius: '0.25rem',
|
||||
padding: '0.2rem 0.5rem',
|
||||
},
|
||||
cancelBtn: {
|
||||
background: 'none', border: '1px solid #475569', color: '#94A3B8', borderRadius: '0.375rem',
|
||||
padding: '0.5rem 1rem', fontSize: '0.8rem', cursor: 'pointer',
|
||||
},
|
||||
confirmBtn: {
|
||||
background: '#14B8A6', border: 'none', color: '#0F172A', borderRadius: '0.375rem',
|
||||
padding: '0.5rem 1.25rem', fontSize: '0.8rem', fontWeight: '600', cursor: 'pointer',
|
||||
},
|
||||
retryBtn: {
|
||||
marginTop: '1rem', background: 'none', border: '1px solid #475569', color: '#94A3B8',
|
||||
borderRadius: '0.375rem', padding: '0.4rem 1rem', fontSize: '0.8rem', cursor: 'pointer',
|
||||
},
|
||||
};
|
||||
@@ -1,25 +1,50 @@
|
||||
{
|
||||
"1.1.1": "Logging & Monitoring",
|
||||
"1.1.3": "Logging & Monitoring",
|
||||
"1.4.1": "Logging & Monitoring",
|
||||
"1.1.1": "Asset Data Quality",
|
||||
"1.1.2": "Asset Data Quality",
|
||||
"1.1.3": "Disaster Recovery",
|
||||
"1.2.2": "Logging & Monitoring",
|
||||
"1.2.4": "End-of-Life OS",
|
||||
"1.2.5": "Endpoint Protection",
|
||||
"1.2.5All": "Endpoint Protection",
|
||||
"1.4.1": "Disaster Recovery",
|
||||
"1.4.2": "Disaster Recovery",
|
||||
"1.5.1B": "Application Security",
|
||||
"1.5.2": "Application Security",
|
||||
"2.3.3i": "Vulnerability Management",
|
||||
"2.3.4i": "Vulnerability Management",
|
||||
"2.3.5i": "Vulnerability Management",
|
||||
"2.3.6i": "Vulnerability Management",
|
||||
"2.3.7i": "Vulnerability Management",
|
||||
"2.3.8i": "Vulnerability Management",
|
||||
"2.3.9i": "Vulnerability Management",
|
||||
"5.2.3": "Access & MFA",
|
||||
"5.2.4": "Access & MFA",
|
||||
"5.2.5": "Access & MFA",
|
||||
"5.2.6": "Access & MFA",
|
||||
"5.2.7": "Access & MFA",
|
||||
"5.2.8": "Access & MFA",
|
||||
"5.3.4": "Endpoint Protection",
|
||||
"5.4.2": "Endpoint Protection",
|
||||
"5.4.3": "Endpoint Protection",
|
||||
"5.4.6": "Vulnerability Management",
|
||||
"5.4.6i": "Vulnerability Management",
|
||||
"5.5.2": "End-of-Life OS",
|
||||
"5.5.4": "Vulnerability Management",
|
||||
"5.5.4i": "Vulnerability Management",
|
||||
"5.5.5": "Decommissioned Assets",
|
||||
"5.6.2A": "Vulnerability Management",
|
||||
"5.6.3": "Asset Data Quality",
|
||||
"5.6.3B": "Access & MFA",
|
||||
"5.7.1": "Logging & Monitoring",
|
||||
"5.8.1": "Application Security",
|
||||
"7.1.1": "Logging & Monitoring",
|
||||
"7.1.4": "Logging & Monitoring",
|
||||
"7.1.4": "Asset Data Quality",
|
||||
"7.6.13": "Disaster Recovery",
|
||||
"7.6.15": "Disaster Recovery",
|
||||
"7.6.16": "Disaster Recovery",
|
||||
"Missing_AppID": "Asset Data Quality",
|
||||
"Missing_DF": "Asset Data Quality",
|
||||
"Missing_EOS": "End-of-Life OS",
|
||||
"Missing_OS": "Asset Data Quality",
|
||||
"5.5.2": "Other"
|
||||
"Vulns_Aging": "Vulnerability Management"
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user