Fix compliance upload: case-insensitive column matching, summary query, page persistence

- Parser (parse_compliance_xlsx.py): column name matching is now
  case-insensitive via _resolve_col() helper. New reports using lowercase
  headers (e.g. 'preferred - hostname') are parsed correctly instead of
  silently returning 0 items.

- Drift checker (driftChecker.js): core column presence check uses
  case-insensitive comparison so lowercase headers no longer trigger
  false 'missing from all detail sheets' breaking findings.

- Summary endpoint (compliance.js): query now selects the most recent
  upload where vertical IS NULL OR vertical = 'NTS_AEO' in a single
  query, instead of preferring NULL-vertical legacy uploads that are
  outdated.

- Page persistence (App.js): localStorage page restore no longer checks
  canAccessPage during useState init (user is null at that point).
  An effect validates the page once auth resolves.

Closes #45
This commit is contained in:
Jordan Ramos
2026-08-10 13:55:10 -06:00
parent ef6a389875
commit d0f6460c0a
4 changed files with 46 additions and 18 deletions

View File

@@ -81,11 +81,12 @@ function compareSchemaToDrift(schema, config) {
// Collect per-column stats first, then classify: if a column is missing from
// ALL detail sheets it's breaking. If missing from only some (e.g. 5.8.1 uses
// CMDB columns), it's cosmetic — the parser handles it via extra_json.
// Column matching is case-insensitive — reports may use varying capitalization.
const coreColMissingMap = {}; // col -> [sheet names missing it]
for (const sheet of detailSheets) {
const sheetCols = new Set(sheet.columns || []);
const sheetColsLower = new Set((sheet.columns || []).map(c => c.toLowerCase()));
for (const coreCol of config.core_cols) {
if (!sheetCols.has(coreCol)) {
if (!sheetColsLower.has(coreCol.toLowerCase())) {
if (!coreColMissingMap[coreCol]) coreColMissingMap[coreCol] = [];
coreColMissingMap[coreCol].push(sheet.name);
}

View File

@@ -641,15 +641,11 @@ function createComplianceRouter(upload) {
}
try {
// Try AEO uploads first (vertical IS NULL), fall back to NTS_AEO multi-vertical upload
// Find the most recent AEO upload — covers both legacy (vertical IS NULL)
// and current (vertical = 'NTS_AEO') upload records.
let { rows: latestRows } = await pool.query(
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical IS NULL ORDER BY id DESC LIMIT 1`
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical IS NULL OR vertical = 'NTS_AEO' ORDER BY id DESC LIMIT 1`
);
if (latestRows.length === 0 || !latestRows[0].summary_json) {
({ rows: latestRows } = await pool.query(
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical = 'NTS_AEO' ORDER BY id DESC LIMIT 1`
));
}
const latestUpload = latestRows[0];
if (!latestUpload || !latestUpload.summary_json) return res.json({ entries: [], overall_scores: {}, upload: null });

View File

@@ -98,6 +98,16 @@ def parse_summary(xl):
return {'entries': entries, 'overall_scores': overall_scores}
def _resolve_col(columns, canonical):
"""Find the actual column name matching `canonical` case-insensitively.
Returns the real column name from the DataFrame, or None if not found."""
lower = canonical.lower()
for c in columns:
if c.lower() == lower:
return c
return None
def parse_sheet(xl, sheet_name, summary_entries):
"""Return list of non-compliant item dicts for a detail sheet."""
try:
@@ -110,9 +120,16 @@ def parse_sheet(xl, sheet_name, summary_entries):
df.columns = [str(c).strip() for c in df.columns]
# Build case-insensitive lookup for core columns
col_compliant = _resolve_col(df.columns, 'Compliant')
col_hostname = _resolve_col(df.columns, 'Preferred - Hostname')
col_ip = _resolve_col(df.columns, 'GRANITE - IPv4_Address')
col_type = _resolve_col(df.columns, 'GRANITE - Type')
col_team = _resolve_col(df.columns, 'Team')
# Filter to non-compliant rows when the Compliant column exists
if 'Compliant' in df.columns:
df = df[df['Compliant'] == False]
if col_compliant:
df = df[df[col_compliant] == False]
if df.empty:
return []
@@ -126,20 +143,23 @@ def parse_sheet(xl, sheet_name, summary_entries):
category = METRIC_CATEGORIES.get(sheet_name, 'Other')
# Build a case-insensitive set for CORE_COLS to exclude from extra_json
core_cols_lower = set(c.lower() for c in CORE_COLS)
items = []
for _, row in df.iterrows():
hostname = safe_str(row.get('Preferred - Hostname', ''))
hostname = safe_str(row.get(col_hostname, '')) if col_hostname else ''
if not hostname:
continue
ip = safe_str(row.get('GRANITE - IPv4_Address', ''))
device_type = safe_str(row.get('GRANITE - Type', ''))
team = safe_str(row.get('Team', ''))
ip = safe_str(row.get(col_ip, '')) if col_ip else ''
device_type = safe_str(row.get(col_type, '')) if col_type else ''
team = safe_str(row.get(col_team, '')) if col_team else ''
# Everything non-core goes into extra_json
extra = {}
for col in df.columns:
if col in CORE_COLS:
if col.lower() in core_cols_lower:
continue
val = row.get(col)
if pd.isna(val) if not isinstance(val, str) else False:

View File

@@ -29,15 +29,26 @@ export default function App() {
const [currentPage, setCurrentPageRaw] = useState(() => {
try {
const saved = localStorage.getItem('cve-dashboard-page');
return saved && canAccessPage(saved, user?.group) ? saved : 'home';
return saved || 'home';
} catch { return 'home'; }
});
const setCurrentPage = (page) => {
if (!canAccessPage(page, user?.group)) { setCurrentPageRaw('home'); return; }
setCurrentPageRaw(page);
try { localStorage.setItem('cve-dashboard-page', page); } catch {}
try { localStorage.setItem('cve-dashboard-page', page); } catch {};
};
// Once auth resolves, validate the restored page against the user's group.
// If the page isn't accessible, fall back to home.
React.useEffect(() => {
if (!authLoading && user) {
setCurrentPageRaw(prev => {
if (!canAccessPage(prev, user.group)) return 'home';
return prev;
});
}
}, [authLoading, user]);
const [navOpen, setNavOpen] = useState(false);
const [calendarFilter, setCalendarFilter] = useState(null);
const [reportingExcFilter, setReportingExcFilter] = useState(null);