Fix compliance upload: case-insensitive column matching, summary query, page persistence
- Parser (parse_compliance_xlsx.py): column name matching is now case-insensitive via _resolve_col() helper. New reports using lowercase headers (e.g. 'preferred - hostname') are parsed correctly instead of silently returning 0 items. - Drift checker (driftChecker.js): core column presence check uses case-insensitive comparison so lowercase headers no longer trigger false 'missing from all detail sheets' breaking findings. - Summary endpoint (compliance.js): query now selects the most recent upload where vertical IS NULL OR vertical = 'NTS_AEO' in a single query, instead of preferring NULL-vertical legacy uploads that are outdated. - Page persistence (App.js): localStorage page restore no longer checks canAccessPage during useState init (user is null at that point). An effect validates the page once auth resolves. Closes #45
This commit is contained in:
@@ -81,11 +81,12 @@ function compareSchemaToDrift(schema, config) {
|
||||
// Collect per-column stats first, then classify: if a column is missing from
|
||||
// ALL detail sheets it's breaking. If missing from only some (e.g. 5.8.1 uses
|
||||
// CMDB columns), it's cosmetic — the parser handles it via extra_json.
|
||||
// Column matching is case-insensitive — reports may use varying capitalization.
|
||||
const coreColMissingMap = {}; // col -> [sheet names missing it]
|
||||
for (const sheet of detailSheets) {
|
||||
const sheetCols = new Set(sheet.columns || []);
|
||||
const sheetColsLower = new Set((sheet.columns || []).map(c => c.toLowerCase()));
|
||||
for (const coreCol of config.core_cols) {
|
||||
if (!sheetCols.has(coreCol)) {
|
||||
if (!sheetColsLower.has(coreCol.toLowerCase())) {
|
||||
if (!coreColMissingMap[coreCol]) coreColMissingMap[coreCol] = [];
|
||||
coreColMissingMap[coreCol].push(sheet.name);
|
||||
}
|
||||
|
||||
@@ -641,15 +641,11 @@ function createComplianceRouter(upload) {
|
||||
}
|
||||
|
||||
try {
|
||||
// Try AEO uploads first (vertical IS NULL), fall back to NTS_AEO multi-vertical upload
|
||||
// Find the most recent AEO upload — covers both legacy (vertical IS NULL)
|
||||
// and current (vertical = 'NTS_AEO') upload records.
|
||||
let { rows: latestRows } = await pool.query(
|
||||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical IS NULL ORDER BY id DESC LIMIT 1`
|
||||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical IS NULL OR vertical = 'NTS_AEO' ORDER BY id DESC LIMIT 1`
|
||||
);
|
||||
if (latestRows.length === 0 || !latestRows[0].summary_json) {
|
||||
({ rows: latestRows } = await pool.query(
|
||||
`SELECT id, summary_json, report_date, uploaded_at FROM compliance_uploads WHERE vertical = 'NTS_AEO' ORDER BY id DESC LIMIT 1`
|
||||
));
|
||||
}
|
||||
const latestUpload = latestRows[0];
|
||||
if (!latestUpload || !latestUpload.summary_json) return res.json({ entries: [], overall_scores: {}, upload: null });
|
||||
|
||||
|
||||
@@ -98,6 +98,16 @@ def parse_summary(xl):
|
||||
return {'entries': entries, 'overall_scores': overall_scores}
|
||||
|
||||
|
||||
def _resolve_col(columns, canonical):
|
||||
"""Find the actual column name matching `canonical` case-insensitively.
|
||||
Returns the real column name from the DataFrame, or None if not found."""
|
||||
lower = canonical.lower()
|
||||
for c in columns:
|
||||
if c.lower() == lower:
|
||||
return c
|
||||
return None
|
||||
|
||||
|
||||
def parse_sheet(xl, sheet_name, summary_entries):
|
||||
"""Return list of non-compliant item dicts for a detail sheet."""
|
||||
try:
|
||||
@@ -110,9 +120,16 @@ def parse_sheet(xl, sheet_name, summary_entries):
|
||||
|
||||
df.columns = [str(c).strip() for c in df.columns]
|
||||
|
||||
# Build case-insensitive lookup for core columns
|
||||
col_compliant = _resolve_col(df.columns, 'Compliant')
|
||||
col_hostname = _resolve_col(df.columns, 'Preferred - Hostname')
|
||||
col_ip = _resolve_col(df.columns, 'GRANITE - IPv4_Address')
|
||||
col_type = _resolve_col(df.columns, 'GRANITE - Type')
|
||||
col_team = _resolve_col(df.columns, 'Team')
|
||||
|
||||
# Filter to non-compliant rows when the Compliant column exists
|
||||
if 'Compliant' in df.columns:
|
||||
df = df[df['Compliant'] == False]
|
||||
if col_compliant:
|
||||
df = df[df[col_compliant] == False]
|
||||
|
||||
if df.empty:
|
||||
return []
|
||||
@@ -126,20 +143,23 @@ def parse_sheet(xl, sheet_name, summary_entries):
|
||||
|
||||
category = METRIC_CATEGORIES.get(sheet_name, 'Other')
|
||||
|
||||
# Build a case-insensitive set for CORE_COLS to exclude from extra_json
|
||||
core_cols_lower = set(c.lower() for c in CORE_COLS)
|
||||
|
||||
items = []
|
||||
for _, row in df.iterrows():
|
||||
hostname = safe_str(row.get('Preferred - Hostname', ''))
|
||||
hostname = safe_str(row.get(col_hostname, '')) if col_hostname else ''
|
||||
if not hostname:
|
||||
continue
|
||||
|
||||
ip = safe_str(row.get('GRANITE - IPv4_Address', ''))
|
||||
device_type = safe_str(row.get('GRANITE - Type', ''))
|
||||
team = safe_str(row.get('Team', ''))
|
||||
ip = safe_str(row.get(col_ip, '')) if col_ip else ''
|
||||
device_type = safe_str(row.get(col_type, '')) if col_type else ''
|
||||
team = safe_str(row.get(col_team, '')) if col_team else ''
|
||||
|
||||
# Everything non-core goes into extra_json
|
||||
extra = {}
|
||||
for col in df.columns:
|
||||
if col in CORE_COLS:
|
||||
if col.lower() in core_cols_lower:
|
||||
continue
|
||||
val = row.get(col)
|
||||
if pd.isna(val) if not isinstance(val, str) else False:
|
||||
|
||||
@@ -29,15 +29,26 @@ export default function App() {
|
||||
const [currentPage, setCurrentPageRaw] = useState(() => {
|
||||
try {
|
||||
const saved = localStorage.getItem('cve-dashboard-page');
|
||||
return saved && canAccessPage(saved, user?.group) ? saved : 'home';
|
||||
return saved || 'home';
|
||||
} catch { return 'home'; }
|
||||
});
|
||||
const setCurrentPage = (page) => {
|
||||
if (!canAccessPage(page, user?.group)) { setCurrentPageRaw('home'); return; }
|
||||
setCurrentPageRaw(page);
|
||||
try { localStorage.setItem('cve-dashboard-page', page); } catch {}
|
||||
try { localStorage.setItem('cve-dashboard-page', page); } catch {};
|
||||
};
|
||||
|
||||
// Once auth resolves, validate the restored page against the user's group.
|
||||
// If the page isn't accessible, fall back to home.
|
||||
React.useEffect(() => {
|
||||
if (!authLoading && user) {
|
||||
setCurrentPageRaw(prev => {
|
||||
if (!canAccessPage(prev, user.group)) return 'home';
|
||||
return prev;
|
||||
});
|
||||
}
|
||||
}, [authLoading, user]);
|
||||
|
||||
const [navOpen, setNavOpen] = useState(false);
|
||||
const [calendarFilter, setCalendarFilter] = useState(null);
|
||||
const [reportingExcFilter, setReportingExcFilter] = useState(null);
|
||||
|
||||
Reference in New Issue
Block a user