From 5a9df2103f06f5842da5c4adfb7e75bbc92e1187 Mon Sep 17 00:00:00 2001 From: root Date: Fri, 1 May 2026 19:28:29 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20aggregate=20anomaly=20data=20per=20day?= =?UTF-8?q?=20instead=20of=20taking=20latest=20=E2=80=94=20fixes=20missing?= =?UTF-8?q?=20returned=20bars=20when=20multiple=20syncs=20per=20day?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .gitignore | 6 ++ backend/scripts/diagnose-chart-alignment.js | 83 +++++++++++++++++++ .../src/components/pages/IvantiCountsChart.js | 53 ++++++++---- 3 files changed, 124 insertions(+), 18 deletions(-) create mode 100644 backend/scripts/diagnose-chart-alignment.js diff --git a/.gitignore b/.gitignore index e2f92d2..20107c3 100644 --- a/.gitignore +++ b/.gitignore @@ -78,3 +78,9 @@ docs/card-lookup-results.csv docs/card-prod-archer-firewall-request.md docs/granite-reassignment-upload.csv docs/granite-reassignment-upload.xlsx + +# Production DB copies +cve_database_prod.db +cve_database.db.prod +cve_database.db.backup +database.db diff --git a/backend/scripts/diagnose-chart-alignment.js b/backend/scripts/diagnose-chart-alignment.js new file mode 100644 index 0000000..5b34dfc --- /dev/null +++ b/backend/scripts/diagnose-chart-alignment.js @@ -0,0 +1,83 @@ +#!/usr/bin/env node +// Diagnostic: check alignment between counts history dates and anomaly log dates +// Usage: node backend/scripts/diagnose-chart-alignment.js + +const path = require('path'); +const sqlite3 = require('sqlite3').verbose(); +const DB_PATH = path.join(__dirname, '..', 'cve_database.db'); + +function dbAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) reject(err); + else resolve(rows || []); + }); + }); +} + +function fmtDate(d) { + if (!d) return ''; + const p = d.split('-'); + if (p.length === 3) return `${p[1]}/${p[2]}/${p[0].slice(2)}`; + return d; +} + +function extractDate(ts) { + if (!ts) return ''; + return ts.split('T')[0].split(' ')[0]; +} + +async function main() { + const db = new sqlite3.Database(DB_PATH); + + // Get counts history dates (same query as the API) + const countsRows = await dbAll(db, + `SELECT date FROM ( + SELECT DATE(recorded_at) AS date, + ROW_NUMBER() OVER (PARTITION BY DATE(recorded_at) ORDER BY recorded_at DESC) AS rn + FROM ivanti_counts_history + ) WHERE rn = 1 ORDER BY date ASC` + ); + const countsDates = new Set(countsRows.map(r => fmtDate(r.date))); + + // Get anomaly history (same query as the API) + const anomalyRows = await dbAll(db, + `SELECT sync_timestamp, newly_archived_count, returned_count, return_classification_json + FROM ivanti_sync_anomaly_log + ORDER BY sync_timestamp DESC LIMIT 30` + ); + + console.log('=== Counts History Dates (last 10) ==='); + const lastTen = countsRows.slice(-10); + for (const r of lastTen) { + console.log(` ${r.date} → ${fmtDate(r.date)}`); + } + + console.log('\n=== Anomaly Log Entries with Activity ==='); + for (const a of anomalyRows) { + if (a.newly_archived_count === 0 && a.returned_count === 0) continue; + const rawDate = extractDate(a.sync_timestamp); + const dateKey = fmtDate(rawDate); + const inCounts = countsDates.has(dateKey); + console.log(` ${a.sync_timestamp} → raw="${rawDate}" → key="${dateKey}" | archived=${a.newly_archived_count} returned=${a.returned_count} | in counts: ${inCounts ? 'YES' : '*** NO ***'}`); + } + + console.log('\n=== All Anomaly Dates NOT in Counts History ==='); + let missingCount = 0; + for (const a of anomalyRows) { + const rawDate = extractDate(a.sync_timestamp); + const dateKey = fmtDate(rawDate); + if (!countsDates.has(dateKey)) { + console.log(` MISSING: ${a.sync_timestamp} → "${dateKey}" (archived=${a.newly_archived_count}, returned=${a.returned_count})`); + missingCount++; + } + } + if (missingCount === 0) console.log(' (none — all anomaly dates have matching counts history)'); + + db.close(); +} + +main().catch(err => { + console.error('Fatal error:', err); + process.exit(1); +}); diff --git a/frontend/src/components/pages/IvantiCountsChart.js b/frontend/src/components/pages/IvantiCountsChart.js index 5f1dbbd..62cb186 100644 --- a/frontend/src/components/pages/IvantiCountsChart.js +++ b/frontend/src/components/pages/IvantiCountsChart.js @@ -226,34 +226,44 @@ export default function IvantiCountsChart() { ); // Build archive activity data aligned to the same date axis as the main chart. - // Aggregate anomaly rows by date (take the last sync per day, matching the - // counts history pattern), then merge onto the chartData date set. + // Aggregate anomaly rows by date — sum archived/returned counts and merge + // classifications across all syncs that day, then align to the chartData dates. const archiveData = useMemo(() => { if (!anomalies.length || !chartData.length) return []; - // Group anomalies by date, keep the latest per day + // Aggregate all anomaly rows per date (sum counts, merge classifications) const byDate = {}; for (const a of anomalies) { const rawDate = extractDate(a.sync_timestamp); const dateKey = fmtDate(rawDate); - // anomaly/history returns newest first, so first seen per date is the latest if (!byDate[dateKey]) { - byDate[dateKey] = a; + byDate[dateKey] = { + archived: 0, + returned: 0, + classification: {}, + return_classification: {}, + is_significant: false, + }; + } + const entry = byDate[dateKey]; + entry.archived += (a.newly_archived_count || 0); + entry.returned += (a.returned_count || 0); + if (a.is_significant) entry.is_significant = true; + + // Merge classification counts + for (const [key, val] of Object.entries(a.classification || {})) { + entry.classification[key] = (entry.classification[key] || 0) + (val || 0); + } + for (const [key, val] of Object.entries(a.return_classification || {})) { + entry.return_classification[key] = (entry.return_classification[key] || 0) + (val || 0); } } // Map onto the chart date axis so both charts share the same X positions return chartData.map(point => { - const anomaly = byDate[point.date]; - if (anomaly) { - return { - date: point.date, - archived: anomaly.newly_archived_count || 0, - returned: anomaly.returned_count || 0, - classification: anomaly.classification || {}, - return_classification: anomaly.return_classification || {}, - is_significant: anomaly.is_significant, - }; + const agg = byDate[point.date]; + if (agg) { + return { date: point.date, ...agg }; } return { date: point.date, archived: 0, returned: 0, classification: {}, return_classification: {}, is_significant: false }; }); @@ -377,13 +387,13 @@ export default function IvantiCountsChart() { }}> Archive Activity - + } /> - + {archiveData.map((entry, idx) => ( ))} - + + {archiveData.map((entry, idx) => ( + + ))} +