fix: aggregate anomaly data per day instead of taking latest — fixes missing returned bars when multiple syncs per day
This commit is contained in:
root
6
.gitignore
vendored
6
.gitignore
vendored
@@ -78,3 +78,9 @@ docs/card-lookup-results.csv
|
||||
docs/card-prod-archer-firewall-request.md
|
||||
docs/granite-reassignment-upload.csv
|
||||
docs/granite-reassignment-upload.xlsx
|
||||
|
||||
# Production DB copies
|
||||
cve_database_prod.db
|
||||
cve_database.db.prod
|
||||
cve_database.db.backup
|
||||
database.db
|
||||
|
||||
83
backend/scripts/diagnose-chart-alignment.js
Normal file
83
backend/scripts/diagnose-chart-alignment.js
Normal file
@@ -0,0 +1,83 @@
|
||||
#!/usr/bin/env node
|
||||
// Diagnostic: check alignment between counts history dates and anomaly log dates
|
||||
// Usage: node backend/scripts/diagnose-chart-alignment.js
|
||||
|
||||
const path = require('path');
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const DB_PATH = path.join(__dirname, '..', 'cve_database.db');
|
||||
|
||||
function dbAll(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(sql, params, (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows || []);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function fmtDate(d) {
|
||||
if (!d) return '';
|
||||
const p = d.split('-');
|
||||
if (p.length === 3) return `${p[1]}/${p[2]}/${p[0].slice(2)}`;
|
||||
return d;
|
||||
}
|
||||
|
||||
function extractDate(ts) {
|
||||
if (!ts) return '';
|
||||
return ts.split('T')[0].split(' ')[0];
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const db = new sqlite3.Database(DB_PATH);
|
||||
|
||||
// Get counts history dates (same query as the API)
|
||||
const countsRows = await dbAll(db,
|
||||
`SELECT date FROM (
|
||||
SELECT DATE(recorded_at) AS date,
|
||||
ROW_NUMBER() OVER (PARTITION BY DATE(recorded_at) ORDER BY recorded_at DESC) AS rn
|
||||
FROM ivanti_counts_history
|
||||
) WHERE rn = 1 ORDER BY date ASC`
|
||||
);
|
||||
const countsDates = new Set(countsRows.map(r => fmtDate(r.date)));
|
||||
|
||||
// Get anomaly history (same query as the API)
|
||||
const anomalyRows = await dbAll(db,
|
||||
`SELECT sync_timestamp, newly_archived_count, returned_count, return_classification_json
|
||||
FROM ivanti_sync_anomaly_log
|
||||
ORDER BY sync_timestamp DESC LIMIT 30`
|
||||
);
|
||||
|
||||
console.log('=== Counts History Dates (last 10) ===');
|
||||
const lastTen = countsRows.slice(-10);
|
||||
for (const r of lastTen) {
|
||||
console.log(` ${r.date} → ${fmtDate(r.date)}`);
|
||||
}
|
||||
|
||||
console.log('\n=== Anomaly Log Entries with Activity ===');
|
||||
for (const a of anomalyRows) {
|
||||
if (a.newly_archived_count === 0 && a.returned_count === 0) continue;
|
||||
const rawDate = extractDate(a.sync_timestamp);
|
||||
const dateKey = fmtDate(rawDate);
|
||||
const inCounts = countsDates.has(dateKey);
|
||||
console.log(` ${a.sync_timestamp} → raw="${rawDate}" → key="${dateKey}" | archived=${a.newly_archived_count} returned=${a.returned_count} | in counts: ${inCounts ? 'YES' : '*** NO ***'}`);
|
||||
}
|
||||
|
||||
console.log('\n=== All Anomaly Dates NOT in Counts History ===');
|
||||
let missingCount = 0;
|
||||
for (const a of anomalyRows) {
|
||||
const rawDate = extractDate(a.sync_timestamp);
|
||||
const dateKey = fmtDate(rawDate);
|
||||
if (!countsDates.has(dateKey)) {
|
||||
console.log(` MISSING: ${a.sync_timestamp} → "${dateKey}" (archived=${a.newly_archived_count}, returned=${a.returned_count})`);
|
||||
missingCount++;
|
||||
}
|
||||
}
|
||||
if (missingCount === 0) console.log(' (none — all anomaly dates have matching counts history)');
|
||||
|
||||
db.close();
|
||||
}
|
||||
|
||||
main().catch(err => {
|
||||
console.error('Fatal error:', err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -226,34 +226,44 @@ export default function IvantiCountsChart() {
|
||||
);
|
||||
|
||||
// Build archive activity data aligned to the same date axis as the main chart.
|
||||
// Aggregate anomaly rows by date (take the last sync per day, matching the
|
||||
// counts history pattern), then merge onto the chartData date set.
|
||||
// Aggregate anomaly rows by date — sum archived/returned counts and merge
|
||||
// classifications across all syncs that day, then align to the chartData dates.
|
||||
const archiveData = useMemo(() => {
|
||||
if (!anomalies.length || !chartData.length) return [];
|
||||
|
||||
// Group anomalies by date, keep the latest per day
|
||||
// Aggregate all anomaly rows per date (sum counts, merge classifications)
|
||||
const byDate = {};
|
||||
for (const a of anomalies) {
|
||||
const rawDate = extractDate(a.sync_timestamp);
|
||||
const dateKey = fmtDate(rawDate);
|
||||
// anomaly/history returns newest first, so first seen per date is the latest
|
||||
if (!byDate[dateKey]) {
|
||||
byDate[dateKey] = a;
|
||||
byDate[dateKey] = {
|
||||
archived: 0,
|
||||
returned: 0,
|
||||
classification: {},
|
||||
return_classification: {},
|
||||
is_significant: false,
|
||||
};
|
||||
}
|
||||
const entry = byDate[dateKey];
|
||||
entry.archived += (a.newly_archived_count || 0);
|
||||
entry.returned += (a.returned_count || 0);
|
||||
if (a.is_significant) entry.is_significant = true;
|
||||
|
||||
// Merge classification counts
|
||||
for (const [key, val] of Object.entries(a.classification || {})) {
|
||||
entry.classification[key] = (entry.classification[key] || 0) + (val || 0);
|
||||
}
|
||||
for (const [key, val] of Object.entries(a.return_classification || {})) {
|
||||
entry.return_classification[key] = (entry.return_classification[key] || 0) + (val || 0);
|
||||
}
|
||||
}
|
||||
|
||||
// Map onto the chart date axis so both charts share the same X positions
|
||||
return chartData.map(point => {
|
||||
const anomaly = byDate[point.date];
|
||||
if (anomaly) {
|
||||
return {
|
||||
date: point.date,
|
||||
archived: anomaly.newly_archived_count || 0,
|
||||
returned: anomaly.returned_count || 0,
|
||||
classification: anomaly.classification || {},
|
||||
return_classification: anomaly.return_classification || {},
|
||||
is_significant: anomaly.is_significant,
|
||||
};
|
||||
const agg = byDate[point.date];
|
||||
if (agg) {
|
||||
return { date: point.date, ...agg };
|
||||
}
|
||||
return { date: point.date, archived: 0, returned: 0, classification: {}, return_classification: {}, is_significant: false };
|
||||
});
|
||||
@@ -377,13 +387,13 @@ export default function IvantiCountsChart() {
|
||||
}}>
|
||||
Archive Activity
|
||||
</div>
|
||||
<ResponsiveContainer width="100%" height={64}>
|
||||
<ResponsiveContainer width="100%" height={80}>
|
||||
<BarChart data={archiveData} margin={{ top: 2, right: 12, bottom: 0, left: -12 }}>
|
||||
<CartesianGrid {...GRID_STYLE} />
|
||||
<XAxis dataKey="date" tick={false} axisLine={false} />
|
||||
<YAxis tick={AXIS_STYLE} allowDecimals={false} width={30} />
|
||||
<Tooltip content={<ArchiveTooltip />} />
|
||||
<Bar dataKey="archived" name="Archived" stackId="a" maxBarSize={12}>
|
||||
<Bar dataKey="archived" name="Archived" stackId="a" maxBarSize={14}>
|
||||
{archiveData.map((entry, idx) => (
|
||||
<Cell
|
||||
key={`arch-${idx}`}
|
||||
@@ -391,7 +401,14 @@ export default function IvantiCountsChart() {
|
||||
/>
|
||||
))}
|
||||
</Bar>
|
||||
<Bar dataKey="returned" name="Returned" stackId="a" fill={TEAL} maxBarSize={12} />
|
||||
<Bar dataKey="returned" name="Returned" stackId="a" maxBarSize={14}>
|
||||
{archiveData.map((entry, idx) => (
|
||||
<Cell
|
||||
key={`ret-${idx}`}
|
||||
fill={TEAL}
|
||||
/>
|
||||
))}
|
||||
</Bar>
|
||||
</BarChart>
|
||||
</ResponsiveContainer>
|
||||
</div>
|
||||
|
||||
Reference in New Issue
Block a user