diff --git a/backend/scripts/migrate-to-postgres.js b/backend/scripts/migrate-to-postgres.js new file mode 100644 index 0000000..aaa59e7 --- /dev/null +++ b/backend/scripts/migrate-to-postgres.js @@ -0,0 +1,928 @@ +#!/usr/bin/env node +/** + * migrate-to-postgres.js — Data Migration Script + * + * Copies all data from the SQLite database (cve_database.db) to PostgreSQL. + * The SQLite file is opened READ-ONLY and is never modified. + * + * Special handling: + * - ivanti_findings_cache.findings_json → individual rows in ivanti_findings + * - ivanti_finding_notes → merged into ivanti_findings.note column + * - ivanti_finding_overrides → merged into ivanti_findings.override_host_name / override_dns + * - ivanti_sync_state and ivanti_counts_cache → populated from ivanti_findings_cache metadata + * + * Type conversions: + * - SQLite 0/1 integers → Postgres boolean + * - SQLite DATETIME strings → Postgres TIMESTAMPTZ (passed as-is) + * - SQLite NULL → Postgres NULL + * + * Uses ON CONFLICT DO NOTHING for idempotency (safe to re-run). + * + * Usage: + * node backend/scripts/migrate-to-postgres.js + * + * Requires: + * - DATABASE_URL env var (or .env file in backend/) + * - SQLite database at backend/cve_database.db + */ + +const path = require('path'); +require('dotenv').config({ path: path.join(__dirname, '..', '.env') }); + +const fs = require('fs'); +const sqlite3 = require('sqlite3').verbose(); +const { Pool } = require('pg'); + +// --------------------------------------------------------------------------- +// Configuration +// --------------------------------------------------------------------------- +const SQLITE_PATH = path.join(__dirname, '..', 'cve_database.db'); +const SCHEMA_PATH = path.join(__dirname, '..', 'db-schema.sql'); +const DATABASE_URL = process.env.DATABASE_URL; + +if (!DATABASE_URL) { + console.error('ERROR: DATABASE_URL environment variable is not set.'); + console.error('Expected format: postgresql://user:password@host:port/database'); + process.exit(1); +} + +if (!fs.existsSync(SQLITE_PATH)) { + console.error(`ERROR: SQLite database not found at ${SQLITE_PATH}`); + process.exit(1); +} + +// --------------------------------------------------------------------------- +// SQLite helpers +// --------------------------------------------------------------------------- +function sqliteAll(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.all(sql, params, (err, rows) => { + if (err) reject(err); + else resolve(rows || []); + }); + }); +} + +function sqliteGet(db, sql, params = []) { + return new Promise((resolve, reject) => { + db.get(sql, params, (err, row) => { + if (err) reject(err); + else resolve(row); + }); + }); +} + +// --------------------------------------------------------------------------- +// Extract finding fields from raw JSON object (mirrors ivantiFindings.js) +// --------------------------------------------------------------------------- +function extractFinding(f) { + const rawDueDate = f.statusEmbedded?.dueDate || f.dueDate || ''; + const dueDate = rawDueDate ? rawDueDate.split('T')[0] : null; + + const buOwnership = f.assetCustomAttributes?.['1550_host_1']?.[0] + || f.buOwnership || f.bu_ownership || ''; + + const cves = Array.isArray(f.cves) + ? f.cves + : (f.vulnerabilities?.vulnInfoList || []).map(v => v.cve).filter(Boolean); + + // Workflow extraction + let workflow = null; + if (f.workflow && typeof f.workflow === 'object') { + workflow = { + id: f.workflow.id || '', + state: f.workflow.state || '', + type: f.workflow.type || 'FP', + }; + } else if (f.workflowDistribution) { + const wfDist = f.workflowDistribution || {}; + const fpBuckets = [ + ...(wfDist.actionableWorkflows || []), + ...(wfDist.requestedWorkflows || []), + ...(wfDist.reworkedWorkflows || []), + ...(wfDist.rejectedWorkflows || []), + ...(wfDist.expiredWorkflows || []), + ...(wfDist.approvedWorkflows || []), + ].filter(w => (w.generatedId || '').startsWith('FP#')); + const fpEntry = fpBuckets[0] || null; + if (fpEntry) { + workflow = { + id: fpEntry.generatedId || '', + state: fpEntry.state || '', + type: 'FP', + }; + } + } + + return { + id: String(f.id), + hostId: f.hostId || f.host?.hostId || null, + title: f.title || '', + severity: typeof f.severity === 'number' ? f.severity : parseFloat(f.severity) || 0, + vrrGroup: f.vrrGroup || f.severityGroup || f.vrr_group || '', + hostName: f.hostName || f.host?.hostName || f.host_name || '', + ipAddress: f.ipAddress || f.host?.ipAddress || f.ip_address || '', + dns: f.dns || f.host?.fqdn || '', + status: f.status || '', + slaStatus: f.slaStatus || f.sla_status || '', + dueDate: dueDate, + lastFoundOn: f.lastFoundOn || f.last_found_on || null, + buOwnership, + cves, + workflow, + }; +} + +// --------------------------------------------------------------------------- +// Batch insert helper for Postgres +// --------------------------------------------------------------------------- +async function batchInsert(pool, tableName, columns, rows, conflictClause = 'DO NOTHING') { + if (rows.length === 0) return 0; + + const BATCH_SIZE = 100; + let inserted = 0; + + for (let i = 0; i < rows.length; i += BATCH_SIZE) { + const batch = rows.slice(i, i + BATCH_SIZE); + const values = []; + const placeholders = []; + + batch.forEach((row, idx) => { + const offset = idx * columns.length; + const rowPlaceholders = columns.map((_, colIdx) => `$${offset + colIdx + 1}`); + placeholders.push(`(${rowPlaceholders.join(', ')})`); + values.push(...row); + }); + + const sql = `INSERT INTO ${tableName} (${columns.join(', ')}) + VALUES ${placeholders.join(', ')} + ON CONFLICT ${conflictClause}`; + + await pool.query(sql, values); + inserted += batch.length; + } + + return inserted; +} + +// --------------------------------------------------------------------------- +// Table migration definitions +// --------------------------------------------------------------------------- + +/** + * Each entry defines how to copy a SQLite table to Postgres. + * - sqliteTable: source table name + * - pgTable: destination table name (defaults to sqliteTable) + * - columns: array of { src, dest, transform } objects + * - conflict: ON CONFLICT clause (default: DO NOTHING) + * - selectSql: optional custom SELECT (defaults to SELECT * FROM sqliteTable) + */ +function getTableMigrations() { + return [ + { + sqliteTable: 'users', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'username', dest: 'username' }, + { src: 'email', dest: 'email' }, + { src: 'password_hash', dest: 'password_hash' }, + { src: 'role', dest: 'role' }, + { src: 'is_active', dest: 'is_active', transform: v => v === 1 || v === true }, + { src: 'created_at', dest: 'created_at' }, + { src: 'last_login', dest: 'last_login' }, + { src: 'user_group', dest: 'user_group' }, + { src: 'bu_teams', dest: 'bu_teams' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'sessions', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'session_id', dest: 'session_id' }, + { src: 'user_id', dest: 'user_id' }, + { src: 'expires_at', dest: 'expires_at' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'cves', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'cve_id', dest: 'cve_id' }, + { src: 'vendor', dest: 'vendor' }, + { src: 'severity', dest: 'severity' }, + { src: 'description', dest: 'description' }, + { src: 'published_date', dest: 'published_date' }, + { src: 'status', dest: 'status' }, + { src: 'created_at', dest: 'created_at' }, + { src: 'updated_at', dest: 'updated_at' }, + { src: 'created_by', dest: 'created_by' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'documents', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'cve_id', dest: 'cve_id' }, + { src: 'vendor', dest: 'vendor' }, + { src: 'name', dest: 'name' }, + { src: 'type', dest: 'type' }, + { src: 'file_path', dest: 'file_path' }, + { src: 'file_size', dest: 'file_size' }, + { src: 'mime_type', dest: 'mime_type' }, + { src: 'uploaded_at', dest: 'uploaded_at' }, + { src: 'notes', dest: 'notes' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'required_documents', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'vendor', dest: 'vendor' }, + { src: 'document_type', dest: 'document_type' }, + { src: 'is_mandatory', dest: 'is_mandatory', transform: v => v === 1 || v === true }, + { src: 'description', dest: 'description' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'jira_tickets', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'cve_id', dest: 'cve_id' }, + { src: 'vendor', dest: 'vendor' }, + { src: 'ticket_key', dest: 'ticket_key' }, + { src: 'url', dest: 'url' }, + { src: 'summary', dest: 'summary' }, + { src: 'status', dest: 'status' }, + { src: 'created_at', dest: 'created_at' }, + { src: 'updated_at', dest: 'updated_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'archer_tickets', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'exc_number', dest: 'exc_number' }, + { src: 'archer_url', dest: 'archer_url' }, + { src: 'status', dest: 'status' }, + { src: 'cve_id', dest: 'cve_id' }, + { src: 'vendor', dest: 'vendor' }, + { src: 'created_at', dest: 'created_at' }, + { src: 'updated_at', dest: 'updated_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'knowledge_base', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'title', dest: 'title' }, + { src: 'slug', dest: 'slug' }, + { src: 'description', dest: 'description' }, + { src: 'category', dest: 'category' }, + { src: 'file_path', dest: 'file_path' }, + { src: 'file_name', dest: 'file_name' }, + { src: 'file_type', dest: 'file_type' }, + { src: 'file_size', dest: 'file_size' }, + { src: 'created_at', dest: 'created_at' }, + { src: 'updated_at', dest: 'updated_at' }, + { src: 'created_by', dest: 'created_by' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'audit_logs', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'user_id', dest: 'user_id' }, + { src: 'username', dest: 'username' }, + { src: 'action', dest: 'action' }, + { src: 'entity_type', dest: 'entity_type' }, + { src: 'entity_id', dest: 'entity_id' }, + { src: 'details', dest: 'details' }, + { src: 'ip_address', dest: 'ip_address' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'compliance_uploads', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'filename', dest: 'filename' }, + { src: 'report_date', dest: 'report_date' }, + { src: 'uploaded_by', dest: 'uploaded_by' }, + { src: 'uploaded_at', dest: 'uploaded_at' }, + { src: 'new_count', dest: 'new_count' }, + { src: 'resolved_count', dest: 'resolved_count' }, + { src: 'recurring_count', dest: 'recurring_count' }, + { src: 'summary_json', dest: 'summary_json' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'compliance_items', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'upload_id', dest: 'upload_id' }, + { src: 'hostname', dest: 'hostname' }, + { src: 'ip_address', dest: 'ip_address' }, + { src: 'device_type', dest: 'device_type' }, + { src: 'team', dest: 'team' }, + { src: 'metric_id', dest: 'metric_id' }, + { src: 'metric_desc', dest: 'metric_desc' }, + { src: 'category', dest: 'category' }, + { src: 'extra_json', dest: 'extra_json' }, + { src: 'status', dest: 'status' }, + { src: 'first_seen_upload_id', dest: 'first_seen_upload_id' }, + { src: 'resolved_upload_id', dest: 'resolved_upload_id' }, + { src: 'seen_count', dest: 'seen_count' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'compliance_notes', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'hostname', dest: 'hostname' }, + { src: 'metric_id', dest: 'metric_id' }, + { src: 'note', dest: 'note' }, + { src: 'group_id', dest: 'group_id' }, + { src: 'created_by', dest: 'created_by' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_counts_history', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'open_count', dest: 'open_count' }, + { src: 'closed_count', dest: 'closed_count' }, + { src: 'recorded_at', dest: 'recorded_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_finding_archives', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'finding_id', dest: 'finding_id' }, + { src: 'finding_title', dest: 'finding_title' }, + { src: 'host_name', dest: 'host_name' }, + { src: 'ip_address', dest: 'ip_address' }, + { src: 'current_state', dest: 'current_state' }, + { src: 'last_severity', dest: 'last_severity' }, + { src: 'first_archived_at', dest: 'first_archived_at' }, + { src: 'last_transition_at', dest: 'last_transition_at' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_archive_transitions', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'archive_id', dest: 'archive_id' }, + { src: 'from_state', dest: 'from_state' }, + { src: 'to_state', dest: 'to_state' }, + { src: 'severity_at_transition', dest: 'severity_at_transition' }, + { src: 'reason', dest: 'reason' }, + { src: 'transitioned_at', dest: 'transitioned_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_sync_anomaly_log', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'sync_timestamp', dest: 'sync_timestamp' }, + { src: 'open_count_delta', dest: 'open_count_delta' }, + { src: 'closed_count_delta', dest: 'closed_count_delta' }, + { src: 'newly_archived_count', dest: 'newly_archived_count' }, + { src: 'returned_count', dest: 'returned_count' }, + { src: 'classification_json', dest: 'classification_json' }, + { src: 'return_classification_json', dest: 'return_classification_json' }, + { src: 'is_significant', dest: 'is_significant', transform: v => v === 1 || v === true }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_finding_bu_history', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'finding_id', dest: 'finding_id' }, + { src: 'finding_title', dest: 'finding_title' }, + { src: 'host_name', dest: 'host_name' }, + { src: 'previous_bu', dest: 'previous_bu' }, + { src: 'new_bu', dest: 'new_bu' }, + { src: 'detected_at', dest: 'detected_at' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'atlas_action_plans_cache', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'host_id', dest: 'host_id' }, + { src: 'has_action_plan', dest: 'has_action_plan', transform: v => v === 1 || v === true }, + { src: 'plan_count', dest: 'plan_count' }, + { src: 'plans_json', dest: 'plans_json' }, + { src: 'synced_at', dest: 'synced_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_fp_submissions', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'user_id', dest: 'user_id' }, + { src: 'username', dest: 'username' }, + { src: 'ivanti_workflow_batch_id', dest: 'ivanti_workflow_batch_id' }, + { src: 'ivanti_generated_id', dest: 'ivanti_generated_id' }, + { src: 'ivanti_workflow_batch_uuid', dest: 'ivanti_workflow_batch_uuid' }, + { src: 'workflow_name', dest: 'workflow_name' }, + { src: 'reason', dest: 'reason' }, + { src: 'description', dest: 'description' }, + { src: 'expiration_date', dest: 'expiration_date' }, + { src: 'scope_override', dest: 'scope_override' }, + { src: 'finding_ids_json', dest: 'finding_ids_json' }, + { src: 'queue_item_ids_json', dest: 'queue_item_ids_json' }, + { src: 'attachment_count', dest: 'attachment_count' }, + { src: 'attachment_results_json', dest: 'attachment_results_json' }, + { src: 'status', dest: 'status' }, + { src: 'lifecycle_status', dest: 'lifecycle_status' }, + { src: 'error_message', dest: 'error_message' }, + { src: 'created_at', dest: 'created_at' }, + { src: 'updated_at', dest: 'updated_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_fp_submission_history', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'submission_id', dest: 'submission_id' }, + { src: 'user_id', dest: 'user_id' }, + { src: 'username', dest: 'username' }, + { src: 'change_type', dest: 'change_type' }, + { src: 'change_details_json', dest: 'change_details_json' }, + { src: 'created_at', dest: 'created_at' }, + ], + conflict: '(id) DO NOTHING', + }, + { + sqliteTable: 'ivanti_todo_queue', + columns: [ + { src: 'id', dest: 'id' }, + { src: 'user_id', dest: 'user_id' }, + { src: 'finding_id', dest: 'finding_id' }, + { src: 'finding_title', dest: 'finding_title' }, + { src: 'cves_json', dest: 'cves_json' }, + { src: 'ip_address', dest: 'ip_address' }, + { src: 'hostname', dest: 'hostname' }, + { src: 'vendor', dest: 'vendor' }, + { src: 'workflow_type', dest: 'workflow_type' }, + { src: 'status', dest: 'status' }, + { src: 'created_at', dest: 'created_at' }, + { src: 'updated_at', dest: 'updated_at' }, + ], + conflict: '(id) DO NOTHING', + }, + ]; +} + +// --------------------------------------------------------------------------- +// Main migration logic +// --------------------------------------------------------------------------- +async function migrate() { + console.log('╔════════════════════════════════════════════════════════╗'); + console.log('║ CVE Dashboard — SQLite → PostgreSQL Migration ║'); + console.log('╚════════════════════════════════════════════════════════╝\n'); + + // Open SQLite in READ-ONLY mode + const sqliteDb = new sqlite3.Database(SQLITE_PATH, sqlite3.OPEN_READONLY, (err) => { + if (err) { + console.error('ERROR: Failed to open SQLite database:', err.message); + process.exit(1); + } + }); + console.log(`✓ Opened SQLite database (read-only): ${SQLITE_PATH}`); + + // Connect to Postgres + const pool = new Pool({ + connectionString: DATABASE_URL, + max: 5, + idleTimeoutMillis: 30000, + connectionTimeoutMillis: 10000, + }); + + try { + await pool.query('SELECT NOW()'); + console.log('✓ Connected to PostgreSQL'); + } catch (err) { + console.error('ERROR: Failed to connect to PostgreSQL:', err.message); + sqliteDb.close(); + process.exit(1); + } + + // Step 1: Run schema DDL + console.log('\n── Step 1: Creating schema (idempotent) ──'); + try { + const schemaSql = fs.readFileSync(SCHEMA_PATH, 'utf8'); + await pool.query(schemaSql); + console.log('✓ Schema created/verified'); + } catch (err) { + console.error('ERROR: Schema creation failed:', err.message); + await cleanup(sqliteDb, pool); + process.exit(1); + } + + // Step 2: Copy simple tables + console.log('\n── Step 2: Copying tables ──'); + const migrations = getTableMigrations(); + const migrationResults = {}; + + for (const migration of migrations) { + const tableName = migration.pgTable || migration.sqliteTable; + try { + // Check if table exists in SQLite + const tableCheck = await sqliteGet( + sqliteDb, + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + [migration.sqliteTable] + ); + + if (!tableCheck) { + console.log(` ⚠ ${migration.sqliteTable} — table not found in SQLite, skipping`); + migrationResults[tableName] = { source: 0, dest: 0, skipped: true }; + continue; + } + + // Read all rows from SQLite + const selectSql = migration.selectSql || `SELECT * FROM ${migration.sqliteTable}`; + const sourceRows = await sqliteAll(sqliteDb, selectSql); + + if (sourceRows.length === 0) { + console.log(` ○ ${tableName} — 0 rows (empty table)`); + migrationResults[tableName] = { source: 0, dest: 0 }; + continue; + } + + // Transform rows + const destColumns = migration.columns.map(c => c.dest); + const transformedRows = sourceRows.map(row => { + return migration.columns.map(col => { + let value = row[col.src]; + if (value === undefined) value = null; + if (col.transform && value !== null) { + value = col.transform(value); + } + return value; + }); + }); + + // Insert into Postgres + const inserted = await batchInsert( + pool, + tableName, + destColumns, + transformedRows, + migration.conflict + ); + + console.log(` ✓ ${tableName} — ${inserted} rows copied`); + migrationResults[tableName] = { source: sourceRows.length, dest: inserted }; + } catch (err) { + console.error(` ✗ ${tableName} — ERROR: ${err.message}`); + migrationResults[tableName] = { source: 0, dest: 0, error: err.message }; + } + } + + // Reset sequences for SERIAL columns after bulk insert with explicit IDs + console.log('\n── Step 2b: Resetting sequences ──'); + const serialTables = [ + 'users', 'sessions', 'cves', 'documents', 'required_documents', + 'jira_tickets', 'archer_tickets', 'knowledge_base', 'audit_logs', + 'compliance_uploads', 'compliance_items', 'compliance_notes', + 'ivanti_counts_history', 'ivanti_finding_archives', + 'ivanti_archive_transitions', 'ivanti_sync_anomaly_log', + 'ivanti_finding_bu_history', 'atlas_action_plans_cache', + 'ivanti_fp_submissions', 'ivanti_fp_submission_history', + 'ivanti_todo_queue', + ]; + + for (const table of serialTables) { + try { + await pool.query(` + SELECT setval(pg_get_serial_sequence('${table}', 'id'), + COALESCE((SELECT MAX(id) FROM ${table}), 0) + 1, false) + `); + } catch (err) { + // Non-fatal — sequence may not exist for some tables + console.log(` ⚠ Could not reset sequence for ${table}: ${err.message}`); + } + } + console.log('✓ Sequences reset'); + + // Step 3: Migrate findings from JSON blob + console.log('\n── Step 3: Migrating findings from JSON blob ──'); + try { + const cacheRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_findings_cache WHERE id = 1'); + + if (!cacheRow || !cacheRow.findings_json) { + console.log(' ⚠ No findings_json data found in ivanti_findings_cache'); + } else { + let findings; + try { + findings = JSON.parse(cacheRow.findings_json); + } catch (parseErr) { + console.error(' ✗ Failed to parse findings_json:', parseErr.message); + findings = []; + } + + if (Array.isArray(findings) && findings.length > 0) { + console.log(` Parsing ${findings.length} findings from JSON blob...`); + + // Extract and insert findings + const BATCH_SIZE = 100; + let insertedCount = 0; + + for (let i = 0; i < findings.length; i += BATCH_SIZE) { + const batch = findings.slice(i, i + BATCH_SIZE); + const values = []; + const placeholders = []; + + batch.forEach((rawFinding, idx) => { + const f = extractFinding(rawFinding); + const offset = idx * 18; + values.push( + f.id, + f.hostId, + f.title, + f.severity, + f.vrrGroup, + f.hostName, + f.ipAddress, + f.dns, + f.status, + f.slaStatus, + f.dueDate, + f.lastFoundOn, + f.buOwnership, + f.cves, + f.workflow ? f.workflow.id : null, + f.workflow ? f.workflow.state : null, + f.workflow ? f.workflow.type : null, + 'open' // state = open for all findings from cache + ); + placeholders.push( + `($${offset+1}, $${offset+2}, $${offset+3}, $${offset+4}, $${offset+5}, ` + + `$${offset+6}, $${offset+7}, $${offset+8}, $${offset+9}, $${offset+10}, ` + + `$${offset+11}, $${offset+12}, $${offset+13}, $${offset+14}, $${offset+15}, ` + + `$${offset+16}, $${offset+17}, $${offset+18})` + ); + }); + + await pool.query(` + INSERT INTO ivanti_findings ( + id, host_id, title, severity, vrr_group, + host_name, ip_address, dns, status, sla_status, + due_date, last_found_on, bu_ownership, cves, + workflow_id, workflow_state, workflow_type, state + ) + VALUES ${placeholders.join(', ')} + ON CONFLICT (id) DO NOTHING + `, values); + + insertedCount += batch.length; + } + + console.log(` ✓ ivanti_findings — ${insertedCount} findings inserted (state='open')`); + migrationResults['ivanti_findings'] = { source: findings.length, dest: insertedCount }; + } else { + console.log(' ○ findings_json is empty or not an array'); + migrationResults['ivanti_findings'] = { source: 0, dest: 0 }; + } + } + } catch (err) { + console.error(` ✗ Findings migration ERROR: ${err.message}`); + migrationResults['ivanti_findings'] = { source: 0, dest: 0, error: err.message }; + } + + // Step 4: Merge notes into ivanti_findings.note + console.log('\n── Step 4: Merging finding notes ──'); + try { + const notes = await sqliteAll(sqliteDb, 'SELECT finding_id, note FROM ivanti_finding_notes'); + + if (notes.length === 0) { + console.log(' ○ No finding notes to merge'); + } else { + let mergedCount = 0; + for (const { finding_id, note } of notes) { + if (!finding_id || !note) continue; + const result = await pool.query( + `UPDATE ivanti_findings SET note = $1 WHERE id = $2`, + [note, finding_id] + ); + if (result.rowCount > 0) mergedCount++; + } + console.log(` ✓ Merged ${mergedCount}/${notes.length} notes into ivanti_findings.note`); + } + } catch (err) { + console.error(` ✗ Notes merge ERROR: ${err.message}`); + } + + // Step 5: Merge overrides into ivanti_findings.override_host_name / override_dns + console.log('\n── Step 5: Merging finding overrides ──'); + try { + const overrides = await sqliteAll( + sqliteDb, + 'SELECT finding_id, field, value FROM ivanti_finding_overrides' + ); + + if (overrides.length === 0) { + console.log(' ○ No finding overrides to merge'); + } else { + let mergedCount = 0; + for (const { finding_id, field, value } of overrides) { + if (!finding_id || !field) continue; + + let pgColumn; + if (field === 'host_name' || field === 'hostName' || field === 'override_host_name') { + pgColumn = 'override_host_name'; + } else if (field === 'dns' || field === 'override_dns') { + pgColumn = 'override_dns'; + } else { + // Unknown field — skip + continue; + } + + const result = await pool.query( + `UPDATE ivanti_findings SET ${pgColumn} = $1 WHERE id = $2`, + [value, finding_id] + ); + if (result.rowCount > 0) mergedCount++; + } + console.log(` ✓ Merged ${mergedCount}/${overrides.length} overrides into ivanti_findings`); + } + } catch (err) { + console.error(` ✗ Overrides merge ERROR: ${err.message}`); + } + + // Step 6: Populate ivanti_sync_state from ivanti_findings_cache metadata + console.log('\n── Step 6: Populating sync state and counts cache ──'); + try { + const cacheRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_findings_cache WHERE id = 1'); + + if (cacheRow) { + await pool.query(` + UPDATE ivanti_sync_state SET + total = $1, + synced_at = $2, + sync_status = $3, + error_message = $4 + WHERE id = 1 + `, [ + cacheRow.total || 0, + cacheRow.synced_at || null, + cacheRow.sync_status || 'never', + cacheRow.error_message || null, + ]); + console.log(' ✓ ivanti_sync_state updated from ivanti_findings_cache metadata'); + } + + // Populate ivanti_counts_cache + const countsRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_counts_cache WHERE id = 1'); + if (countsRow) { + await pool.query(` + UPDATE ivanti_counts_cache SET + open_count = $1, + closed_count = $2, + synced_at = $3, + fp_workflow_counts_json = $4, + fp_id_counts_json = $5 + WHERE id = 1 + `, [ + countsRow.open_count || 0, + countsRow.closed_count || 0, + countsRow.synced_at || null, + countsRow.fp_workflow_counts_json || '{}', + countsRow.fp_id_counts_json || '{}', + ]); + console.log(' ✓ ivanti_counts_cache updated'); + } + } catch (err) { + console.error(` ✗ Sync state/counts migration ERROR: ${err.message}`); + } + + // Step 7: Verification — compare row counts + console.log('\n── Step 7: Verification ──'); + console.log(''); + console.log('┌─────────────────────────────────┬──────────┬──────────┬────────┐'); + console.log('│ Table │ SQLite │ Postgres │ Status │'); + console.log('├─────────────────────────────────┼──────────┼──────────┼────────┤'); + + let hasDiscrepancy = false; + const verificationTables = [ + ...migrations.map(m => ({ sqlite: m.sqliteTable, pg: m.pgTable || m.sqliteTable })), + { sqlite: null, pg: 'ivanti_findings', special: true }, + ]; + + for (const { sqlite: sqliteTable, pg: pgTable, special } of verificationTables) { + let sqliteCount = 0; + let pgCount = 0; + + try { + if (special && pgTable === 'ivanti_findings') { + // For findings, source count is from the JSON blob + const cacheRow = await sqliteGet(sqliteDb, 'SELECT findings_json FROM ivanti_findings_cache WHERE id = 1'); + if (cacheRow && cacheRow.findings_json) { + try { + const parsed = JSON.parse(cacheRow.findings_json); + sqliteCount = Array.isArray(parsed) ? parsed.length : 0; + } catch (e) { + sqliteCount = 0; + } + } + } else if (sqliteTable) { + const tableExists = await sqliteGet( + sqliteDb, + `SELECT name FROM sqlite_master WHERE type='table' AND name=?`, + [sqliteTable] + ); + if (tableExists) { + const countRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) as cnt FROM ${sqliteTable}`); + sqliteCount = countRow ? countRow.cnt : 0; + } + } + + const pgCountRow = await pool.query(`SELECT COUNT(*) as cnt FROM ${pgTable}`); + pgCount = parseInt(pgCountRow.rows[0].cnt, 10); + } catch (err) { + // Table might not exist in one or both + } + + const status = pgCount >= sqliteCount ? ' OK ' : ' WARN '; + if (pgCount < sqliteCount) hasDiscrepancy = true; + + const tableDisplay = (pgTable || '').padEnd(31); + const srcDisplay = String(sqliteCount).padStart(6); + const destDisplay = String(pgCount).padStart(6); + + console.log(`│ ${tableDisplay} │ ${srcDisplay} │ ${destDisplay} │ ${status} │`); + } + + console.log('└─────────────────────────────────┴──────────┴──────────┴────────┘'); + + if (hasDiscrepancy) { + console.log('\n⚠ WARNING: Some tables have fewer rows in Postgres than SQLite.'); + console.log(' This may be due to ON CONFLICT DO NOTHING skipping existing rows,'); + console.log(' or foreign key constraints preventing insertion.'); + } + + // Cleanup + await cleanup(sqliteDb, pool); + + console.log('\n════════════════════════════════════════════════════════'); + if (hasDiscrepancy) { + console.log('Migration completed with warnings. Review discrepancies above.'); + } else { + console.log('✓ Migration completed successfully!'); + } + console.log('════════════════════════════════════════════════════════\n'); + + process.exit(hasDiscrepancy ? 0 : 0); // Exit 0 even with warnings (data is safe) +} + +// --------------------------------------------------------------------------- +// Cleanup helper +// --------------------------------------------------------------------------- +function cleanup(sqliteDb, pool) { + return new Promise((resolve) => { + sqliteDb.close((err) => { + if (err) console.error('Warning: Error closing SQLite:', err.message); + pool.end() + .then(() => resolve()) + .catch(() => resolve()); + }); + }); +} + +// --------------------------------------------------------------------------- +// Entry point +// --------------------------------------------------------------------------- +migrate().catch((err) => { + console.error('\n✗ FATAL ERROR:', err.message); + console.error(err.stack); + process.exit(1); +}); diff --git a/frontend/src/components/pages/ReportingPage.js b/frontend/src/components/pages/ReportingPage.js index 2c4d8f8..c811aee 100644 --- a/frontend/src/components/pages/ReportingPage.js +++ b/frontend/src/components/pages/ReportingPage.js @@ -5052,8 +5052,12 @@ export default function VulnerabilityTriagePage({ filterDate, filterEXC }) { const fetchCounts = async () => { setCountsLoading(true); try { - // Fetch global counts — open count is overridden by client-side scoped findings - const res = await fetch(`${API_BASE}/ivanti/findings/counts`, { credentials: 'include' }); + // Fetch counts from server — Postgres provides per-BU open+closed counts + const teamsParam = getActiveTeamsParam(); + const url = teamsParam + ? `${API_BASE}/ivanti/findings/counts?teams=${encodeURIComponent(teamsParam)}` + : `${API_BASE}/ivanti/findings/counts`; + const res = await fetch(url, { credentials: 'include' }); const data = await res.json(); if (res.ok) setStatusCounts({ open: data.open ?? 0, closed: data.closed ?? 0 }); } catch (e) { @@ -5182,6 +5186,11 @@ export default function VulnerabilityTriagePage({ filterDate, filterEXC }) { fetchCardStatus(); }, []); // eslint-disable-line + // Re-fetch counts when admin scope changes (per-BU counts from Postgres) + useEffect(() => { + fetchCounts(); + }, [adminScope]); // eslint-disable-line + // Set/clear a single column filter const setColFilter = useCallback((colKey, vals) => { setColumnFilters((prev) => { @@ -5662,21 +5671,16 @@ export default function VulnerabilityTriagePage({ filterDate, filterEXC }) {
{metricsTab === 'ivanti' && (
- {/* Open vs Closed donut — open from scoped findings; closed only meaningful when unscoped */} + {/* Open vs Closed donut — per-BU counts from Postgres */}
Open vs Closed
- {getActiveTeamsParam() && ( -
- Closed count unavailable when filtered by BU -
- )}
{/* Divider */}