feat(postgres): data migration + per-BU closed counts in frontend
- Create backend/scripts/migrate-to-postgres.js (one-time SQLite→Postgres copy) - Successfully migrated: 6 users, 21 CVEs, 6307 findings, 20965 compliance items, 138 archives, 67 atlas plans, all notes/overrides merged - All 22 tables verified with matching row counts - Frontend StatusDonut now uses server-provided per-BU counts (no more N/A) - Counts endpoint called with teams param on scope change - Re-fetch counts when admin scope toggle changes
This commit is contained in:
Jordan Ramos
928
backend/scripts/migrate-to-postgres.js
Normal file
928
backend/scripts/migrate-to-postgres.js
Normal file
@@ -0,0 +1,928 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* migrate-to-postgres.js — Data Migration Script
|
||||
*
|
||||
* Copies all data from the SQLite database (cve_database.db) to PostgreSQL.
|
||||
* The SQLite file is opened READ-ONLY and is never modified.
|
||||
*
|
||||
* Special handling:
|
||||
* - ivanti_findings_cache.findings_json → individual rows in ivanti_findings
|
||||
* - ivanti_finding_notes → merged into ivanti_findings.note column
|
||||
* - ivanti_finding_overrides → merged into ivanti_findings.override_host_name / override_dns
|
||||
* - ivanti_sync_state and ivanti_counts_cache → populated from ivanti_findings_cache metadata
|
||||
*
|
||||
* Type conversions:
|
||||
* - SQLite 0/1 integers → Postgres boolean
|
||||
* - SQLite DATETIME strings → Postgres TIMESTAMPTZ (passed as-is)
|
||||
* - SQLite NULL → Postgres NULL
|
||||
*
|
||||
* Uses ON CONFLICT DO NOTHING for idempotency (safe to re-run).
|
||||
*
|
||||
* Usage:
|
||||
* node backend/scripts/migrate-to-postgres.js
|
||||
*
|
||||
* Requires:
|
||||
* - DATABASE_URL env var (or .env file in backend/)
|
||||
* - SQLite database at backend/cve_database.db
|
||||
*/
|
||||
|
||||
const path = require('path');
|
||||
require('dotenv').config({ path: path.join(__dirname, '..', '.env') });
|
||||
|
||||
const fs = require('fs');
|
||||
const sqlite3 = require('sqlite3').verbose();
|
||||
const { Pool } = require('pg');
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Configuration
|
||||
// ---------------------------------------------------------------------------
|
||||
const SQLITE_PATH = path.join(__dirname, '..', 'cve_database.db');
|
||||
const SCHEMA_PATH = path.join(__dirname, '..', 'db-schema.sql');
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
|
||||
if (!DATABASE_URL) {
|
||||
console.error('ERROR: DATABASE_URL environment variable is not set.');
|
||||
console.error('Expected format: postgresql://user:password@host:port/database');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!fs.existsSync(SQLITE_PATH)) {
|
||||
console.error(`ERROR: SQLite database not found at ${SQLITE_PATH}`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// SQLite helpers
|
||||
// ---------------------------------------------------------------------------
|
||||
function sqliteAll(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.all(sql, params, (err, rows) => {
|
||||
if (err) reject(err);
|
||||
else resolve(rows || []);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
function sqliteGet(db, sql, params = []) {
|
||||
return new Promise((resolve, reject) => {
|
||||
db.get(sql, params, (err, row) => {
|
||||
if (err) reject(err);
|
||||
else resolve(row);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extract finding fields from raw JSON object (mirrors ivantiFindings.js)
|
||||
// ---------------------------------------------------------------------------
|
||||
function extractFinding(f) {
|
||||
const rawDueDate = f.statusEmbedded?.dueDate || f.dueDate || '';
|
||||
const dueDate = rawDueDate ? rawDueDate.split('T')[0] : null;
|
||||
|
||||
const buOwnership = f.assetCustomAttributes?.['1550_host_1']?.[0]
|
||||
|| f.buOwnership || f.bu_ownership || '';
|
||||
|
||||
const cves = Array.isArray(f.cves)
|
||||
? f.cves
|
||||
: (f.vulnerabilities?.vulnInfoList || []).map(v => v.cve).filter(Boolean);
|
||||
|
||||
// Workflow extraction
|
||||
let workflow = null;
|
||||
if (f.workflow && typeof f.workflow === 'object') {
|
||||
workflow = {
|
||||
id: f.workflow.id || '',
|
||||
state: f.workflow.state || '',
|
||||
type: f.workflow.type || 'FP',
|
||||
};
|
||||
} else if (f.workflowDistribution) {
|
||||
const wfDist = f.workflowDistribution || {};
|
||||
const fpBuckets = [
|
||||
...(wfDist.actionableWorkflows || []),
|
||||
...(wfDist.requestedWorkflows || []),
|
||||
...(wfDist.reworkedWorkflows || []),
|
||||
...(wfDist.rejectedWorkflows || []),
|
||||
...(wfDist.expiredWorkflows || []),
|
||||
...(wfDist.approvedWorkflows || []),
|
||||
].filter(w => (w.generatedId || '').startsWith('FP#'));
|
||||
const fpEntry = fpBuckets[0] || null;
|
||||
if (fpEntry) {
|
||||
workflow = {
|
||||
id: fpEntry.generatedId || '',
|
||||
state: fpEntry.state || '',
|
||||
type: 'FP',
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
id: String(f.id),
|
||||
hostId: f.hostId || f.host?.hostId || null,
|
||||
title: f.title || '',
|
||||
severity: typeof f.severity === 'number' ? f.severity : parseFloat(f.severity) || 0,
|
||||
vrrGroup: f.vrrGroup || f.severityGroup || f.vrr_group || '',
|
||||
hostName: f.hostName || f.host?.hostName || f.host_name || '',
|
||||
ipAddress: f.ipAddress || f.host?.ipAddress || f.ip_address || '',
|
||||
dns: f.dns || f.host?.fqdn || '',
|
||||
status: f.status || '',
|
||||
slaStatus: f.slaStatus || f.sla_status || '',
|
||||
dueDate: dueDate,
|
||||
lastFoundOn: f.lastFoundOn || f.last_found_on || null,
|
||||
buOwnership,
|
||||
cves,
|
||||
workflow,
|
||||
};
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Batch insert helper for Postgres
|
||||
// ---------------------------------------------------------------------------
|
||||
async function batchInsert(pool, tableName, columns, rows, conflictClause = 'DO NOTHING') {
|
||||
if (rows.length === 0) return 0;
|
||||
|
||||
const BATCH_SIZE = 100;
|
||||
let inserted = 0;
|
||||
|
||||
for (let i = 0; i < rows.length; i += BATCH_SIZE) {
|
||||
const batch = rows.slice(i, i + BATCH_SIZE);
|
||||
const values = [];
|
||||
const placeholders = [];
|
||||
|
||||
batch.forEach((row, idx) => {
|
||||
const offset = idx * columns.length;
|
||||
const rowPlaceholders = columns.map((_, colIdx) => `$${offset + colIdx + 1}`);
|
||||
placeholders.push(`(${rowPlaceholders.join(', ')})`);
|
||||
values.push(...row);
|
||||
});
|
||||
|
||||
const sql = `INSERT INTO ${tableName} (${columns.join(', ')})
|
||||
VALUES ${placeholders.join(', ')}
|
||||
ON CONFLICT ${conflictClause}`;
|
||||
|
||||
await pool.query(sql, values);
|
||||
inserted += batch.length;
|
||||
}
|
||||
|
||||
return inserted;
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Table migration definitions
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
/**
|
||||
* Each entry defines how to copy a SQLite table to Postgres.
|
||||
* - sqliteTable: source table name
|
||||
* - pgTable: destination table name (defaults to sqliteTable)
|
||||
* - columns: array of { src, dest, transform } objects
|
||||
* - conflict: ON CONFLICT clause (default: DO NOTHING)
|
||||
* - selectSql: optional custom SELECT (defaults to SELECT * FROM sqliteTable)
|
||||
*/
|
||||
function getTableMigrations() {
|
||||
return [
|
||||
{
|
||||
sqliteTable: 'users',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'username', dest: 'username' },
|
||||
{ src: 'email', dest: 'email' },
|
||||
{ src: 'password_hash', dest: 'password_hash' },
|
||||
{ src: 'role', dest: 'role' },
|
||||
{ src: 'is_active', dest: 'is_active', transform: v => v === 1 || v === true },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'last_login', dest: 'last_login' },
|
||||
{ src: 'user_group', dest: 'user_group' },
|
||||
{ src: 'bu_teams', dest: 'bu_teams' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'sessions',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'session_id', dest: 'session_id' },
|
||||
{ src: 'user_id', dest: 'user_id' },
|
||||
{ src: 'expires_at', dest: 'expires_at' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'cves',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'cve_id', dest: 'cve_id' },
|
||||
{ src: 'vendor', dest: 'vendor' },
|
||||
{ src: 'severity', dest: 'severity' },
|
||||
{ src: 'description', dest: 'description' },
|
||||
{ src: 'published_date', dest: 'published_date' },
|
||||
{ src: 'status', dest: 'status' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'updated_at', dest: 'updated_at' },
|
||||
{ src: 'created_by', dest: 'created_by' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'documents',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'cve_id', dest: 'cve_id' },
|
||||
{ src: 'vendor', dest: 'vendor' },
|
||||
{ src: 'name', dest: 'name' },
|
||||
{ src: 'type', dest: 'type' },
|
||||
{ src: 'file_path', dest: 'file_path' },
|
||||
{ src: 'file_size', dest: 'file_size' },
|
||||
{ src: 'mime_type', dest: 'mime_type' },
|
||||
{ src: 'uploaded_at', dest: 'uploaded_at' },
|
||||
{ src: 'notes', dest: 'notes' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'required_documents',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'vendor', dest: 'vendor' },
|
||||
{ src: 'document_type', dest: 'document_type' },
|
||||
{ src: 'is_mandatory', dest: 'is_mandatory', transform: v => v === 1 || v === true },
|
||||
{ src: 'description', dest: 'description' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'jira_tickets',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'cve_id', dest: 'cve_id' },
|
||||
{ src: 'vendor', dest: 'vendor' },
|
||||
{ src: 'ticket_key', dest: 'ticket_key' },
|
||||
{ src: 'url', dest: 'url' },
|
||||
{ src: 'summary', dest: 'summary' },
|
||||
{ src: 'status', dest: 'status' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'updated_at', dest: 'updated_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'archer_tickets',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'exc_number', dest: 'exc_number' },
|
||||
{ src: 'archer_url', dest: 'archer_url' },
|
||||
{ src: 'status', dest: 'status' },
|
||||
{ src: 'cve_id', dest: 'cve_id' },
|
||||
{ src: 'vendor', dest: 'vendor' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'updated_at', dest: 'updated_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'knowledge_base',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'title', dest: 'title' },
|
||||
{ src: 'slug', dest: 'slug' },
|
||||
{ src: 'description', dest: 'description' },
|
||||
{ src: 'category', dest: 'category' },
|
||||
{ src: 'file_path', dest: 'file_path' },
|
||||
{ src: 'file_name', dest: 'file_name' },
|
||||
{ src: 'file_type', dest: 'file_type' },
|
||||
{ src: 'file_size', dest: 'file_size' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'updated_at', dest: 'updated_at' },
|
||||
{ src: 'created_by', dest: 'created_by' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'audit_logs',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'user_id', dest: 'user_id' },
|
||||
{ src: 'username', dest: 'username' },
|
||||
{ src: 'action', dest: 'action' },
|
||||
{ src: 'entity_type', dest: 'entity_type' },
|
||||
{ src: 'entity_id', dest: 'entity_id' },
|
||||
{ src: 'details', dest: 'details' },
|
||||
{ src: 'ip_address', dest: 'ip_address' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'compliance_uploads',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'filename', dest: 'filename' },
|
||||
{ src: 'report_date', dest: 'report_date' },
|
||||
{ src: 'uploaded_by', dest: 'uploaded_by' },
|
||||
{ src: 'uploaded_at', dest: 'uploaded_at' },
|
||||
{ src: 'new_count', dest: 'new_count' },
|
||||
{ src: 'resolved_count', dest: 'resolved_count' },
|
||||
{ src: 'recurring_count', dest: 'recurring_count' },
|
||||
{ src: 'summary_json', dest: 'summary_json' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'compliance_items',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'upload_id', dest: 'upload_id' },
|
||||
{ src: 'hostname', dest: 'hostname' },
|
||||
{ src: 'ip_address', dest: 'ip_address' },
|
||||
{ src: 'device_type', dest: 'device_type' },
|
||||
{ src: 'team', dest: 'team' },
|
||||
{ src: 'metric_id', dest: 'metric_id' },
|
||||
{ src: 'metric_desc', dest: 'metric_desc' },
|
||||
{ src: 'category', dest: 'category' },
|
||||
{ src: 'extra_json', dest: 'extra_json' },
|
||||
{ src: 'status', dest: 'status' },
|
||||
{ src: 'first_seen_upload_id', dest: 'first_seen_upload_id' },
|
||||
{ src: 'resolved_upload_id', dest: 'resolved_upload_id' },
|
||||
{ src: 'seen_count', dest: 'seen_count' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'compliance_notes',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'hostname', dest: 'hostname' },
|
||||
{ src: 'metric_id', dest: 'metric_id' },
|
||||
{ src: 'note', dest: 'note' },
|
||||
{ src: 'group_id', dest: 'group_id' },
|
||||
{ src: 'created_by', dest: 'created_by' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_counts_history',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'open_count', dest: 'open_count' },
|
||||
{ src: 'closed_count', dest: 'closed_count' },
|
||||
{ src: 'recorded_at', dest: 'recorded_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_finding_archives',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'finding_id', dest: 'finding_id' },
|
||||
{ src: 'finding_title', dest: 'finding_title' },
|
||||
{ src: 'host_name', dest: 'host_name' },
|
||||
{ src: 'ip_address', dest: 'ip_address' },
|
||||
{ src: 'current_state', dest: 'current_state' },
|
||||
{ src: 'last_severity', dest: 'last_severity' },
|
||||
{ src: 'first_archived_at', dest: 'first_archived_at' },
|
||||
{ src: 'last_transition_at', dest: 'last_transition_at' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_archive_transitions',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'archive_id', dest: 'archive_id' },
|
||||
{ src: 'from_state', dest: 'from_state' },
|
||||
{ src: 'to_state', dest: 'to_state' },
|
||||
{ src: 'severity_at_transition', dest: 'severity_at_transition' },
|
||||
{ src: 'reason', dest: 'reason' },
|
||||
{ src: 'transitioned_at', dest: 'transitioned_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_sync_anomaly_log',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'sync_timestamp', dest: 'sync_timestamp' },
|
||||
{ src: 'open_count_delta', dest: 'open_count_delta' },
|
||||
{ src: 'closed_count_delta', dest: 'closed_count_delta' },
|
||||
{ src: 'newly_archived_count', dest: 'newly_archived_count' },
|
||||
{ src: 'returned_count', dest: 'returned_count' },
|
||||
{ src: 'classification_json', dest: 'classification_json' },
|
||||
{ src: 'return_classification_json', dest: 'return_classification_json' },
|
||||
{ src: 'is_significant', dest: 'is_significant', transform: v => v === 1 || v === true },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_finding_bu_history',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'finding_id', dest: 'finding_id' },
|
||||
{ src: 'finding_title', dest: 'finding_title' },
|
||||
{ src: 'host_name', dest: 'host_name' },
|
||||
{ src: 'previous_bu', dest: 'previous_bu' },
|
||||
{ src: 'new_bu', dest: 'new_bu' },
|
||||
{ src: 'detected_at', dest: 'detected_at' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'atlas_action_plans_cache',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'host_id', dest: 'host_id' },
|
||||
{ src: 'has_action_plan', dest: 'has_action_plan', transform: v => v === 1 || v === true },
|
||||
{ src: 'plan_count', dest: 'plan_count' },
|
||||
{ src: 'plans_json', dest: 'plans_json' },
|
||||
{ src: 'synced_at', dest: 'synced_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_fp_submissions',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'user_id', dest: 'user_id' },
|
||||
{ src: 'username', dest: 'username' },
|
||||
{ src: 'ivanti_workflow_batch_id', dest: 'ivanti_workflow_batch_id' },
|
||||
{ src: 'ivanti_generated_id', dest: 'ivanti_generated_id' },
|
||||
{ src: 'ivanti_workflow_batch_uuid', dest: 'ivanti_workflow_batch_uuid' },
|
||||
{ src: 'workflow_name', dest: 'workflow_name' },
|
||||
{ src: 'reason', dest: 'reason' },
|
||||
{ src: 'description', dest: 'description' },
|
||||
{ src: 'expiration_date', dest: 'expiration_date' },
|
||||
{ src: 'scope_override', dest: 'scope_override' },
|
||||
{ src: 'finding_ids_json', dest: 'finding_ids_json' },
|
||||
{ src: 'queue_item_ids_json', dest: 'queue_item_ids_json' },
|
||||
{ src: 'attachment_count', dest: 'attachment_count' },
|
||||
{ src: 'attachment_results_json', dest: 'attachment_results_json' },
|
||||
{ src: 'status', dest: 'status' },
|
||||
{ src: 'lifecycle_status', dest: 'lifecycle_status' },
|
||||
{ src: 'error_message', dest: 'error_message' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'updated_at', dest: 'updated_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_fp_submission_history',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'submission_id', dest: 'submission_id' },
|
||||
{ src: 'user_id', dest: 'user_id' },
|
||||
{ src: 'username', dest: 'username' },
|
||||
{ src: 'change_type', dest: 'change_type' },
|
||||
{ src: 'change_details_json', dest: 'change_details_json' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
{
|
||||
sqliteTable: 'ivanti_todo_queue',
|
||||
columns: [
|
||||
{ src: 'id', dest: 'id' },
|
||||
{ src: 'user_id', dest: 'user_id' },
|
||||
{ src: 'finding_id', dest: 'finding_id' },
|
||||
{ src: 'finding_title', dest: 'finding_title' },
|
||||
{ src: 'cves_json', dest: 'cves_json' },
|
||||
{ src: 'ip_address', dest: 'ip_address' },
|
||||
{ src: 'hostname', dest: 'hostname' },
|
||||
{ src: 'vendor', dest: 'vendor' },
|
||||
{ src: 'workflow_type', dest: 'workflow_type' },
|
||||
{ src: 'status', dest: 'status' },
|
||||
{ src: 'created_at', dest: 'created_at' },
|
||||
{ src: 'updated_at', dest: 'updated_at' },
|
||||
],
|
||||
conflict: '(id) DO NOTHING',
|
||||
},
|
||||
];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Main migration logic
|
||||
// ---------------------------------------------------------------------------
|
||||
async function migrate() {
|
||||
console.log('╔════════════════════════════════════════════════════════╗');
|
||||
console.log('║ CVE Dashboard — SQLite → PostgreSQL Migration ║');
|
||||
console.log('╚════════════════════════════════════════════════════════╝\n');
|
||||
|
||||
// Open SQLite in READ-ONLY mode
|
||||
const sqliteDb = new sqlite3.Database(SQLITE_PATH, sqlite3.OPEN_READONLY, (err) => {
|
||||
if (err) {
|
||||
console.error('ERROR: Failed to open SQLite database:', err.message);
|
||||
process.exit(1);
|
||||
}
|
||||
});
|
||||
console.log(`✓ Opened SQLite database (read-only): ${SQLITE_PATH}`);
|
||||
|
||||
// Connect to Postgres
|
||||
const pool = new Pool({
|
||||
connectionString: DATABASE_URL,
|
||||
max: 5,
|
||||
idleTimeoutMillis: 30000,
|
||||
connectionTimeoutMillis: 10000,
|
||||
});
|
||||
|
||||
try {
|
||||
await pool.query('SELECT NOW()');
|
||||
console.log('✓ Connected to PostgreSQL');
|
||||
} catch (err) {
|
||||
console.error('ERROR: Failed to connect to PostgreSQL:', err.message);
|
||||
sqliteDb.close();
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Step 1: Run schema DDL
|
||||
console.log('\n── Step 1: Creating schema (idempotent) ──');
|
||||
try {
|
||||
const schemaSql = fs.readFileSync(SCHEMA_PATH, 'utf8');
|
||||
await pool.query(schemaSql);
|
||||
console.log('✓ Schema created/verified');
|
||||
} catch (err) {
|
||||
console.error('ERROR: Schema creation failed:', err.message);
|
||||
await cleanup(sqliteDb, pool);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
// Step 2: Copy simple tables
|
||||
console.log('\n── Step 2: Copying tables ──');
|
||||
const migrations = getTableMigrations();
|
||||
const migrationResults = {};
|
||||
|
||||
for (const migration of migrations) {
|
||||
const tableName = migration.pgTable || migration.sqliteTable;
|
||||
try {
|
||||
// Check if table exists in SQLite
|
||||
const tableCheck = await sqliteGet(
|
||||
sqliteDb,
|
||||
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
||||
[migration.sqliteTable]
|
||||
);
|
||||
|
||||
if (!tableCheck) {
|
||||
console.log(` ⚠ ${migration.sqliteTable} — table not found in SQLite, skipping`);
|
||||
migrationResults[tableName] = { source: 0, dest: 0, skipped: true };
|
||||
continue;
|
||||
}
|
||||
|
||||
// Read all rows from SQLite
|
||||
const selectSql = migration.selectSql || `SELECT * FROM ${migration.sqliteTable}`;
|
||||
const sourceRows = await sqliteAll(sqliteDb, selectSql);
|
||||
|
||||
if (sourceRows.length === 0) {
|
||||
console.log(` ○ ${tableName} — 0 rows (empty table)`);
|
||||
migrationResults[tableName] = { source: 0, dest: 0 };
|
||||
continue;
|
||||
}
|
||||
|
||||
// Transform rows
|
||||
const destColumns = migration.columns.map(c => c.dest);
|
||||
const transformedRows = sourceRows.map(row => {
|
||||
return migration.columns.map(col => {
|
||||
let value = row[col.src];
|
||||
if (value === undefined) value = null;
|
||||
if (col.transform && value !== null) {
|
||||
value = col.transform(value);
|
||||
}
|
||||
return value;
|
||||
});
|
||||
});
|
||||
|
||||
// Insert into Postgres
|
||||
const inserted = await batchInsert(
|
||||
pool,
|
||||
tableName,
|
||||
destColumns,
|
||||
transformedRows,
|
||||
migration.conflict
|
||||
);
|
||||
|
||||
console.log(` ✓ ${tableName} — ${inserted} rows copied`);
|
||||
migrationResults[tableName] = { source: sourceRows.length, dest: inserted };
|
||||
} catch (err) {
|
||||
console.error(` ✗ ${tableName} — ERROR: ${err.message}`);
|
||||
migrationResults[tableName] = { source: 0, dest: 0, error: err.message };
|
||||
}
|
||||
}
|
||||
|
||||
// Reset sequences for SERIAL columns after bulk insert with explicit IDs
|
||||
console.log('\n── Step 2b: Resetting sequences ──');
|
||||
const serialTables = [
|
||||
'users', 'sessions', 'cves', 'documents', 'required_documents',
|
||||
'jira_tickets', 'archer_tickets', 'knowledge_base', 'audit_logs',
|
||||
'compliance_uploads', 'compliance_items', 'compliance_notes',
|
||||
'ivanti_counts_history', 'ivanti_finding_archives',
|
||||
'ivanti_archive_transitions', 'ivanti_sync_anomaly_log',
|
||||
'ivanti_finding_bu_history', 'atlas_action_plans_cache',
|
||||
'ivanti_fp_submissions', 'ivanti_fp_submission_history',
|
||||
'ivanti_todo_queue',
|
||||
];
|
||||
|
||||
for (const table of serialTables) {
|
||||
try {
|
||||
await pool.query(`
|
||||
SELECT setval(pg_get_serial_sequence('${table}', 'id'),
|
||||
COALESCE((SELECT MAX(id) FROM ${table}), 0) + 1, false)
|
||||
`);
|
||||
} catch (err) {
|
||||
// Non-fatal — sequence may not exist for some tables
|
||||
console.log(` ⚠ Could not reset sequence for ${table}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
console.log('✓ Sequences reset');
|
||||
|
||||
// Step 3: Migrate findings from JSON blob
|
||||
console.log('\n── Step 3: Migrating findings from JSON blob ──');
|
||||
try {
|
||||
const cacheRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_findings_cache WHERE id = 1');
|
||||
|
||||
if (!cacheRow || !cacheRow.findings_json) {
|
||||
console.log(' ⚠ No findings_json data found in ivanti_findings_cache');
|
||||
} else {
|
||||
let findings;
|
||||
try {
|
||||
findings = JSON.parse(cacheRow.findings_json);
|
||||
} catch (parseErr) {
|
||||
console.error(' ✗ Failed to parse findings_json:', parseErr.message);
|
||||
findings = [];
|
||||
}
|
||||
|
||||
if (Array.isArray(findings) && findings.length > 0) {
|
||||
console.log(` Parsing ${findings.length} findings from JSON blob...`);
|
||||
|
||||
// Extract and insert findings
|
||||
const BATCH_SIZE = 100;
|
||||
let insertedCount = 0;
|
||||
|
||||
for (let i = 0; i < findings.length; i += BATCH_SIZE) {
|
||||
const batch = findings.slice(i, i + BATCH_SIZE);
|
||||
const values = [];
|
||||
const placeholders = [];
|
||||
|
||||
batch.forEach((rawFinding, idx) => {
|
||||
const f = extractFinding(rawFinding);
|
||||
const offset = idx * 18;
|
||||
values.push(
|
||||
f.id,
|
||||
f.hostId,
|
||||
f.title,
|
||||
f.severity,
|
||||
f.vrrGroup,
|
||||
f.hostName,
|
||||
f.ipAddress,
|
||||
f.dns,
|
||||
f.status,
|
||||
f.slaStatus,
|
||||
f.dueDate,
|
||||
f.lastFoundOn,
|
||||
f.buOwnership,
|
||||
f.cves,
|
||||
f.workflow ? f.workflow.id : null,
|
||||
f.workflow ? f.workflow.state : null,
|
||||
f.workflow ? f.workflow.type : null,
|
||||
'open' // state = open for all findings from cache
|
||||
);
|
||||
placeholders.push(
|
||||
`($${offset+1}, $${offset+2}, $${offset+3}, $${offset+4}, $${offset+5}, ` +
|
||||
`$${offset+6}, $${offset+7}, $${offset+8}, $${offset+9}, $${offset+10}, ` +
|
||||
`$${offset+11}, $${offset+12}, $${offset+13}, $${offset+14}, $${offset+15}, ` +
|
||||
`$${offset+16}, $${offset+17}, $${offset+18})`
|
||||
);
|
||||
});
|
||||
|
||||
await pool.query(`
|
||||
INSERT INTO ivanti_findings (
|
||||
id, host_id, title, severity, vrr_group,
|
||||
host_name, ip_address, dns, status, sla_status,
|
||||
due_date, last_found_on, bu_ownership, cves,
|
||||
workflow_id, workflow_state, workflow_type, state
|
||||
)
|
||||
VALUES ${placeholders.join(', ')}
|
||||
ON CONFLICT (id) DO NOTHING
|
||||
`, values);
|
||||
|
||||
insertedCount += batch.length;
|
||||
}
|
||||
|
||||
console.log(` ✓ ivanti_findings — ${insertedCount} findings inserted (state='open')`);
|
||||
migrationResults['ivanti_findings'] = { source: findings.length, dest: insertedCount };
|
||||
} else {
|
||||
console.log(' ○ findings_json is empty or not an array');
|
||||
migrationResults['ivanti_findings'] = { source: 0, dest: 0 };
|
||||
}
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` ✗ Findings migration ERROR: ${err.message}`);
|
||||
migrationResults['ivanti_findings'] = { source: 0, dest: 0, error: err.message };
|
||||
}
|
||||
|
||||
// Step 4: Merge notes into ivanti_findings.note
|
||||
console.log('\n── Step 4: Merging finding notes ──');
|
||||
try {
|
||||
const notes = await sqliteAll(sqliteDb, 'SELECT finding_id, note FROM ivanti_finding_notes');
|
||||
|
||||
if (notes.length === 0) {
|
||||
console.log(' ○ No finding notes to merge');
|
||||
} else {
|
||||
let mergedCount = 0;
|
||||
for (const { finding_id, note } of notes) {
|
||||
if (!finding_id || !note) continue;
|
||||
const result = await pool.query(
|
||||
`UPDATE ivanti_findings SET note = $1 WHERE id = $2`,
|
||||
[note, finding_id]
|
||||
);
|
||||
if (result.rowCount > 0) mergedCount++;
|
||||
}
|
||||
console.log(` ✓ Merged ${mergedCount}/${notes.length} notes into ivanti_findings.note`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` ✗ Notes merge ERROR: ${err.message}`);
|
||||
}
|
||||
|
||||
// Step 5: Merge overrides into ivanti_findings.override_host_name / override_dns
|
||||
console.log('\n── Step 5: Merging finding overrides ──');
|
||||
try {
|
||||
const overrides = await sqliteAll(
|
||||
sqliteDb,
|
||||
'SELECT finding_id, field, value FROM ivanti_finding_overrides'
|
||||
);
|
||||
|
||||
if (overrides.length === 0) {
|
||||
console.log(' ○ No finding overrides to merge');
|
||||
} else {
|
||||
let mergedCount = 0;
|
||||
for (const { finding_id, field, value } of overrides) {
|
||||
if (!finding_id || !field) continue;
|
||||
|
||||
let pgColumn;
|
||||
if (field === 'host_name' || field === 'hostName' || field === 'override_host_name') {
|
||||
pgColumn = 'override_host_name';
|
||||
} else if (field === 'dns' || field === 'override_dns') {
|
||||
pgColumn = 'override_dns';
|
||||
} else {
|
||||
// Unknown field — skip
|
||||
continue;
|
||||
}
|
||||
|
||||
const result = await pool.query(
|
||||
`UPDATE ivanti_findings SET ${pgColumn} = $1 WHERE id = $2`,
|
||||
[value, finding_id]
|
||||
);
|
||||
if (result.rowCount > 0) mergedCount++;
|
||||
}
|
||||
console.log(` ✓ Merged ${mergedCount}/${overrides.length} overrides into ivanti_findings`);
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` ✗ Overrides merge ERROR: ${err.message}`);
|
||||
}
|
||||
|
||||
// Step 6: Populate ivanti_sync_state from ivanti_findings_cache metadata
|
||||
console.log('\n── Step 6: Populating sync state and counts cache ──');
|
||||
try {
|
||||
const cacheRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_findings_cache WHERE id = 1');
|
||||
|
||||
if (cacheRow) {
|
||||
await pool.query(`
|
||||
UPDATE ivanti_sync_state SET
|
||||
total = $1,
|
||||
synced_at = $2,
|
||||
sync_status = $3,
|
||||
error_message = $4
|
||||
WHERE id = 1
|
||||
`, [
|
||||
cacheRow.total || 0,
|
||||
cacheRow.synced_at || null,
|
||||
cacheRow.sync_status || 'never',
|
||||
cacheRow.error_message || null,
|
||||
]);
|
||||
console.log(' ✓ ivanti_sync_state updated from ivanti_findings_cache metadata');
|
||||
}
|
||||
|
||||
// Populate ivanti_counts_cache
|
||||
const countsRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_counts_cache WHERE id = 1');
|
||||
if (countsRow) {
|
||||
await pool.query(`
|
||||
UPDATE ivanti_counts_cache SET
|
||||
open_count = $1,
|
||||
closed_count = $2,
|
||||
synced_at = $3,
|
||||
fp_workflow_counts_json = $4,
|
||||
fp_id_counts_json = $5
|
||||
WHERE id = 1
|
||||
`, [
|
||||
countsRow.open_count || 0,
|
||||
countsRow.closed_count || 0,
|
||||
countsRow.synced_at || null,
|
||||
countsRow.fp_workflow_counts_json || '{}',
|
||||
countsRow.fp_id_counts_json || '{}',
|
||||
]);
|
||||
console.log(' ✓ ivanti_counts_cache updated');
|
||||
}
|
||||
} catch (err) {
|
||||
console.error(` ✗ Sync state/counts migration ERROR: ${err.message}`);
|
||||
}
|
||||
|
||||
// Step 7: Verification — compare row counts
|
||||
console.log('\n── Step 7: Verification ──');
|
||||
console.log('');
|
||||
console.log('┌─────────────────────────────────┬──────────┬──────────┬────────┐');
|
||||
console.log('│ Table │ SQLite │ Postgres │ Status │');
|
||||
console.log('├─────────────────────────────────┼──────────┼──────────┼────────┤');
|
||||
|
||||
let hasDiscrepancy = false;
|
||||
const verificationTables = [
|
||||
...migrations.map(m => ({ sqlite: m.sqliteTable, pg: m.pgTable || m.sqliteTable })),
|
||||
{ sqlite: null, pg: 'ivanti_findings', special: true },
|
||||
];
|
||||
|
||||
for (const { sqlite: sqliteTable, pg: pgTable, special } of verificationTables) {
|
||||
let sqliteCount = 0;
|
||||
let pgCount = 0;
|
||||
|
||||
try {
|
||||
if (special && pgTable === 'ivanti_findings') {
|
||||
// For findings, source count is from the JSON blob
|
||||
const cacheRow = await sqliteGet(sqliteDb, 'SELECT findings_json FROM ivanti_findings_cache WHERE id = 1');
|
||||
if (cacheRow && cacheRow.findings_json) {
|
||||
try {
|
||||
const parsed = JSON.parse(cacheRow.findings_json);
|
||||
sqliteCount = Array.isArray(parsed) ? parsed.length : 0;
|
||||
} catch (e) {
|
||||
sqliteCount = 0;
|
||||
}
|
||||
}
|
||||
} else if (sqliteTable) {
|
||||
const tableExists = await sqliteGet(
|
||||
sqliteDb,
|
||||
`SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
|
||||
[sqliteTable]
|
||||
);
|
||||
if (tableExists) {
|
||||
const countRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) as cnt FROM ${sqliteTable}`);
|
||||
sqliteCount = countRow ? countRow.cnt : 0;
|
||||
}
|
||||
}
|
||||
|
||||
const pgCountRow = await pool.query(`SELECT COUNT(*) as cnt FROM ${pgTable}`);
|
||||
pgCount = parseInt(pgCountRow.rows[0].cnt, 10);
|
||||
} catch (err) {
|
||||
// Table might not exist in one or both
|
||||
}
|
||||
|
||||
const status = pgCount >= sqliteCount ? ' OK ' : ' WARN ';
|
||||
if (pgCount < sqliteCount) hasDiscrepancy = true;
|
||||
|
||||
const tableDisplay = (pgTable || '').padEnd(31);
|
||||
const srcDisplay = String(sqliteCount).padStart(6);
|
||||
const destDisplay = String(pgCount).padStart(6);
|
||||
|
||||
console.log(`│ ${tableDisplay} │ ${srcDisplay} │ ${destDisplay} │ ${status} │`);
|
||||
}
|
||||
|
||||
console.log('└─────────────────────────────────┴──────────┴──────────┴────────┘');
|
||||
|
||||
if (hasDiscrepancy) {
|
||||
console.log('\n⚠ WARNING: Some tables have fewer rows in Postgres than SQLite.');
|
||||
console.log(' This may be due to ON CONFLICT DO NOTHING skipping existing rows,');
|
||||
console.log(' or foreign key constraints preventing insertion.');
|
||||
}
|
||||
|
||||
// Cleanup
|
||||
await cleanup(sqliteDb, pool);
|
||||
|
||||
console.log('\n════════════════════════════════════════════════════════');
|
||||
if (hasDiscrepancy) {
|
||||
console.log('Migration completed with warnings. Review discrepancies above.');
|
||||
} else {
|
||||
console.log('✓ Migration completed successfully!');
|
||||
}
|
||||
console.log('════════════════════════════════════════════════════════\n');
|
||||
|
||||
process.exit(hasDiscrepancy ? 0 : 0); // Exit 0 even with warnings (data is safe)
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Cleanup helper
|
||||
// ---------------------------------------------------------------------------
|
||||
function cleanup(sqliteDb, pool) {
|
||||
return new Promise((resolve) => {
|
||||
sqliteDb.close((err) => {
|
||||
if (err) console.error('Warning: Error closing SQLite:', err.message);
|
||||
pool.end()
|
||||
.then(() => resolve())
|
||||
.catch(() => resolve());
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Entry point
|
||||
// ---------------------------------------------------------------------------
|
||||
migrate().catch((err) => {
|
||||
console.error('\n✗ FATAL ERROR:', err.message);
|
||||
console.error(err.stack);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user