#!/usr/bin/env node
/**
 * migrate-to-postgres.js — Data Migration Script
 *
 * Copies all data from the SQLite database (cve_database.db) to PostgreSQL.
 * The SQLite file is opened READ-ONLY and is never modified.
 *
 * Special handling:
 *   - ivanti_findings_cache.findings_json → individual rows in ivanti_findings
 *   - ivanti_finding_notes → merged into ivanti_findings.note column
 *   - ivanti_finding_overrides → merged into ivanti_findings.override_host_name / override_dns
 *   - ivanti_sync_state and ivanti_counts_cache → populated from ivanti_findings_cache metadata
 *
 * Type conversions:
 *   - SQLite 0/1 integers → Postgres boolean
 *   - SQLite DATETIME strings → Postgres TIMESTAMPTZ (passed as-is)
 *   - SQLite NULL → Postgres NULL
 *
 * Uses ON CONFLICT DO NOTHING for idempotency (safe to re-run).
 *
 * Usage:
 *   node backend/scripts/migrate-to-postgres.js
 *
 * Requires:
 *   - DATABASE_URL env var (or .env file in backend/)
 *   - SQLite database at backend/cve_database.db
 */

const path = require('path');
require('dotenv').config({ path: path.join(__dirname, '..', '.env') });

const fs = require('fs');
const sqlite3 = require('sqlite3').verbose();
const { Pool } = require('pg');

// ---------------------------------------------------------------------------
// Configuration
// ---------------------------------------------------------------------------
const SQLITE_PATH = path.join(__dirname, '..', 'cve_database.db');
const SCHEMA_PATH = path.join(__dirname, '..', 'db-schema.sql');
const DATABASE_URL = process.env.DATABASE_URL;

if (!DATABASE_URL) {
    console.error('ERROR: DATABASE_URL environment variable is not set.');
    console.error('Expected format: postgresql://user:password@host:port/database');
    process.exit(1);
}

if (!fs.existsSync(SQLITE_PATH)) {
    console.error(`ERROR: SQLite database not found at ${SQLITE_PATH}`);
    process.exit(1);
}

// ---------------------------------------------------------------------------
// SQLite helpers
// ---------------------------------------------------------------------------
function sqliteAll(db, sql, params = []) {
    return new Promise((resolve, reject) => {
        db.all(sql, params, (err, rows) => {
            if (err) reject(err);
            else resolve(rows || []);
        });
    });
}

function sqliteGet(db, sql, params = []) {
    return new Promise((resolve, reject) => {
        db.get(sql, params, (err, row) => {
            if (err) reject(err);
            else resolve(row);
        });
    });
}

// ---------------------------------------------------------------------------
// Extract finding fields from raw JSON object (mirrors ivantiFindings.js)
// ---------------------------------------------------------------------------
function extractFinding(f) {
    const rawDueDate = f.statusEmbedded?.dueDate || f.dueDate || '';
    const dueDate = rawDueDate ? rawDueDate.split('T')[0] : null;

    const buOwnership = f.assetCustomAttributes?.['1550_host_1']?.[0]
        || f.buOwnership || f.bu_ownership || '';

    const cves = Array.isArray(f.cves)
        ? f.cves
        : (f.vulnerabilities?.vulnInfoList || []).map(v => v.cve).filter(Boolean);

    // Workflow extraction
    let workflow = null;
    if (f.workflow && typeof f.workflow === 'object') {
        workflow = {
            id: f.workflow.id || '',
            state: f.workflow.state || '',
            type: f.workflow.type || 'FP',
        };
    } else if (f.workflowDistribution) {
        const wfDist = f.workflowDistribution || {};
        const fpBuckets = [
            ...(wfDist.actionableWorkflows || []),
            ...(wfDist.requestedWorkflows || []),
            ...(wfDist.reworkedWorkflows || []),
            ...(wfDist.rejectedWorkflows || []),
            ...(wfDist.expiredWorkflows || []),
            ...(wfDist.approvedWorkflows || []),
        ].filter(w => (w.generatedId || '').startsWith('FP#'));
        const fpEntry = fpBuckets[0] || null;
        if (fpEntry) {
            workflow = {
                id: fpEntry.generatedId || '',
                state: fpEntry.state || '',
                type: 'FP',
            };
        }
    }

    return {
        id: String(f.id),
        hostId: f.hostId || f.host?.hostId || null,
        title: f.title || '',
        severity: typeof f.severity === 'number' ? f.severity : parseFloat(f.severity) || 0,
        vrrGroup: f.vrrGroup || f.severityGroup || f.vrr_group || '',
        hostName: f.hostName || f.host?.hostName || f.host_name || '',
        ipAddress: f.ipAddress || f.host?.ipAddress || f.ip_address || '',
        dns: f.dns || f.host?.fqdn || '',
        status: f.status || '',
        slaStatus: f.slaStatus || f.sla_status || '',
        dueDate: dueDate,
        lastFoundOn: f.lastFoundOn || f.last_found_on || null,
        buOwnership,
        cves,
        workflow,
    };
}

// ---------------------------------------------------------------------------
// Batch insert helper for Postgres
// ---------------------------------------------------------------------------
async function batchInsert(pool, tableName, columns, rows, conflictClause = 'DO NOTHING') {
    if (rows.length === 0) return 0;

    const BATCH_SIZE = 100;
    let inserted = 0;

    for (let i = 0; i < rows.length; i += BATCH_SIZE) {
        const batch = rows.slice(i, i + BATCH_SIZE);
        const values = [];
        const placeholders = [];

        batch.forEach((row, idx) => {
            const offset = idx * columns.length;
            const rowPlaceholders = columns.map((_, colIdx) => `$${offset + colIdx + 1}`);
            placeholders.push(`(${rowPlaceholders.join(', ')})`);
            values.push(...row);
        });

        const sql = `INSERT INTO ${tableName} (${columns.join(', ')})
            VALUES ${placeholders.join(', ')}
            ON CONFLICT ${conflictClause}`;

        await pool.query(sql, values);
        inserted += batch.length;
    }

    return inserted;
}

// ---------------------------------------------------------------------------
// Table migration definitions
// ---------------------------------------------------------------------------

/**
 * Each entry defines how to copy a SQLite table to Postgres.
 * - sqliteTable: source table name
 * - pgTable: destination table name (defaults to sqliteTable)
 * - columns: array of { src, dest, transform } objects
 * - conflict: ON CONFLICT clause (default: DO NOTHING)
 * - selectSql: optional custom SELECT (defaults to SELECT * FROM sqliteTable)
 */
function getTableMigrations() {
    return [
        {
            sqliteTable: 'users',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'username', dest: 'username' },
                { src: 'email', dest: 'email' },
                { src: 'password_hash', dest: 'password_hash' },
                { src: 'role', dest: 'role' },
                { src: 'is_active', dest: 'is_active', transform: v => v === 1 || v === true },
                { src: 'created_at', dest: 'created_at' },
                { src: 'last_login', dest: 'last_login' },
                { src: 'user_group', dest: 'user_group', transform: v => v || 'Read_Only' },
                { src: 'bu_teams', dest: 'bu_teams', transform: v => v || '' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'sessions',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'session_id', dest: 'session_id' },
                { src: 'user_id', dest: 'user_id' },
                { src: 'expires_at', dest: 'expires_at' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'cves',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'cve_id', dest: 'cve_id' },
                { src: 'vendor', dest: 'vendor' },
                { src: 'severity', dest: 'severity' },
                { src: 'description', dest: 'description' },
                { src: 'published_date', dest: 'published_date' },
                { src: 'status', dest: 'status' },
                { src: 'created_at', dest: 'created_at' },
                { src: 'updated_at', dest: 'updated_at' },
                { src: 'created_by', dest: 'created_by' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'documents',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'cve_id', dest: 'cve_id' },
                { src: 'vendor', dest: 'vendor' },
                { src: 'name', dest: 'name' },
                { src: 'type', dest: 'type' },
                { src: 'file_path', dest: 'file_path' },
                { src: 'file_size', dest: 'file_size' },
                { src: 'mime_type', dest: 'mime_type' },
                { src: 'uploaded_at', dest: 'uploaded_at' },
                { src: 'notes', dest: 'notes' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'required_documents',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'vendor', dest: 'vendor' },
                { src: 'document_type', dest: 'document_type' },
                { src: 'is_mandatory', dest: 'is_mandatory', transform: v => v === 1 || v === true },
                { src: 'description', dest: 'description' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'jira_tickets',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'cve_id', dest: 'cve_id' },
                { src: 'vendor', dest: 'vendor' },
                { src: 'ticket_key', dest: 'ticket_key' },
                { src: 'url', dest: 'url' },
                { src: 'summary', dest: 'summary' },
                { src: 'status', dest: 'status' },
                { src: 'created_at', dest: 'created_at' },
                { src: 'updated_at', dest: 'updated_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'archer_tickets',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'exc_number', dest: 'exc_number' },
                { src: 'archer_url', dest: 'archer_url' },
                { src: 'status', dest: 'status' },
                { src: 'cve_id', dest: 'cve_id' },
                { src: 'vendor', dest: 'vendor' },
                { src: 'created_at', dest: 'created_at' },
                { src: 'updated_at', dest: 'updated_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'knowledge_base',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'title', dest: 'title' },
                { src: 'slug', dest: 'slug' },
                { src: 'description', dest: 'description' },
                { src: 'category', dest: 'category' },
                { src: 'file_path', dest: 'file_path' },
                { src: 'file_name', dest: 'file_name' },
                { src: 'file_type', dest: 'file_type' },
                { src: 'file_size', dest: 'file_size' },
                { src: 'created_at', dest: 'created_at' },
                { src: 'updated_at', dest: 'updated_at' },
                { src: 'created_by', dest: 'created_by' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'audit_logs',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'user_id', dest: 'user_id' },
                { src: 'username', dest: 'username' },
                { src: 'action', dest: 'action' },
                { src: 'entity_type', dest: 'entity_type' },
                { src: 'entity_id', dest: 'entity_id' },
                { src: 'details', dest: 'details' },
                { src: 'ip_address', dest: 'ip_address' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'compliance_uploads',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'filename', dest: 'filename' },
                { src: 'report_date', dest: 'report_date' },
                { src: 'uploaded_by', dest: 'uploaded_by' },
                { src: 'uploaded_at', dest: 'uploaded_at' },
                { src: 'new_count', dest: 'new_count' },
                { src: 'resolved_count', dest: 'resolved_count' },
                { src: 'recurring_count', dest: 'recurring_count' },
                { src: 'summary_json', dest: 'summary_json' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'compliance_items',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'upload_id', dest: 'upload_id' },
                { src: 'hostname', dest: 'hostname' },
                { src: 'ip_address', dest: 'ip_address' },
                { src: 'device_type', dest: 'device_type' },
                { src: 'team', dest: 'team' },
                { src: 'metric_id', dest: 'metric_id' },
                { src: 'metric_desc', dest: 'metric_desc' },
                { src: 'category', dest: 'category' },
                { src: 'extra_json', dest: 'extra_json' },
                { src: 'status', dest: 'status' },
                { src: 'first_seen_upload_id', dest: 'first_seen_upload_id' },
                { src: 'resolved_upload_id', dest: 'resolved_upload_id' },
                { src: 'seen_count', dest: 'seen_count' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'compliance_notes',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'hostname', dest: 'hostname' },
                { src: 'metric_id', dest: 'metric_id' },
                { src: 'note', dest: 'note' },
                { src: 'group_id', dest: 'group_id' },
                { src: 'created_by', dest: 'created_by' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_counts_history',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'open_count', dest: 'open_count' },
                { src: 'closed_count', dest: 'closed_count' },
                { src: 'recorded_at', dest: 'recorded_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_finding_archives',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'finding_id', dest: 'finding_id' },
                { src: 'finding_title', dest: 'finding_title' },
                { src: 'host_name', dest: 'host_name' },
                { src: 'ip_address', dest: 'ip_address' },
                { src: 'current_state', dest: 'current_state' },
                { src: 'last_severity', dest: 'last_severity' },
                { src: 'first_archived_at', dest: 'first_archived_at' },
                { src: 'last_transition_at', dest: 'last_transition_at' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_archive_transitions',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'archive_id', dest: 'archive_id' },
                { src: 'from_state', dest: 'from_state' },
                { src: 'to_state', dest: 'to_state' },
                { src: 'severity_at_transition', dest: 'severity_at_transition' },
                { src: 'reason', dest: 'reason' },
                { src: 'transitioned_at', dest: 'transitioned_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_sync_anomaly_log',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'sync_timestamp', dest: 'sync_timestamp' },
                { src: 'open_count_delta', dest: 'open_count_delta' },
                { src: 'closed_count_delta', dest: 'closed_count_delta' },
                { src: 'newly_archived_count', dest: 'newly_archived_count' },
                { src: 'returned_count', dest: 'returned_count' },
                { src: 'classification_json', dest: 'classification_json' },
                { src: 'return_classification_json', dest: 'return_classification_json' },
                { src: 'is_significant', dest: 'is_significant', transform: v => v === 1 || v === true },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_finding_bu_history',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'finding_id', dest: 'finding_id' },
                { src: 'finding_title', dest: 'finding_title' },
                { src: 'host_name', dest: 'host_name' },
                { src: 'previous_bu', dest: 'previous_bu' },
                { src: 'new_bu', dest: 'new_bu' },
                { src: 'detected_at', dest: 'detected_at' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'atlas_action_plans_cache',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'host_id', dest: 'host_id' },
                { src: 'has_action_plan', dest: 'has_action_plan', transform: v => v === 1 || v === true },
                { src: 'plan_count', dest: 'plan_count' },
                { src: 'plans_json', dest: 'plans_json' },
                { src: 'synced_at', dest: 'synced_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_fp_submissions',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'user_id', dest: 'user_id' },
                { src: 'username', dest: 'username' },
                { src: 'ivanti_workflow_batch_id', dest: 'ivanti_workflow_batch_id' },
                { src: 'ivanti_generated_id', dest: 'ivanti_generated_id' },
                { src: 'ivanti_workflow_batch_uuid', dest: 'ivanti_workflow_batch_uuid' },
                { src: 'workflow_name', dest: 'workflow_name' },
                { src: 'reason', dest: 'reason' },
                { src: 'description', dest: 'description' },
                { src: 'expiration_date', dest: 'expiration_date' },
                { src: 'scope_override', dest: 'scope_override' },
                { src: 'finding_ids_json', dest: 'finding_ids_json' },
                { src: 'queue_item_ids_json', dest: 'queue_item_ids_json' },
                { src: 'attachment_count', dest: 'attachment_count' },
                { src: 'attachment_results_json', dest: 'attachment_results_json' },
                { src: 'status', dest: 'status' },
                { src: 'lifecycle_status', dest: 'lifecycle_status' },
                { src: 'error_message', dest: 'error_message' },
                { src: 'created_at', dest: 'created_at' },
                { src: 'updated_at', dest: 'updated_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_fp_submission_history',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'submission_id', dest: 'submission_id' },
                { src: 'user_id', dest: 'user_id' },
                { src: 'username', dest: 'username' },
                { src: 'change_type', dest: 'change_type' },
                { src: 'change_details_json', dest: 'change_details_json' },
                { src: 'created_at', dest: 'created_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
        {
            sqliteTable: 'ivanti_todo_queue',
            columns: [
                { src: 'id', dest: 'id' },
                { src: 'user_id', dest: 'user_id' },
                { src: 'finding_id', dest: 'finding_id' },
                { src: 'finding_title', dest: 'finding_title' },
                { src: 'cves_json', dest: 'cves_json' },
                { src: 'ip_address', dest: 'ip_address' },
                { src: 'hostname', dest: 'hostname' },
                { src: 'vendor', dest: 'vendor' },
                { src: 'workflow_type', dest: 'workflow_type' },
                { src: 'status', dest: 'status' },
                { src: 'created_at', dest: 'created_at' },
                { src: 'updated_at', dest: 'updated_at' },
            ],
            conflict: '(id) DO NOTHING',
        },
    ];
}

// ---------------------------------------------------------------------------
// Main migration logic
// ---------------------------------------------------------------------------
async function migrate() {
    console.log('╔════════════════════════════════════════════════════════╗');
    console.log('║   CVE Dashboard — SQLite → PostgreSQL Migration       ║');
    console.log('╚════════════════════════════════════════════════════════╝\n');

    // Open SQLite in READ-ONLY mode
    const sqliteDb = new sqlite3.Database(SQLITE_PATH, sqlite3.OPEN_READONLY, (err) => {
        if (err) {
            console.error('ERROR: Failed to open SQLite database:', err.message);
            process.exit(1);
        }
    });
    console.log(`✓ Opened SQLite database (read-only): ${SQLITE_PATH}`);

    // Connect to Postgres
    const pool = new Pool({
        connectionString: DATABASE_URL,
        max: 5,
        idleTimeoutMillis: 30000,
        connectionTimeoutMillis: 10000,
    });

    try {
        await pool.query('SELECT NOW()');
        console.log('✓ Connected to PostgreSQL');
    } catch (err) {
        console.error('ERROR: Failed to connect to PostgreSQL:', err.message);
        sqliteDb.close();
        process.exit(1);
    }

    // Step 1: Run schema DDL
    console.log('\n── Step 1: Creating schema (idempotent) ──');
    try {
        const schemaSql = fs.readFileSync(SCHEMA_PATH, 'utf8');
        await pool.query(schemaSql);
        console.log('✓ Schema created/verified');
    } catch (err) {
        console.error('ERROR: Schema creation failed:', err.message);
        await cleanup(sqliteDb, pool);
        process.exit(1);
    }

    // Step 2: Copy simple tables
    console.log('\n── Step 2: Copying tables ──');
    const migrations = getTableMigrations();
    const migrationResults = {};

    for (const migration of migrations) {
        const tableName = migration.pgTable || migration.sqliteTable;
        try {
            // Check if table exists in SQLite
            const tableCheck = await sqliteGet(
                sqliteDb,
                `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
                [migration.sqliteTable]
            );

            if (!tableCheck) {
                console.log(`  ⚠ ${migration.sqliteTable} — table not found in SQLite, skipping`);
                migrationResults[tableName] = { source: 0, dest: 0, skipped: true };
                continue;
            }

            // Read all rows from SQLite
            const selectSql = migration.selectSql || `SELECT * FROM ${migration.sqliteTable}`;
            const sourceRows = await sqliteAll(sqliteDb, selectSql);

            if (sourceRows.length === 0) {
                console.log(`  ○ ${tableName} — 0 rows (empty table)`);
                migrationResults[tableName] = { source: 0, dest: 0 };
                continue;
            }

            // Transform rows
            const destColumns = migration.columns.map(c => c.dest);
            const transformedRows = sourceRows.map(row => {
                return migration.columns.map(col => {
                    let value = row[col.src];
                    if (value === undefined) value = null;
                    if (col.transform) {
                        value = col.transform(value);
                    }
                    return value;
                });
            });

            // Insert into Postgres
            const inserted = await batchInsert(
                pool,
                tableName,
                destColumns,
                transformedRows,
                migration.conflict
            );

            console.log(`  ✓ ${tableName} — ${inserted} rows copied`);
            migrationResults[tableName] = { source: sourceRows.length, dest: inserted };
        } catch (err) {
            console.error(`  ✗ ${tableName} — ERROR: ${err.message}`);
            migrationResults[tableName] = { source: 0, dest: 0, error: err.message };
        }
    }

    // Reset sequences for SERIAL columns after bulk insert with explicit IDs
    console.log('\n── Step 2b: Resetting sequences ──');
    const serialTables = [
        'users', 'sessions', 'cves', 'documents', 'required_documents',
        'jira_tickets', 'archer_tickets', 'knowledge_base', 'audit_logs',
        'compliance_uploads', 'compliance_items', 'compliance_notes',
        'ivanti_counts_history', 'ivanti_finding_archives',
        'ivanti_archive_transitions', 'ivanti_sync_anomaly_log',
        'ivanti_finding_bu_history', 'atlas_action_plans_cache',
        'ivanti_fp_submissions', 'ivanti_fp_submission_history',
        'ivanti_todo_queue',
    ];

    for (const table of serialTables) {
        try {
            await pool.query(`
                SELECT setval(pg_get_serial_sequence('${table}', 'id'),
                    COALESCE((SELECT MAX(id) FROM ${table}), 0) + 1, false)
            `);
        } catch (err) {
            // Non-fatal — sequence may not exist for some tables
            console.log(`  ⚠ Could not reset sequence for ${table}: ${err.message}`);
        }
    }
    console.log('✓ Sequences reset');

    // Step 3: Migrate findings from JSON blob
    console.log('\n── Step 3: Migrating findings from JSON blob ──');
    try {
        const cacheRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_findings_cache WHERE id = 1');

        if (!cacheRow || !cacheRow.findings_json) {
            console.log('  ⚠ No findings_json data found in ivanti_findings_cache');
        } else {
            let findings;
            try {
                findings = JSON.parse(cacheRow.findings_json);
            } catch (parseErr) {
                console.error('  ✗ Failed to parse findings_json:', parseErr.message);
                findings = [];
            }

            if (Array.isArray(findings) && findings.length > 0) {
                console.log(`  Parsing ${findings.length} findings from JSON blob...`);

                // Extract and insert findings
                const BATCH_SIZE = 100;
                let insertedCount = 0;

                for (let i = 0; i < findings.length; i += BATCH_SIZE) {
                    const batch = findings.slice(i, i + BATCH_SIZE);
                    const values = [];
                    const placeholders = [];

                    batch.forEach((rawFinding, idx) => {
                        const f = extractFinding(rawFinding);
                        const offset = idx * 18;
                        values.push(
                            f.id,
                            f.hostId,
                            f.title,
                            f.severity,
                            f.vrrGroup,
                            f.hostName,
                            f.ipAddress,
                            f.dns,
                            f.status,
                            f.slaStatus,
                            f.dueDate,
                            f.lastFoundOn,
                            f.buOwnership,
                            f.cves,
                            f.workflow ? f.workflow.id : null,
                            f.workflow ? f.workflow.state : null,
                            f.workflow ? f.workflow.type : null,
                            'open'  // state = open for all findings from cache
                        );
                        placeholders.push(
                            `($${offset+1}, $${offset+2}, $${offset+3}, $${offset+4}, $${offset+5}, ` +
                            `$${offset+6}, $${offset+7}, $${offset+8}, $${offset+9}, $${offset+10}, ` +
                            `$${offset+11}, $${offset+12}, $${offset+13}, $${offset+14}, $${offset+15}, ` +
                            `$${offset+16}, $${offset+17}, $${offset+18})`
                        );
                    });

                    await pool.query(`
                        INSERT INTO ivanti_findings (
                            id, host_id, title, severity, vrr_group,
                            host_name, ip_address, dns, status, sla_status,
                            due_date, last_found_on, bu_ownership, cves,
                            workflow_id, workflow_state, workflow_type, state
                        )
                        VALUES ${placeholders.join(', ')}
                        ON CONFLICT (id) DO NOTHING
                    `, values);

                    insertedCount += batch.length;
                }

                console.log(`  ✓ ivanti_findings — ${insertedCount} findings inserted (state='open')`);
                migrationResults['ivanti_findings'] = { source: findings.length, dest: insertedCount };
            } else {
                console.log('  ○ findings_json is empty or not an array');
                migrationResults['ivanti_findings'] = { source: 0, dest: 0 };
            }
        }
    } catch (err) {
        console.error(`  ✗ Findings migration ERROR: ${err.message}`);
        migrationResults['ivanti_findings'] = { source: 0, dest: 0, error: err.message };
    }

    // Step 4: Merge notes into ivanti_findings.note
    console.log('\n── Step 4: Merging finding notes ──');
    try {
        const notes = await sqliteAll(sqliteDb, 'SELECT finding_id, note FROM ivanti_finding_notes');

        if (notes.length === 0) {
            console.log('  ○ No finding notes to merge');
        } else {
            let mergedCount = 0;
            for (const { finding_id, note } of notes) {
                if (!finding_id || !note) continue;
                const result = await pool.query(
                    `UPDATE ivanti_findings SET note = $1 WHERE id = $2`,
                    [note, finding_id]
                );
                if (result.rowCount > 0) mergedCount++;
            }
            console.log(`  ✓ Merged ${mergedCount}/${notes.length} notes into ivanti_findings.note`);
        }
    } catch (err) {
        console.error(`  ✗ Notes merge ERROR: ${err.message}`);
    }

    // Step 5: Merge overrides into ivanti_findings.override_host_name / override_dns
    console.log('\n── Step 5: Merging finding overrides ──');
    try {
        const overrides = await sqliteAll(
            sqliteDb,
            'SELECT finding_id, field, value FROM ivanti_finding_overrides'
        );

        if (overrides.length === 0) {
            console.log('  ○ No finding overrides to merge');
        } else {
            let mergedCount = 0;
            for (const { finding_id, field, value } of overrides) {
                if (!finding_id || !field) continue;

                let pgColumn;
                if (field === 'host_name' || field === 'hostName' || field === 'override_host_name') {
                    pgColumn = 'override_host_name';
                } else if (field === 'dns' || field === 'override_dns') {
                    pgColumn = 'override_dns';
                } else {
                    // Unknown field — skip
                    continue;
                }

                const result = await pool.query(
                    `UPDATE ivanti_findings SET ${pgColumn} = $1 WHERE id = $2`,
                    [value, finding_id]
                );
                if (result.rowCount > 0) mergedCount++;
            }
            console.log(`  ✓ Merged ${mergedCount}/${overrides.length} overrides into ivanti_findings`);
        }
    } catch (err) {
        console.error(`  ✗ Overrides merge ERROR: ${err.message}`);
    }

    // Step 6: Populate ivanti_sync_state from ivanti_findings_cache metadata
    console.log('\n── Step 6: Populating sync state and counts cache ──');
    try {
        const cacheRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_findings_cache WHERE id = 1');

        if (cacheRow) {
            await pool.query(`
                UPDATE ivanti_sync_state SET
                    total = $1,
                    synced_at = $2,
                    sync_status = $3,
                    error_message = $4
                WHERE id = 1
            `, [
                cacheRow.total || 0,
                cacheRow.synced_at || null,
                cacheRow.sync_status || 'never',
                cacheRow.error_message || null,
            ]);
            console.log('  ✓ ivanti_sync_state updated from ivanti_findings_cache metadata');
        }

        // Populate ivanti_counts_cache
        const countsRow = await sqliteGet(sqliteDb, 'SELECT * FROM ivanti_counts_cache WHERE id = 1');
        if (countsRow) {
            await pool.query(`
                UPDATE ivanti_counts_cache SET
                    open_count = $1,
                    closed_count = $2,
                    synced_at = $3,
                    fp_workflow_counts_json = $4,
                    fp_id_counts_json = $5
                WHERE id = 1
            `, [
                countsRow.open_count || 0,
                countsRow.closed_count || 0,
                countsRow.synced_at || null,
                countsRow.fp_workflow_counts_json || '{}',
                countsRow.fp_id_counts_json || '{}',
            ]);
            console.log('  ✓ ivanti_counts_cache updated');
        }
    } catch (err) {
        console.error(`  ✗ Sync state/counts migration ERROR: ${err.message}`);
    }

    // Step 7: Verification — compare row counts
    console.log('\n── Step 7: Verification ──');
    console.log('');
    console.log('┌─────────────────────────────────┬──────────┬──────────┬────────┐');
    console.log('│ Table                           │  SQLite  │ Postgres │ Status │');
    console.log('├─────────────────────────────────┼──────────┼──────────┼────────┤');

    let hasDiscrepancy = false;
    const verificationTables = [
        ...migrations.map(m => ({ sqlite: m.sqliteTable, pg: m.pgTable || m.sqliteTable })),
        { sqlite: null, pg: 'ivanti_findings', special: true },
    ];

    for (const { sqlite: sqliteTable, pg: pgTable, special } of verificationTables) {
        let sqliteCount = 0;
        let pgCount = 0;

        try {
            if (special && pgTable === 'ivanti_findings') {
                // For findings, source count is from the JSON blob
                const cacheRow = await sqliteGet(sqliteDb, 'SELECT findings_json FROM ivanti_findings_cache WHERE id = 1');
                if (cacheRow && cacheRow.findings_json) {
                    try {
                        const parsed = JSON.parse(cacheRow.findings_json);
                        sqliteCount = Array.isArray(parsed) ? parsed.length : 0;
                    } catch (e) {
                        sqliteCount = 0;
                    }
                }
            } else if (sqliteTable) {
                const tableExists = await sqliteGet(
                    sqliteDb,
                    `SELECT name FROM sqlite_master WHERE type='table' AND name=?`,
                    [sqliteTable]
                );
                if (tableExists) {
                    const countRow = await sqliteGet(sqliteDb, `SELECT COUNT(*) as cnt FROM ${sqliteTable}`);
                    sqliteCount = countRow ? countRow.cnt : 0;
                }
            }

            const pgCountRow = await pool.query(`SELECT COUNT(*) as cnt FROM ${pgTable}`);
            pgCount = parseInt(pgCountRow.rows[0].cnt, 10);
        } catch (err) {
            // Table might not exist in one or both
        }

        const status = pgCount >= sqliteCount ? '  OK  ' : ' WARN ';
        if (pgCount < sqliteCount) hasDiscrepancy = true;

        const tableDisplay = (pgTable || '').padEnd(31);
        const srcDisplay = String(sqliteCount).padStart(6);
        const destDisplay = String(pgCount).padStart(6);

        console.log(`│ ${tableDisplay} │ ${srcDisplay}   │ ${destDisplay}   │ ${status} │`);
    }

    console.log('└─────────────────────────────────┴──────────┴──────────┴────────┘');

    if (hasDiscrepancy) {
        console.log('\n⚠ WARNING: Some tables have fewer rows in Postgres than SQLite.');
        console.log('  This may be due to ON CONFLICT DO NOTHING skipping existing rows,');
        console.log('  or foreign key constraints preventing insertion.');
    }

    // Cleanup
    await cleanup(sqliteDb, pool);

    console.log('\n════════════════════════════════════════════════════════');
    if (hasDiscrepancy) {
        console.log('Migration completed with warnings. Review discrepancies above.');
    } else {
        console.log('✓ Migration completed successfully!');
    }
    console.log('════════════════════════════════════════════════════════\n');

    process.exit(hasDiscrepancy ? 0 : 0); // Exit 0 even with warnings (data is safe)
}

// ---------------------------------------------------------------------------
// Cleanup helper
// ---------------------------------------------------------------------------
function cleanup(sqliteDb, pool) {
    return new Promise((resolve) => {
        sqliteDb.close((err) => {
            if (err) console.error('Warning: Error closing SQLite:', err.message);
            pool.end()
                .then(() => resolve())
                .catch(() => resolve());
        });
    });
}

// ---------------------------------------------------------------------------
// Entry point
// ---------------------------------------------------------------------------
migrate().catch((err) => {
    console.error('\n✗ FATAL ERROR:', err.message);
    console.error(err.stack);
    process.exit(1);
});