const fs = require('fs'); const path = require('path'); const { Pool } = require('pg'); const Papa = require('papaparse'); const CSV_PATH = process.env.CSV_PATH || path.join(__dirname, '..', 'EURUSD.csv'); const DATABASE_URL = process.env.DATABASE_URL; if (!DATABASE_URL) { console.error('❌ DATABASE_URL environment variable is not set'); process.exit(1); } async function loadInitialData() { console.log('Checking if initial data needs to be loaded...'); const pool = new Pool({ connectionString: DATABASE_URL, max: 2 }); try { // Check if candles table exists and has any data const tableCheck = await pool.query( "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'candles') AS exists" ); if (!tableCheck.rows[0].exists) { console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.'); return; } const countResult = await pool.query('SELECT COUNT(*) as count FROM candles'); const count = parseInt(countResult.rows[0].count, 10); if (count > 0) { console.log(`Database already has ${count} candles. Skipping initial data load.`); return; } console.log('Database is empty. Loading initial data from CSV...'); // Check if CSV file exists if (!fs.existsSync(CSV_PATH)) { console.log(`CSV file not found at ${CSV_PATH}. Skipping initial data load.`); return; } // Create a default chart for the initial data const chartName = 'EURUSD'; const chartResult = await pool.query( 'INSERT INTO charts (name, created_at) VALUES ($1, NOW()) RETURNING id', [chartName] ); const chartId = chartResult.rows[0].id; console.log(`Created chart "${chartName}" with id ${chartId}`); // Read and parse CSV const csvContent = fs.readFileSync(CSV_PATH, 'utf8'); const results = Papa.parse(csvContent, { header: true, dynamicTyping: true, skipEmptyLines: true, }); const rows = results.data; if (rows.length === 0) { console.log('CSV file is empty.'); return; } console.log(`Parsed ${rows.length} rows from CSV`); // Parse candle data const candleData = rows.map((row) => { let timestamp; if (typeof row.time === 'string') { const date = new Date(row.time); if (isNaN(date.getTime())) { throw new Error(`Invalid date format: ${row.time}`); } timestamp = date.toISOString(); } else if (typeof row.time === 'number') { timestamp = new Date(row.time * 1000).toISOString(); } else { throw new Error(`Invalid time value: ${row.time}`); } return { time: timestamp, open: Number(row.open), high: Number(row.high), low: Number(row.low), close: Number(row.close), }; }); // Batch insert using parameterized queries const BATCH_SIZE = 500; let inserted = 0; for (let i = 0; i < candleData.length; i += BATCH_SIZE) { const batch = candleData.slice(i, i + BATCH_SIZE); const values = []; const params = []; let paramIdx = 1; for (const candle of batch) { values.push(`($${paramIdx}, $${paramIdx + 1}, $${paramIdx + 2}, $${paramIdx + 3}, $${paramIdx + 4}, $${paramIdx + 5})`); params.push(chartId, candle.time, candle.open, candle.high, candle.low, candle.close); paramIdx += 6; } await pool.query( `INSERT INTO candles (chart_id, time, open, high, low, close) VALUES ${values.join(', ')}`, params ); inserted += batch.length; } console.log(`Successfully loaded ${inserted} candles into the database.`); } catch (error) { console.error('Error loading initial data:', error); process.exit(1); } finally { await pool.end(); } } loadInitialData();