candle-annotator/scripts/load-initial-data.js

const fs = require('fs');
const path = require('path');
const { Pool } = require('pg');
const Papa = require('papaparse');

const CSV_PATH = process.env.CSV_PATH || path.join(__dirname, '..', 'EURUSD.csv');

const DATABASE_URL = process.env.DATABASE_URL;
if (!DATABASE_URL) {
  console.error('❌ DATABASE_URL environment variable is not set');
  process.exit(1);
}

async function loadInitialData() {
  console.log('Checking if initial data needs to be loaded...');

  const pool = new Pool({ connectionString: DATABASE_URL, max: 2 });

  try {
    // Check if candles table exists and has any data
    const tableCheck = await pool.query(
      "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'candles') AS exists"
    );

    if (!tableCheck.rows[0].exists) {
      console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.');
      return;
    }

    const countResult = await pool.query('SELECT COUNT(*) as count FROM candles');
    const count = parseInt(countResult.rows[0].count, 10);

    if (count > 0) {
      console.log(`Database already has ${count} candles. Skipping initial data load.`);
      return;
    }

    console.log('Database is empty. Loading initial data from CSV...');

    // Check if CSV file exists
    if (!fs.existsSync(CSV_PATH)) {
      console.log(`CSV file not found at ${CSV_PATH}. Skipping initial data load.`);
      return;
    }

    // Create a default chart for the initial data
    const chartName = 'EURUSD';
    const chartResult = await pool.query(
      'INSERT INTO charts (name, created_at) VALUES ($1, NOW()) RETURNING id',
      [chartName]
    );
    const chartId = chartResult.rows[0].id;
    console.log(`Created chart "${chartName}" with id ${chartId}`);

    // Read and parse CSV
    const csvContent = fs.readFileSync(CSV_PATH, 'utf8');

    const results = Papa.parse(csvContent, {
      header: true,
      dynamicTyping: true,
      skipEmptyLines: true,
    });

    const rows = results.data;

    if (rows.length === 0) {
      console.log('CSV file is empty.');
      return;
    }

    console.log(`Parsed ${rows.length} rows from CSV`);

    // Parse candle data
    const candleData = rows.map((row) => {
      let timestamp;

      if (typeof row.time === 'string') {
        const date = new Date(row.time);
        if (isNaN(date.getTime())) {
          throw new Error(`Invalid date format: ${row.time}`);
        }
        timestamp = date.toISOString();
      } else if (typeof row.time === 'number') {
        timestamp = new Date(row.time * 1000).toISOString();
      } else {
        throw new Error(`Invalid time value: ${row.time}`);
      }

      return {
        time: timestamp,
        open: Number(row.open),
        high: Number(row.high),
        low: Number(row.low),
        close: Number(row.close),
      };
    });

    // Batch insert using parameterized queries
    const BATCH_SIZE = 500;
    let inserted = 0;

    for (let i = 0; i < candleData.length; i += BATCH_SIZE) {
      const batch = candleData.slice(i, i + BATCH_SIZE);
      const values = [];
      const params = [];
      let paramIdx = 1;

      for (const candle of batch) {
        values.push(`($${paramIdx}, $${paramIdx + 1}, $${paramIdx + 2}, $${paramIdx + 3}, $${paramIdx + 4}, $${paramIdx + 5})`);
        params.push(chartId, candle.time, candle.open, candle.high, candle.low, candle.close);
        paramIdx += 6;
      }

      await pool.query(
        `INSERT INTO candles (chart_id, time, open, high, low, close) VALUES ${values.join(', ')}`,
        params
      );
      inserted += batch.length;
    }

    console.log(`Successfully loaded ${inserted} candles into the database.`);
  } catch (error) {
    console.error('Error loading initial data:', error);
    process.exit(1);
  } finally {
    await pool.end();
  }
}

loadInitialData();