candle-annotator/scripts/load-initial-data.js

const fs = require('fs');
const path = require('path');
const Database = require('better-sqlite3');
const Papa = require('papaparse');

const DB_PATH = process.env.DATABASE_PATH || path.join(__dirname, '..', 'data', 'candles.db');
const CSV_PATH = process.env.CSV_PATH || path.join(__dirname, '..', 'EURUSD.csv');

async function loadInitialData() {
  console.log('Checking if initial data needs to be loaded...');

  const db = new Database(DB_PATH);

  try {
    // Check if candles table exists and has any data
    const tableExists = db.prepare(
      "SELECT name FROM sqlite_master WHERE type='table' AND name='candles'"
    ).get();

    if (!tableExists) {
      console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.');
      db.close();
      return;
    }

    const count = db.prepare('SELECT COUNT(*) as count FROM candles').get();

    if (count.count > 0) {
      console.log(`Database already has ${count.count} candles. Skipping initial data load.`);
      db.close();
      return;
    }

    console.log('Database is empty. Loading initial data from CSV...');

    // Check if CSV file exists
    if (!fs.existsSync(CSV_PATH)) {
      console.log(`CSV file not found at ${CSV_PATH}. Skipping initial data load.`);
      db.close();
      return;
    }

    // Create a default chart for the initial data
    const chartName = 'EURUSD';
    const now = Math.floor(Date.now() / 1000);
    const chartResult = db.prepare(
      'INSERT INTO charts (name, created_at) VALUES (?, ?) RETURNING id'
    ).get(chartName, now);
    const chartId = chartResult.id;
    console.log(`Created chart "${chartName}" with id ${chartId}`);

    // Read and parse CSV
    const csvContent = fs.readFileSync(CSV_PATH, 'utf8');

    Papa.parse(csvContent, {
      header: true,
      dynamicTyping: true,
      skipEmptyLines: true,
      complete: (results) => {
        try {
          const rows = results.data;

          if (rows.length === 0) {
            console.log('CSV file is empty.');
            db.close();
            return;
          }

          console.log(`Parsed ${rows.length} rows from CSV`);

          // Prepare insert statement
          const insert = db.prepare(
            'INSERT INTO candles (chart_id, time, open, high, low, close) VALUES (?, ?, ?, ?, ?, ?)'
          );

          const insertMany = db.transaction((candles) => {
            for (const candle of candles) {
              insert.run(chartId, candle.time, candle.open, candle.high, candle.low, candle.close);
            }
          });

          // Parse and prepare candle data
          const candleData = rows.map((row) => {
            let timestamp;

            // Handle both date strings and Unix timestamps
            if (typeof row.time === 'string') {
              // Try parsing as date string
              const date = new Date(row.time);
              if (isNaN(date.getTime())) {
                throw new Error(`Invalid date format: ${row.time}`);
              }
              timestamp = Math.floor(date.getTime() / 1000);
            } else if (typeof row.time === 'number') {
              timestamp = row.time;
            } else {
              throw new Error(`Invalid time value: ${row.time}`);
            }

            return {
              time: timestamp,
              open: Number(row.open),
              high: Number(row.high),
              low: Number(row.low),
              close: Number(row.close),
            };
          });

          // Insert all candles in a transaction
          insertMany(candleData);

          console.log(`Successfully loaded ${candleData.length} candles into the database.`);
        } catch (error) {
          console.error('Error loading initial data:', error);
        } finally {
          db.close();
        }
      },
      error: (error) => {
        console.error('CSV parsing error:', error);
        db.close();
      },
    });
  } catch (error) {
    console.error('Error checking database:', error);
    db.close();
  }
}

loadInitialData();