diff --git a/scripts/load-initial-data.js b/scripts/load-initial-data.js index 02dfaed..7b20973 100644 --- a/scripts/load-initial-data.js +++ b/scripts/load-initial-data.js @@ -1,33 +1,37 @@ const fs = require('fs'); const path = require('path'); -const Database = require('better-sqlite3'); +const { Pool } = require('pg'); const Papa = require('papaparse'); -const DB_PATH = process.env.DATABASE_PATH || path.join(__dirname, '..', 'data', 'candles.db'); const CSV_PATH = process.env.CSV_PATH || path.join(__dirname, '..', 'EURUSD.csv'); +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error('❌ DATABASE_URL environment variable is not set'); + process.exit(1); +} + async function loadInitialData() { console.log('Checking if initial data needs to be loaded...'); - const db = new Database(DB_PATH); + const pool = new Pool({ connectionString: DATABASE_URL, max: 2 }); try { // Check if candles table exists and has any data - const tableExists = db.prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='candles'" - ).get(); + const tableCheck = await pool.query( + "SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'candles') AS exists" + ); - if (!tableExists) { + if (!tableCheck.rows[0].exists) { console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.'); - db.close(); return; } - const count = db.prepare('SELECT COUNT(*) as count FROM candles').get(); + const countResult = await pool.query('SELECT COUNT(*) as count FROM candles'); + const count = parseInt(countResult.rows[0].count, 10); - if (count.count > 0) { - console.log(`Database already has ${count.count} candles. Skipping initial data load.`); - db.close(); + if (count > 0) { + console.log(`Database already has ${count} candles. Skipping initial data load.`); return; } @@ -36,94 +40,90 @@ async function loadInitialData() { // Check if CSV file exists if (!fs.existsSync(CSV_PATH)) { console.log(`CSV file not found at ${CSV_PATH}. Skipping initial data load.`); - db.close(); return; } // Create a default chart for the initial data const chartName = 'EURUSD'; - const now = Math.floor(Date.now() / 1000); - const chartResult = db.prepare( - 'INSERT INTO charts (name, created_at) VALUES (?, ?) RETURNING id' - ).get(chartName, now); - const chartId = chartResult.id; + const chartResult = await pool.query( + 'INSERT INTO charts (name, created_at) VALUES ($1, NOW()) RETURNING id', + [chartName] + ); + const chartId = chartResult.rows[0].id; console.log(`Created chart "${chartName}" with id ${chartId}`); // Read and parse CSV const csvContent = fs.readFileSync(CSV_PATH, 'utf8'); - Papa.parse(csvContent, { + const results = Papa.parse(csvContent, { header: true, dynamicTyping: true, skipEmptyLines: true, - complete: (results) => { - try { - const rows = results.data; - - if (rows.length === 0) { - console.log('CSV file is empty.'); - db.close(); - return; - } - - console.log(`Parsed ${rows.length} rows from CSV`); - - // Prepare insert statement - const insert = db.prepare( - 'INSERT INTO candles (chart_id, time, open, high, low, close) VALUES (?, ?, ?, ?, ?, ?)' - ); - - const insertMany = db.transaction((candles) => { - for (const candle of candles) { - insert.run(chartId, candle.time, candle.open, candle.high, candle.low, candle.close); - } - }); - - // Parse and prepare candle data - const candleData = rows.map((row) => { - let timestamp; - - // Handle both date strings and Unix timestamps - if (typeof row.time === 'string') { - // Try parsing as date string - const date = new Date(row.time); - if (isNaN(date.getTime())) { - throw new Error(`Invalid date format: ${row.time}`); - } - timestamp = Math.floor(date.getTime() / 1000); - } else if (typeof row.time === 'number') { - timestamp = row.time; - } else { - throw new Error(`Invalid time value: ${row.time}`); - } - - return { - time: timestamp, - open: Number(row.open), - high: Number(row.high), - low: Number(row.low), - close: Number(row.close), - }; - }); - - // Insert all candles in a transaction - insertMany(candleData); - - console.log(`Successfully loaded ${candleData.length} candles into the database.`); - } catch (error) { - console.error('Error loading initial data:', error); - } finally { - db.close(); - } - }, - error: (error) => { - console.error('CSV parsing error:', error); - db.close(); - }, }); + + const rows = results.data; + + if (rows.length === 0) { + console.log('CSV file is empty.'); + return; + } + + console.log(`Parsed ${rows.length} rows from CSV`); + + // Parse candle data + const candleData = rows.map((row) => { + let timestamp; + + if (typeof row.time === 'string') { + const date = new Date(row.time); + if (isNaN(date.getTime())) { + throw new Error(`Invalid date format: ${row.time}`); + } + timestamp = date.toISOString(); + } else if (typeof row.time === 'number') { + timestamp = new Date(row.time * 1000).toISOString(); + } else { + throw new Error(`Invalid time value: ${row.time}`); + } + + return { + time: timestamp, + open: Number(row.open), + high: Number(row.high), + low: Number(row.low), + close: Number(row.close), + }; + }); + + // Batch insert using parameterized queries + const BATCH_SIZE = 500; + let inserted = 0; + + for (let i = 0; i < candleData.length; i += BATCH_SIZE) { + const batch = candleData.slice(i, i + BATCH_SIZE); + const values = []; + const params = []; + let paramIdx = 1; + + for (const candle of batch) { + values.push(`($${paramIdx}, $${paramIdx + 1}, $${paramIdx + 2}, $${paramIdx + 3}, $${paramIdx + 4}, $${paramIdx + 5})`); + params.push(chartId, candle.time, candle.open, candle.high, candle.low, candle.close); + paramIdx += 6; + } + + await pool.query( + `INSERT INTO candles (chart_id, time, open, high, low, close) VALUES ${values.join(', ')}`, + params + ); + inserted += batch.length; + } + + console.log(`Successfully loaded ${inserted} candles into the database.`); } catch (error) { - console.error('Error checking database:', error); - db.close(); + console.error('Error loading initial data:', error); + process.exit(1); + } finally { + await pool.end(); } } diff --git a/scripts/run-migrations.js b/scripts/run-migrations.js index b4dd2bc..80f7f20 100644 --- a/scripts/run-migrations.js +++ b/scripts/run-migrations.js @@ -1,51 +1,25 @@ const path = require('path'); -const fs = require('fs'); -const Database = require('better-sqlite3'); +const { Pool } = require('pg'); +const { drizzle } = require('drizzle-orm/node-postgres'); +const { migrate } = require('drizzle-orm/node-postgres/migrator'); -const dataDir = path.join(__dirname, '..', 'data'); -const dbPath = path.join(dataDir, 'candles.db'); const migrationsFolder = path.join(__dirname, '..', 'drizzle'); -// Ensure data directory exists -if (!fs.existsSync(dataDir)) { - fs.mkdirSync(dataDir, { recursive: true }); -} - -// Check for inconsistent DB state before migrating -if (fs.existsSync(dbPath)) { - try { - const checkDb = new Database(dbPath); - const hasDrizzleMigrations = checkDb.prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'" - ).get(); - const hasAnyTables = checkDb.prepare( - "SELECT name FROM sqlite_master WHERE type='table' AND name != '__drizzle_migrations'" - ).get(); - checkDb.close(); - - if (hasAnyTables && !hasDrizzleMigrations) { - console.log('⚠️ Database has tables but no migration tracking. Recreating...'); - fs.unlinkSync(dbPath); - } - } catch { - console.log('⚠️ Database file is corrupted. Recreating...'); - try { fs.unlinkSync(dbPath); } catch {} - } -} - -// Run migrations using better-sqlite3 directly -const { drizzle } = require('drizzle-orm/better-sqlite3'); -const { migrate } = require('drizzle-orm/better-sqlite3/migrator'); - -const sqlite = new Database(dbPath); -const db = drizzle(sqlite); - -try { - migrate(db, { migrationsFolder }); - console.log('✅ Database migrations completed'); -} catch (error) { - console.error('❌ Migration failed:', error); +const DATABASE_URL = process.env.DATABASE_URL; +if (!DATABASE_URL) { + console.error('❌ DATABASE_URL environment variable is not set'); process.exit(1); -} finally { - sqlite.close(); } + +const pool = new Pool({ connectionString: DATABASE_URL, max: 2 }); +const db = drizzle(pool); + +migrate(db, { migrationsFolder }) + .then(() => { + console.log('✅ Database migrations completed'); + return pool.end(); + }) + .catch((error) => { + console.error('❌ Migration failed:', error); + pool.end().finally(() => process.exit(1)); + }); diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index e9d2053..458d071 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -1,35 +1,51 @@ import { Pool } from 'pg'; -import { drizzle } from 'drizzle-orm/node-postgres'; +import { drizzle, type NodePgDatabase } from 'drizzle-orm/node-postgres'; import { migrate } from 'drizzle-orm/node-postgres/migrator'; import * as schema from './schema'; import path from 'path'; -// Read DATABASE_URL from environment -const DATABASE_URL = process.env.DATABASE_URL; +let _pool: Pool | null = null; +let _db: NodePgDatabase | null = null; +let _migrated = false; -if (!DATABASE_URL) { - throw new Error('DATABASE_URL environment variable is not set'); +function getPool(): Pool { + if (!_pool) { + const DATABASE_URL = process.env.DATABASE_URL; + if (!DATABASE_URL) { + throw new Error('DATABASE_URL environment variable is not set'); + } + _pool = new Pool({ connectionString: DATABASE_URL, max: 10 }); + } + return _pool; } -// Create PostgreSQL connection pool -const pool = new Pool({ - connectionString: DATABASE_URL, - max: 10, -}); +function getDb(): NodePgDatabase { + if (!_db) { + _db = drizzle(getPool(), { schema }); + } + return _db; +} -export const db = drizzle(pool, { schema }); - -// Run migrations at startup (skip during build phase) -const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build'; - -if (!isBuildTime) { +export async function runMigrations() { + if (_migrated) return; + const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build'; + if (isBuildTime) { + console.log('ℹ️ Skipping migrations during build phase'); + return; + } try { - await migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') }); + await migrate(getDb(), { migrationsFolder: path.join(process.cwd(), 'drizzle') }); console.log('✅ Database migrations completed'); + _migrated = true; } catch (error) { console.error('❌ Migration failed:', error); throw error; } -} else { - console.log('ℹ️ Skipping migrations during build phase'); } + +// Lazy proxy: db is accessed as a module export but only connects on first use +export const db = new Proxy({} as NodePgDatabase, { + get(_target, prop, receiver) { + return Reflect.get(getDb(), prop, receiver); + }, +});