fix: resolve database migration failures and startup ordering

- Add scripts/run-migrations.js to run migrations before data loading
- Fix startup.sh ordering: migrations -> data load -> app start
- Fix migration 0005 missing statement-breakpoint between ALTER TABLE statements
- Add migration 0005 to drizzle journal (was missing)
- Fix load-initial-data.js to check table existence before querying
- Fix load-initial-data.js to create chart record before inserting candles (chart_id NOT NULL constraint)
- Simplify db/index.ts migration error handling (remove overly broad 'already exists' catch)
- Add pre-migration check for inconsistent DB state (tables without migration tracking)

scripts/load-initial-data.js

@@ -12,7 +12,17 @@ async function loadInitialData() {
   const db = new Database(DB_PATH);
 
   try {
-    // Check if candles table has any data
+    // Check if candles table exists and has any data
+    const tableExists = db.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='candles'"
+    ).get();
+
+    if (!tableExists) {
+      console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.');
+      db.close();
+      return;
+    }
+
     const count = db.prepare('SELECT COUNT(*) as count FROM candles').get();
 
     if (count.count > 0) {
@@ -30,6 +40,15 @@ async function loadInitialData() {
       return;
     }
 
+    // Create a default chart for the initial data
+    const chartName = 'EURUSD';
+    const now = Math.floor(Date.now() / 1000);
+    const chartResult = db.prepare(
+      'INSERT INTO charts (name, created_at) VALUES (?, ?) RETURNING id'
+    ).get(chartName, now);
+    const chartId = chartResult.id;
+    console.log(`Created chart "${chartName}" with id ${chartId}`);
+
     // Read and parse CSV
     const csvContent = fs.readFileSync(CSV_PATH, 'utf8');
 
@@ -51,12 +70,12 @@ async function loadInitialData() {
 
           // Prepare insert statement
           const insert = db.prepare(
-            'INSERT INTO candles (time, open, high, low, close) VALUES (?, ?, ?, ?, ?)'
+            'INSERT INTO candles (chart_id, time, open, high, low, close) VALUES (?, ?, ?, ?, ?, ?)'
           );
 
           const insertMany = db.transaction((candles) => {
             for (const candle of candles) {
-              insert.run(candle.time, candle.open, candle.high, candle.low, candle.close);
+              insert.run(chartId, candle.time, candle.open, candle.high, candle.low, candle.close);
             }
           });
 

scripts/run-migrations.js

@@ -0,0 +1,51 @@
+const path = require('path');
+const fs = require('fs');
+const Database = require('better-sqlite3');
+
+const dataDir = path.join(__dirname, '..', 'data');
+const dbPath = path.join(dataDir, 'candles.db');
+const migrationsFolder = path.join(__dirname, '..', 'drizzle');
+
+// Ensure data directory exists
+if (!fs.existsSync(dataDir)) {
+  fs.mkdirSync(dataDir, { recursive: true });
+}
+
+// Check for inconsistent DB state before migrating
+if (fs.existsSync(dbPath)) {
+  try {
+    const checkDb = new Database(dbPath);
+    const hasDrizzleMigrations = checkDb.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'"
+    ).get();
+    const hasAnyTables = checkDb.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name != '__drizzle_migrations'"
+    ).get();
+    checkDb.close();
+
+    if (hasAnyTables && !hasDrizzleMigrations) {
+      console.log('⚠️  Database has tables but no migration tracking. Recreating...');
+      fs.unlinkSync(dbPath);
+    }
+  } catch {
+    console.log('⚠️  Database file is corrupted. Recreating...');
+    try { fs.unlinkSync(dbPath); } catch {}
+  }
+}
+
+// Run migrations using better-sqlite3 directly
+const { drizzle } = require('drizzle-orm/better-sqlite3');
+const { migrate } = require('drizzle-orm/better-sqlite3/migrator');
+
+const sqlite = new Database(dbPath);
+const db = drizzle(sqlite);
+
+try {
+  migrate(db, { migrationsFolder });
+  console.log('✅ Database migrations completed');
+} catch (error) {
+  console.error('❌ Migration failed:', error);
+  process.exit(1);
+} finally {
+  sqlite.close();
+}

scripts/startup.sh

@@ -2,7 +2,7 @@
 set -e
 
 echo "Running database migrations..."
-# Migrations are run automatically by the app on startup
+node /app/scripts/run-migrations.js
 
 echo "Loading initial data if needed..."
 node /app/scripts/load-initial-data.js