fix: resolve database migration failures and startup ordering

- Add scripts/run-migrations.js to run migrations before data loading
- Fix startup.sh ordering: migrations -> data load -> app start
- Fix migration 0005 missing statement-breakpoint between ALTER TABLE statements
- Add migration 0005 to drizzle journal (was missing)
- Fix load-initial-data.js to check table existence before querying
- Fix load-initial-data.js to create chart record before inserting candles (chart_id NOT NULL constraint)
- Simplify db/index.ts migration error handling (remove overly broad 'already exists' catch)
- Add pre-migration check for inconsistent DB state (tables without migration tracking)

drizzle/0005_add_source_and_model_prediction_to_span_annotations.sql

@@ -4,7 +4,7 @@
 -- Add source field (defaults to 'human')
 -- Possible values: 'human', 'model', 'human_correction'
 ALTER TABLE `span_annotations` ADD COLUMN `source` TEXT NOT NULL DEFAULT 'human';
-
+--> statement-breakpoint
 -- Add model_prediction field (nullable JSON)
 -- Stores model prediction metadata when user confirms/corrects a prediction
 -- Example: {"label": "bull_flag", "confidence": 0.85, "model_version": "xgb_v1"}

drizzle/meta/_journal.json

@@ -36,6 +36,13 @@
       "when": 1771055000000,
       "tag": "0004_add_default_span_label_types",
       "breakpoints": true
+    },
+    {
+      "idx": 5,
+      "version": "6",
+      "when": 1771156000000,
+      "tag": "0005_add_source_and_model_prediction_to_span_annotations",
+      "breakpoints": true
     }
   ]
 }

scripts/load-initial-data.js

@@ -12,7 +12,17 @@ async function loadInitialData() {
   const db = new Database(DB_PATH);
 
   try {
-    // Check if candles table has any data
+    // Check if candles table exists and has any data
+    const tableExists = db.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='candles'"
+    ).get();
+
+    if (!tableExists) {
+      console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.');
+      db.close();
+      return;
+    }
+
     const count = db.prepare('SELECT COUNT(*) as count FROM candles').get();
 
     if (count.count > 0) {
@@ -30,6 +40,15 @@ async function loadInitialData() {
       return;
     }
 
+    // Create a default chart for the initial data
+    const chartName = 'EURUSD';
+    const now = Math.floor(Date.now() / 1000);
+    const chartResult = db.prepare(
+      'INSERT INTO charts (name, created_at) VALUES (?, ?) RETURNING id'
+    ).get(chartName, now);
+    const chartId = chartResult.id;
+    console.log(`Created chart "${chartName}" with id ${chartId}`);
+
     // Read and parse CSV
     const csvContent = fs.readFileSync(CSV_PATH, 'utf8');
 
@@ -51,12 +70,12 @@ async function loadInitialData() {
 
           // Prepare insert statement
           const insert = db.prepare(
-            'INSERT INTO candles (time, open, high, low, close) VALUES (?, ?, ?, ?, ?)'
+            'INSERT INTO candles (chart_id, time, open, high, low, close) VALUES (?, ?, ?, ?, ?, ?)'
           );
 
           const insertMany = db.transaction((candles) => {
             for (const candle of candles) {
-              insert.run(candle.time, candle.open, candle.high, candle.low, candle.close);
+              insert.run(chartId, candle.time, candle.open, candle.high, candle.low, candle.close);
             }
           });
 

scripts/run-migrations.js

@@ -0,0 +1,51 @@
+const path = require('path');
+const fs = require('fs');
+const Database = require('better-sqlite3');
+
+const dataDir = path.join(__dirname, '..', 'data');
+const dbPath = path.join(dataDir, 'candles.db');
+const migrationsFolder = path.join(__dirname, '..', 'drizzle');
+
+// Ensure data directory exists
+if (!fs.existsSync(dataDir)) {
+  fs.mkdirSync(dataDir, { recursive: true });
+}
+
+// Check for inconsistent DB state before migrating
+if (fs.existsSync(dbPath)) {
+  try {
+    const checkDb = new Database(dbPath);
+    const hasDrizzleMigrations = checkDb.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'"
+    ).get();
+    const hasAnyTables = checkDb.prepare(
+      "SELECT name FROM sqlite_master WHERE type='table' AND name != '__drizzle_migrations'"
+    ).get();
+    checkDb.close();
+
+    if (hasAnyTables && !hasDrizzleMigrations) {
+      console.log('⚠️  Database has tables but no migration tracking. Recreating...');
+      fs.unlinkSync(dbPath);
+    }
+  } catch {
+    console.log('⚠️  Database file is corrupted. Recreating...');
+    try { fs.unlinkSync(dbPath); } catch {}
+  }
+}
+
+// Run migrations using better-sqlite3 directly
+const { drizzle } = require('drizzle-orm/better-sqlite3');
+const { migrate } = require('drizzle-orm/better-sqlite3/migrator');
+
+const sqlite = new Database(dbPath);
+const db = drizzle(sqlite);
+
+try {
+  migrate(db, { migrationsFolder });
+  console.log('✅ Database migrations completed');
+} catch (error) {
+  console.error('❌ Migration failed:', error);
+  process.exit(1);
+} finally {
+  sqlite.close();
+}

scripts/startup.sh

@@ -2,7 +2,7 @@
 set -e
 
 echo "Running database migrations..."
-# Migrations are run automatically by the app on startup
+node /app/scripts/run-migrations.js
 
 echo "Loading initial data if needed..."
 node /app/scripts/load-initial-data.js

src/lib/db/index.ts

@@ -15,8 +15,9 @@ const dbPath = path.join(dataDir, 'candles.db');
 const sqlite = new Database(dbPath);
 export const db = drizzle(sqlite, { schema });
 
-// Run migrations only in runtime (not during build)
-// Check if we're in build mode by looking for the NEXT_PHASE environment variable
+// Run migrations at startup (for local dev).
+// In Docker, migrations are run by scripts/run-migrations.js before the app starts,
+// so this will be a no-op (all migrations already applied).
 const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build';
 
 if (!isBuildTime) {
@@ -24,17 +25,9 @@ if (!isBuildTime) {
     migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') });
     console.log('✅ Database migrations completed');
   } catch (error) {
-    // Check if error is about table already existing - this is expected when container restarts with existing volume
-    const errorMessage = error instanceof Error ? error.message : String(error);
-    const causeMessage = error instanceof Error && error.cause ? String(error.cause) : '';
-    
-    if (errorMessage.includes('already exists') || causeMessage.includes('already exists')) {
-      console.log('ℹ️  Database schema already up to date');
-    } else {
     console.error('❌ Migration failed:', error);
     throw error;
   }
-  }
 } else {
   console.log('ℹ️  Skipping migrations during build phase');
 }