feat: migrate from SQLite to PostgreSQL - complete schema and API updates

- Remove better-sqlite3, add pg driver
- Convert schema to PostgreSQL types (serial, timestamp, boolean, jsonb)
- Generate fresh PostgreSQL migrations
- Update database connection layer with pg.Pool
- Fix all API routes: remove JSON.parse/stringify, use native timestamps and booleans
- Update drizzle.config.ts and .env.example for PostgreSQL

src/app/api/annotation-types/route.ts

@@ -30,7 +30,6 @@ export async function POST(request: NextRequest) {
         return NextResponse.json({ message: 'Types already seeded' });
       }
 
-      const now = Math.floor(Date.now() / 1000);
       const defaultTypes = [
         {
           name: 'break_up',
@@ -38,8 +37,7 @@ export async function POST(request: NextRequest) {
           color: '#10b981',
           category: 'marker',
           icon: 'arrowUp',
-          is_active: 1,
-          created_at: now,
+          is_active: true,
         },
         {
           name: 'break_down',
@@ -47,8 +45,7 @@ export async function POST(request: NextRequest) {
           color: '#ef4444',
           category: 'marker',
           icon: 'arrowDown',
-          is_active: 1,
-          created_at: now,
+          is_active: true,
         },
         {
           name: 'line',
@@ -56,8 +53,7 @@ export async function POST(request: NextRequest) {
           color: '#3b82f6',
           category: 'line',
           icon: 'line',
-          is_active: 1,
-          created_at: now,
+          is_active: true,
         },
       ];
 
@@ -83,8 +79,7 @@ export async function POST(request: NextRequest) {
         color,
         category,
         icon: icon || null,
-        is_active: 1,
-        created_at: Math.floor(Date.now() / 1000),
+        is_active: true,
       })
       .returning();
 

src/app/api/annotations/[id]/route.ts

@@ -30,7 +30,7 @@ export async function PATCH(
 
     const result = await db
       .update(annotations)
-      .set({ geometry: JSON.stringify(geometry) })
+      .set({ geometry })
       .where(eq(annotations.id, id))
       .returning();
 
@@ -41,11 +41,7 @@ export async function PATCH(
       );
     }
 
-    const updated = result[0];
-    return NextResponse.json({
-      ...updated,
-      geometry: updated.geometry ? JSON.parse(updated.geometry as string) : null,
-    });
+    return NextResponse.json(result[0]);
   } catch (error: any) {
     return NextResponse.json(
       { error: error.message || 'Failed to update annotation' },

src/app/api/annotations/route.ts

@@ -23,13 +23,7 @@ export async function GET(request: NextRequest) {
       .from(annotations)
       .where(eq(annotations.chart_id, parseInt(chartId, 10)));
 
-    // Parse geometry from JSON string
-    const parsed = allAnnotations.map((annotation) => ({
-      ...annotation,
-      geometry: annotation.geometry ? JSON.parse(annotation.geometry) : null,
-    }));
-
-    return NextResponse.json(parsed);
+    return NextResponse.json(allAnnotations);
   } catch (error: any) {
     return NextResponse.json(
       { error: error.message || 'Failed to fetch annotations' },
@@ -61,30 +55,18 @@ export async function POST(request: NextRequest) {
       );
     }
 
-    // Serialize geometry to JSON string if present
-    const geometryString = geometry ? JSON.stringify(geometry) : null;
-
     const result = await db
       .insert(annotations)
       .values({
         chart_id,
         timestamp,
         label_type,
-        geometry: geometryString,
+        geometry: geometry || null,
         color: color || '#3b82f6',
-        created_at: Math.floor(Date.now() / 1000),
       })
       .returning();
 
-    const created = result[0];
-
-    return NextResponse.json(
-      {
-        ...created,
-        geometry: created.geometry ? JSON.parse(created.geometry) : null,
-      },
-      { status: 201 }
-    );
+    return NextResponse.json(result[0], { status: 201 });
   } catch (error: any) {
     return NextResponse.json(
       { error: error.message || 'Failed to create annotation' },

src/app/api/export/route.ts

@@ -45,7 +45,7 @@ export async function GET(request: NextRequest) {
           price = candleResult[0].close;
         }
       } else if (annotation.label_type === 'line' && annotation.geometry) {
-        const geometry = JSON.parse(annotation.geometry);
+        const geometry = annotation.geometry as any;
         price = geometry.startPrice || null;
       }
 

src/app/api/export/spans/route.ts

@@ -44,7 +44,7 @@ export async function GET(request: NextRequest) {
           confidence: span.confidence,
           outcome: span.outcome,
           notes: span.notes,
-          sub_spans: span.sub_spans ? JSON.parse(span.sub_spans as string) : null,
+          sub_spans: span.sub_spans,
           color: span.color,
           created_at: span.created_at,
         })),

src/app/api/span-annotations/[id]/route.ts

@@ -34,7 +34,7 @@ export async function PATCH(
     if (confidence !== undefined) updates.confidence = confidence;
     if (outcome !== undefined) updates.outcome = outcome;
     if (notes !== undefined) updates.notes = notes;
-    if (sub_spans !== undefined) updates.sub_spans = sub_spans ? JSON.stringify(sub_spans) : null;
+    if (sub_spans !== undefined) updates.sub_spans = sub_spans || null;
 
     const result = await db
       .update(spanAnnotations)

src/app/api/span-annotations/export/route.ts

@@ -69,17 +69,17 @@ export async function GET(request: NextRequest) {
       // Convert to ML pipeline format
       const annotations = spans.map(span => ({
         id: span.id,
-        start_time: new Date(span.start_time * 1000).toISOString(),
-        end_time: new Date(span.end_time * 1000).toISOString(),
+        start_time: span.start_time,
+        end_time: span.end_time,
         label: span.label,
         confidence: span.confidence,
         outcome: span.outcome,
         notes: span.notes,
-        sub_spans: span.sub_spans ? JSON.parse(span.sub_spans) : null,
+        sub_spans: span.sub_spans,
         color: span.color,
         source: span.source,
-        model_prediction: span.model_prediction ? JSON.parse(span.model_prediction) : null,
-        created_at: new Date(span.created_at * 1000).toISOString(),
+        model_prediction: span.model_prediction,
+        created_at: span.created_at,
       }));
 
       return NextResponse.json({ 
@@ -93,12 +93,10 @@ export async function GET(request: NextRequest) {
       const csvRows = ['id,start_time,end_time,label,confidence,outcome,notes'];
       
       for (const span of spans) {
-        const startTime = new Date(span.start_time * 1000).toISOString();
-        const endTime = new Date(span.end_time * 1000).toISOString();
         const notes = span.notes ? `"${span.notes.replace(/"/g, '""')}"` : '';
         
         csvRows.push(
-          `${span.id},${startTime},${endTime},${span.label},${span.confidence || ''},${span.outcome || ''},${notes}`
+          `${span.id},${span.start_time},${span.end_time},${span.label},${span.confidence || ''},${span.outcome || ''},${notes}`
         );
       }
 

src/app/api/span-annotations/route.ts

@@ -71,11 +71,10 @@ export async function POST(request: NextRequest) {
         confidence: confidence || null,
         outcome: outcome || null,
         notes: notes || null,
-        sub_spans: sub_spans ? JSON.stringify(sub_spans) : null,
+        sub_spans: sub_spans || null,
         color: color || '#2196F3',
         source: source || 'human', // 'human', 'model', or 'human_correction'
-        model_prediction: model_prediction ? JSON.stringify(model_prediction) : null,
-        created_at: Math.floor(Date.now() / 1000),
+        model_prediction: model_prediction || null,
       })
       .returning();
 

src/app/api/span-label-types/route.ts

@@ -9,7 +9,7 @@ export async function GET() {
     const types = await db
       .select()
       .from(spanLabelTypes)
-      .where(eq(spanLabelTypes.is_active, 1))
+      .where(eq(spanLabelTypes.is_active, true))
       .orderBy(spanLabelTypes.sort_order);
     
     return NextResponse.json(types);
@@ -42,9 +42,8 @@ export async function POST(request: NextRequest) {
         display_name,
         color,
         hotkey: hotkey || null,
-        is_active: 1,
+        is_active: true,
         sort_order: sort_order ?? 0,
-        created_at: Math.floor(Date.now() / 1000),
       })
       .returning();
 

src/app/api/upload/route.ts

@@ -85,7 +85,6 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
             // Create the chart
             const [newChart] = await db.insert(charts).values({
               name: chartName,
-              created_at: Math.floor(Date.now() / 1000),
             }).returning();
 
             // Parse and prepare candle data
@@ -99,9 +98,10 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
                 if (isNaN(date.getTime())) {
                   throw new Error(`Invalid date format: ${row.time}`);
                 }
-                timestamp = Math.floor(date.getTime() / 1000);
+                timestamp = date; // PostgreSQL timestamp type expects Date object or ISO string
               } else if (typeof row.time === 'number') {
-                timestamp = row.time;
+                // If Unix timestamp (seconds), convert to Date
+                timestamp = new Date(row.time * 1000);
               } else {
                 throw new Error(`Invalid time value: ${row.time}`);
               }

src/lib/db/index.ts

@@ -1,28 +1,30 @@
-import Database from 'better-sqlite3';
-import { drizzle } from 'drizzle-orm/better-sqlite3';
-import { migrate } from 'drizzle-orm/better-sqlite3/migrator';
+import { Pool } from 'pg';
+import { drizzle } from 'drizzle-orm/node-postgres';
+import { migrate } from 'drizzle-orm/node-postgres/migrator';
 import * as schema from './schema';
 import path from 'path';
-import fs from 'fs';
 
-// Ensure data directory exists
-const dataDir = path.join(process.cwd(), 'data');
-if (!fs.existsSync(dataDir)) {
-  fs.mkdirSync(dataDir, { recursive: true });
+// Read DATABASE_URL from environment
+const DATABASE_URL = process.env.DATABASE_URL;
+
+if (!DATABASE_URL) {
+  throw new Error('DATABASE_URL environment variable is not set');
 }
 
-const dbPath = path.join(dataDir, 'candles.db');
-const sqlite = new Database(dbPath);
-export const db = drizzle(sqlite, { schema });
+// Create PostgreSQL connection pool
+const pool = new Pool({
+  connectionString: DATABASE_URL,
+  max: 10,
+});
 
-// Run migrations at startup (for local dev).
-// In Docker, migrations are run by scripts/run-migrations.js before the app starts,
-// so this will be a no-op (all migrations already applied).
+export const db = drizzle(pool, { schema });
+
+// Run migrations at startup (skip during build phase)
 const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build';
 
 if (!isBuildTime) {
   try {
-    migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') });
+    await migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') });
     console.log('✅ Database migrations completed');
   } catch (error) {
     console.error('❌ Migration failed:', error);

src/lib/db/schema.ts

@@ -1,67 +1,67 @@
-import { sqliteTable, integer, real, text, uniqueIndex } from 'drizzle-orm/sqlite-core';
+import { pgTable, serial, text, timestamp, doublePrecision, integer, boolean, jsonb, uniqueIndex } from 'drizzle-orm/pg-core';
 
-export const charts = sqliteTable('charts', {
-  id: integer('id').primaryKey({ autoIncrement: true }),
+export const charts = pgTable('charts', {
+  id: serial('id').primaryKey(),
   name: text('name').notNull().unique(),
-  created_at: integer('created_at').notNull(),
+  created_at: timestamp('created_at').notNull().defaultNow(),
 });
 
-export const candles = sqliteTable('candles', {
-  id: integer('id').primaryKey({ autoIncrement: true }),
+export const candles = pgTable('candles', {
+  id: serial('id').primaryKey(),
   chart_id: integer('chart_id').notNull().references(() => charts.id),
-  time: integer('time').notNull(),
-  open: real('open').notNull(),
-  high: real('high').notNull(),
-  low: real('low').notNull(),
-  close: real('close').notNull(),
+  time: timestamp('time').notNull(),
+  open: doublePrecision('open').notNull(),
+  high: doublePrecision('high').notNull(),
+  low: doublePrecision('low').notNull(),
+  close: doublePrecision('close').notNull(),
 }, (table) => [
   uniqueIndex('candles_chart_time_unique').on(table.chart_id, table.time),
 ]);
 
-export const annotationTypes = sqliteTable('annotation_types', {
-  id: integer('id').primaryKey({ autoIncrement: true }),
+export const annotationTypes = pgTable('annotation_types', {
+  id: serial('id').primaryKey(),
   name: text('name').notNull().unique(), // internal name (e.g., 'break_up')
   display_name: text('display_name').notNull(), // display name (e.g., 'Break Up')
   color: text('color').notNull(), // hex color code
   category: text('category').notNull(), // 'marker' or 'line'
   icon: text('icon'), // icon name or symbol
-  is_active: integer('is_active').notNull().default(1), // 1 = active, 0 = inactive
-  created_at: integer('created_at').notNull(),
+  is_active: boolean('is_active').notNull().default(true), // true = active, false = inactive
+  created_at: timestamp('created_at').notNull().defaultNow(),
 });
 
-export const annotations = sqliteTable('annotations', {
-  id: integer('id').primaryKey({ autoIncrement: true }),
+export const annotations = pgTable('annotations', {
+  id: serial('id').primaryKey(),
   chart_id: integer('chart_id').notNull().references(() => charts.id),
-  timestamp: integer('timestamp').notNull(),
+  timestamp: timestamp('timestamp').notNull(),
   label_type: text('label_type').notNull(),
-  geometry: text('geometry'), // JSON string for line coordinates
+  geometry: jsonb('geometry'), // JSON for line coordinates
   color: text('color').default('#3b82f6'), // hex color code
-  created_at: integer('created_at').notNull(),
+  created_at: timestamp('created_at').notNull().defaultNow(),
 });
 
-export const spanLabelTypes = sqliteTable('span_label_types', {
-  id: integer('id').primaryKey({ autoIncrement: true }),
+export const spanLabelTypes = pgTable('span_label_types', {
+  id: serial('id').primaryKey(),
   name: text('name').notNull().unique(), // internal name (e.g., 'bull_flag')
   display_name: text('display_name').notNull(), // UI label (e.g., 'Bull Flag')
   color: text('color').notNull(), // hex color for rectangle fill
   hotkey: text('hotkey'), // keyboard shortcut (e.g., '1')
-  is_active: integer('is_active').notNull().default(1), // 1 = active, 0 = inactive
+  is_active: boolean('is_active').notNull().default(true), // true = active, false = inactive
   sort_order: integer('sort_order').notNull().default(0), // display order
-  created_at: integer('created_at').notNull(),
+  created_at: timestamp('created_at').notNull().defaultNow(),
 });
 
-export const spanAnnotations = sqliteTable('span_annotations', {
-  id: integer('id').primaryKey({ autoIncrement: true }),
+export const spanAnnotations = pgTable('span_annotations', {
+  id: serial('id').primaryKey(),
   chart_id: integer('chart_id').notNull().references(() => charts.id),
-  start_time: integer('start_time').notNull(), // Unix timestamp of first candle
-  end_time: integer('end_time').notNull(), // Unix timestamp of last candle
+  start_time: timestamp('start_time').notNull(), // timestamp of first candle
+  end_time: timestamp('end_time').notNull(), // timestamp of last candle
   label: text('label').notNull(), // pattern name referencing span_label_types.name
   confidence: integer('confidence'), // 1-5 scale, nullable
   outcome: text('outcome'), // 'win'|'loss'|'breakeven'|null
   notes: text('notes'), // free-text, nullable
-  sub_spans: text('sub_spans'), // JSON array of sub-span objects, nullable
+  sub_spans: jsonb('sub_spans'), // JSON array of sub-span objects, nullable
   color: text('color').notNull().default('#2196F3'), // hex color
   source: text('source').notNull().default('human'), // 'human'|'model'|'human_correction'
-  model_prediction: text('model_prediction'), // JSON metadata when confirming/correcting predictions
-  created_at: integer('created_at').notNull(),
+  model_prediction: jsonb('model_prediction'), // JSON metadata when confirming/correcting predictions
+  created_at: timestamp('created_at').notNull().defaultNow(),
 });