fix: correct timestamp/boolean types for PostgreSQL schema (Date not int, bool not 0/1)

This commit is contained in:
Marko Djordjevic 2026-02-17 22:50:31 +01:00
parent e00bd4d804
commit 69634909d1
9 changed files with 75 additions and 131 deletions

2
next-env.d.ts vendored
View file

@ -1,6 +1,6 @@
/// <reference types="next" /> /// <reference types="next" />
/// <reference types="next/image-types/global" /> /// <reference types="next/image-types/global" />
import "./.next/dev/types/routes.d.ts"; import "./.next/types/routes.d.ts";
// NOTE: This file should not be edited // NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information. // see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

View file

@ -93,9 +93,9 @@ async function ensureLabelTypes(labels: string[]): Promise<LabelTypeMap> {
display_name: label, display_name: label,
color, color,
hotkey: null, hotkey: null,
is_active: 1, is_active: true,
sort_order: sortOrder++, sort_order: sortOrder++,
created_at: Math.floor(Date.now() / 1000), created_at: new Date(),
}).returning(); }).returning();
labelMap[label] = { labelMap[label] = {
@ -135,8 +135,8 @@ async function importAnnotations(
try { try {
await db.insert(spanAnnotations).values({ await db.insert(spanAnnotations).values({
chart_id: chartId, chart_id: chartId,
start_time: ann.start_time, start_time: new Date(ann.start_time * 1000),
end_time: ann.end_time, end_time: new Date(ann.end_time * 1000),
label: ann.label, label: ann.label,
confidence: ann.confidence || null, confidence: ann.confidence || null,
outcome: null, outcome: null,
@ -145,7 +145,7 @@ async function importAnnotations(
color: labelInfo.color, color: labelInfo.color,
source: ann.source || 'programmatic', source: ann.source || 'programmatic',
model_prediction: null, model_prediction: null,
created_at: Math.floor(Date.now() / 1000), created_at: new Date(),
}); });
imported++; imported++;

View file

@ -19,7 +19,7 @@ async function main() {
} else { } else {
console.log(`Found ${allCharts.length} chart(s):\n`); console.log(`Found ${allCharts.length} chart(s):\n`);
for (const chart of allCharts) { for (const chart of allCharts) {
const date = new Date(chart.created_at * 1000).toISOString(); const date = chart.created_at.toISOString();
console.log(` ID: ${chart.id}`); console.log(` ID: ${chart.id}`);
console.log(` Name: ${chart.name}`); console.log(` Name: ${chart.name}`);
console.log(` Created: ${date}`); console.log(` Created: ${date}`);

View file

@ -79,8 +79,13 @@ const stats: MigrationStats[] = [];
/** /**
* Convert SQLite integer timestamp (Unix seconds) to JavaScript Date * Convert SQLite integer timestamp (Unix seconds) to JavaScript Date
*/ */
function sqliteTimestampToDate(timestamp: number | null): Date | null { function sqliteTimestampToDate(timestamp: number | null): Date | undefined {
if (!timestamp) return null; if (!timestamp) return undefined;
return new Date(timestamp * 1000);
}
function sqliteTimestampToDateRequired(timestamp: number | null): Date {
if (!timestamp) throw new Error(`Required timestamp is null/zero: ${timestamp}`);
return new Date(timestamp * 1000); return new Date(timestamp * 1000);
} }
@ -157,7 +162,7 @@ async function migrateCharts() {
await pg.insert(schema.charts).values({ await pg.insert(schema.charts).values({
id: row.id, id: row.id,
name: row.name, name: row.name,
created_at: sqliteTimestampToDate(row.created_at), created_at: sqliteTimestampToDateRequired(row.created_at),
}); });
migrated++; migrated++;
@ -201,7 +206,7 @@ async function migrateCandles() {
await pg.insert(schema.candles).values({ await pg.insert(schema.candles).values({
id: row.id, id: row.id,
chart_id: row.chart_id, chart_id: row.chart_id,
time: sqliteTimestampToDate(row.time), time: sqliteTimestampToDateRequired(row.time),
open: row.open, open: row.open,
high: row.high, high: row.high,
low: row.low, low: row.low,
@ -254,7 +259,7 @@ async function migrateAnnotationTypes() {
category: row.category, category: row.category,
icon: row.icon, icon: row.icon,
is_active: sqliteBooleanToBoolean(row.is_active), is_active: sqliteBooleanToBoolean(row.is_active),
created_at: sqliteTimestampToDate(row.created_at), created_at: sqliteTimestampToDateRequired(row.created_at),
}); });
migrated++; migrated++;
@ -298,11 +303,11 @@ async function migrateAnnotations() {
await pg.insert(schema.annotations).values({ await pg.insert(schema.annotations).values({
id: row.id, id: row.id,
chart_id: row.chart_id, chart_id: row.chart_id,
timestamp: sqliteTimestampToDate(row.timestamp), timestamp: sqliteTimestampToDateRequired(row.timestamp),
label_type: row.label_type, label_type: row.label_type,
geometry: sqliteJsonToObject(row.geometry), geometry: sqliteJsonToObject(row.geometry),
color: row.color || '#3b82f6', color: row.color || '#3b82f6',
created_at: sqliteTimestampToDate(row.created_at), created_at: sqliteTimestampToDateRequired(row.created_at),
}); });
migrated++; migrated++;
@ -351,7 +356,7 @@ async function migrateSpanLabelTypes() {
hotkey: row.hotkey, hotkey: row.hotkey,
is_active: sqliteBooleanToBoolean(row.is_active), is_active: sqliteBooleanToBoolean(row.is_active),
sort_order: row.sort_order || 0, sort_order: row.sort_order || 0,
created_at: sqliteTimestampToDate(row.created_at), created_at: sqliteTimestampToDateRequired(row.created_at),
}); });
migrated++; migrated++;
@ -395,8 +400,8 @@ async function migrateSpanAnnotations() {
await pg.insert(schema.spanAnnotations).values({ await pg.insert(schema.spanAnnotations).values({
id: row.id, id: row.id,
chart_id: row.chart_id, chart_id: row.chart_id,
start_time: sqliteTimestampToDate(row.start_time), start_time: sqliteTimestampToDateRequired(row.start_time),
end_time: sqliteTimestampToDate(row.end_time), end_time: sqliteTimestampToDateRequired(row.end_time),
label: row.label, label: row.label,
confidence: row.confidence, confidence: row.confidence,
outcome: row.outcome, outcome: row.outcome,
@ -405,7 +410,7 @@ async function migrateSpanAnnotations() {
color: row.color || '#2196F3', color: row.color || '#2196F3',
source: row.source || 'human', source: row.source || 'human',
model_prediction: sqliteJsonToObject(row.model_prediction), model_prediction: sqliteJsonToObject(row.model_prediction),
created_at: sqliteTimestampToDate(row.created_at), created_at: sqliteTimestampToDateRequired(row.created_at),
}); });
migrated++; migrated++;

View file

@ -1,7 +1,11 @@
import { NextRequest, NextResponse } from 'next/server'; import { NextRequest, NextResponse } from 'next/server';
import { db } from '@/lib/db'; import { db } from '@/lib/db';
import { spanAnnotations, candles } from '@/lib/db/schema'; import { spanAnnotations, candles } from '@/lib/db/schema';
import { eq, and } from 'drizzle-orm'; import { eq } from 'drizzle-orm';
function toUnix(d: Date): number {
return Math.floor(d.getTime() / 1000);
}
export async function GET(request: NextRequest) { export async function GET(request: NextRequest) {
try { try {
@ -38,15 +42,15 @@ export async function GET(request: NextRequest) {
total_spans: spans.length, total_spans: spans.length,
spans: spans.map((span) => ({ spans: spans.map((span) => ({
id: span.id, id: span.id,
start_time: span.start_time, start_time: toUnix(span.start_time),
end_time: span.end_time, end_time: toUnix(span.end_time),
label: span.label, label: span.label,
confidence: span.confidence, confidence: span.confidence,
outcome: span.outcome, outcome: span.outcome,
notes: span.notes, notes: span.notes,
sub_spans: span.sub_spans, sub_spans: span.sub_spans,
color: span.color, color: span.color,
created_at: span.created_at, created_at: toUnix(span.created_at),
})), })),
summary: { summary: {
labels: spans.reduce((acc, span) => { labels: spans.reduce((acc, span) => {
@ -69,25 +73,23 @@ export async function GET(request: NextRequest) {
]; ];
for (const span of spans) { for (const span of spans) {
// Find candles in the span range const spanStart = toUnix(span.start_time);
const spanCandles = chartCandles.filter( const spanEnd = toUnix(span.end_time);
(c) => c.time >= span.start_time && c.time <= span.end_time
);
// Add context padding // Add context padding (assuming 1-minute candles)
const contextStart = span.start_time - contextPadding * 60; // Assuming 1-minute candles const contextStart = spanStart - contextPadding * 60;
const contextEnd = span.end_time + contextPadding * 60; const contextEnd = spanEnd + contextPadding * 60;
const contextCandles = chartCandles.filter( const contextCandles = chartCandles.filter((c) => {
(c) => c.time >= contextStart && c.time <= contextEnd const t = toUnix(c.time);
); return t >= contextStart && t <= contextEnd;
});
// Create one row per candle in the context window // Create one row per candle in the context window
contextCandles.forEach((candle, idx) => { contextCandles.forEach((candle) => {
let position = 'context'; const candleTime = toUnix(candle.time);
if (candle.time >= span.start_time && candle.time <= span.end_time) { const position =
position = 'span'; candleTime >= spanStart && candleTime <= spanEnd ? 'span' : 'context';
}
csvRows.push( csvRows.push(
[ [
@ -95,11 +97,11 @@ export async function GET(request: NextRequest) {
span.label, span.label,
span.confidence || '', span.confidence || '',
span.outcome || '', span.outcome || '',
span.start_time, spanStart,
span.end_time, spanEnd,
contextStart, contextStart,
contextEnd, contextEnd,
candle.time, candleTime,
candle.open, candle.open,
candle.high, candle.high,
candle.low, candle.low,
@ -119,24 +121,29 @@ export async function GET(request: NextRequest) {
}, },
}); });
} else if (format === 'bio') { } else if (format === 'bio') {
// BIO-tagged CSV // BIO-tagged CSV — one row per candle, with BIO tags for each label type
// One row per candle, with BIO tags for each label type
// Get all unique labels // Get all unique labels
const labelTypes = Array.from(new Set(spans.map((s) => s.label))); const labelTypes = Array.from(new Set(spans.map((s) => s.label)));
// Pre-compute unix times for spans
const spansUnix = spans.map((s) => ({
...s,
startUnix: toUnix(s.start_time),
endUnix: toUnix(s.end_time),
}));
// Create header // Create header
const header = ['time', 'open', 'high', 'low', 'close']; const header = ['time', 'open', 'high', 'low', 'close'];
labelTypes.forEach((label) => { labelTypes.forEach((label) => header.push(`bio_${label}`));
header.push(`bio_${label}`);
});
const csvRows: string[] = [header.join(',')]; const csvRows: string[] = [header.join(',')];
// Process each candle // Process each candle
chartCandles.forEach((candle) => { chartCandles.forEach((candle) => {
const candleTime = toUnix(candle.time);
const row = [ const row = [
candle.time.toString(), candleTime.toString(),
candle.open.toString(), candle.open.toString(),
candle.high.toString(), candle.high.toString(),
candle.low.toString(), candle.low.toString(),
@ -145,16 +152,14 @@ export async function GET(request: NextRequest) {
// For each label type, determine BIO tag // For each label type, determine BIO tag
labelTypes.forEach((label) => { labelTypes.forEach((label) => {
const spansWithLabel = spans.filter((s) => s.label === label); const spansWithLabel = spansUnix.filter((s) => s.label === label);
let bioTag = 'O'; // Outside by default let bioTag = 'O'; // Outside by default
for (const span of spansWithLabel) { for (const span of spansWithLabel) {
if (candle.time >= span.start_time && candle.time <= span.end_time) { if (candleTime >= span.startUnix && candleTime <= span.endUnix) {
// Check if this is the first candle in the span bioTag = candleTime === span.startUnix ? `B-${label}` : `I-${label}`;
const isFirst = candle.time === span.start_time; break;
bioTag = isFirst ? `B-${label}` : `I-${label}`;
break; // Use first matching span
} }
} }

View file

@ -89,7 +89,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
// Parse and prepare candle data // Parse and prepare candle data
const candleData = rows.map((row) => { const candleData = rows.map((row) => {
let timestamp: number; let timestamp: Date;
// Handle both date strings and Unix timestamps // Handle both date strings and Unix timestamps
if (typeof row.time === 'string') { if (typeof row.time === 'string') {
@ -98,7 +98,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
if (isNaN(date.getTime())) { if (isNaN(date.getTime())) {
throw new Error(`Invalid date format: ${row.time}`); throw new Error(`Invalid date format: ${row.time}`);
} }
timestamp = date; // PostgreSQL timestamp type expects Date object or ISO string timestamp = date;
} else if (typeof row.time === 'number') { } else if (typeof row.time === 'number') {
// If Unix timestamp (seconds), convert to Date // If Unix timestamp (seconds), convert to Date
timestamp = new Date(row.time * 1000); timestamp = new Date(row.time * 1000);

View file

@ -2,8 +2,6 @@ import { db } from './index';
import { annotationTypes } from './schema'; import { annotationTypes } from './schema';
export async function seedAnnotationTypes() { export async function seedAnnotationTypes() {
const now = Math.floor(Date.now() / 1000);
const defaultTypes = [ const defaultTypes = [
{ {
name: 'break_up', name: 'break_up',
@ -11,8 +9,6 @@ export async function seedAnnotationTypes() {
color: '#10b981', color: '#10b981',
category: 'marker', category: 'marker',
icon: 'arrowUp', icon: 'arrowUp',
is_active: 1,
created_at: now,
}, },
{ {
name: 'break_down', name: 'break_down',
@ -20,8 +16,6 @@ export async function seedAnnotationTypes() {
color: '#ef4444', color: '#ef4444',
category: 'marker', category: 'marker',
icon: 'arrowDown', icon: 'arrowDown',
is_active: 1,
created_at: now,
}, },
{ {
name: 'line', name: 'line',
@ -29,14 +23,12 @@ export async function seedAnnotationTypes() {
color: '#3b82f6', color: '#3b82f6',
category: 'line', category: 'line',
icon: 'line', icon: 'line',
is_active: 1,
created_at: now,
}, },
]; ];
// Check if types already exist // Check if types already exist
const existing = await db.select().from(annotationTypes); const existing = await db.select().from(annotationTypes);
if (existing.length === 0) { if (existing.length === 0) {
await db.insert(annotationTypes).values(defaultTypes); await db.insert(annotationTypes).values(defaultTypes);
console.log('Seeded default annotation types'); console.log('Seeded default annotation types');

View file

@ -2,77 +2,19 @@ import { db } from './index';
import { spanLabelTypes } from './schema'; import { spanLabelTypes } from './schema';
export async function seedSpanLabelTypes() { export async function seedSpanLabelTypes() {
const now = Math.floor(Date.now() / 1000);
const defaultTypes = [ const defaultTypes = [
{ { name: 'bull_flag', display_name: 'Bull Flag', color: '#4CAF50', hotkey: '1', sort_order: 1 },
name: 'bull_flag', { name: 'bear_flag', display_name: 'Bear Flag', color: '#F44336', hotkey: '2', sort_order: 2 },
display_name: 'Bull Flag', { name: 'head_and_shoulders', display_name: 'Head and Shoulders', color: '#9C27B0', hotkey: '3', sort_order: 3 },
color: '#4CAF50', { name: 'double_bottom', display_name: 'Double Bottom', color: '#2196F3', hotkey: '4', sort_order: 4 },
hotkey: '1', { name: 'wedge_up', display_name: 'Wedge Up', color: '#FF9800', hotkey: '5', sort_order: 5 },
is_active: 1, { name: 'wedge_down', display_name: 'Wedge Down', color: '#FF5722', hotkey: '6', sort_order: 6 },
sort_order: 1, { name: 'custom', display_name: 'Custom', color: '#607D8B', hotkey: '7', sort_order: 7 },
created_at: now,
},
{
name: 'bear_flag',
display_name: 'Bear Flag',
color: '#F44336',
hotkey: '2',
is_active: 1,
sort_order: 2,
created_at: now,
},
{
name: 'head_and_shoulders',
display_name: 'Head and Shoulders',
color: '#9C27B0',
hotkey: '3',
is_active: 1,
sort_order: 3,
created_at: now,
},
{
name: 'double_bottom',
display_name: 'Double Bottom',
color: '#2196F3',
hotkey: '4',
is_active: 1,
sort_order: 4,
created_at: now,
},
{
name: 'wedge_up',
display_name: 'Wedge Up',
color: '#FF9800',
hotkey: '5',
is_active: 1,
sort_order: 5,
created_at: now,
},
{
name: 'wedge_down',
display_name: 'Wedge Down',
color: '#FF5722',
hotkey: '6',
is_active: 1,
sort_order: 6,
created_at: now,
},
{
name: 'custom',
display_name: 'Custom',
color: '#607D8B',
hotkey: '7',
is_active: 1,
sort_order: 7,
created_at: now,
},
]; ];
// Check if types already exist // Check if types already exist
const existing = await db.select().from(spanLabelTypes); const existing = await db.select().from(spanLabelTypes);
if (existing.length === 0) { if (existing.length === 0) {
await db.insert(spanLabelTypes).values(defaultTypes); await db.insert(spanLabelTypes).values(defaultTypes);
console.log('Seeded default span label types'); console.log('Seeded default span label types');