trentuna/candle-annotator
8
0
Fork 0
Code Issues Pull requests Releases Packages Activity Actions

fix: correct timestamp/boolean types for PostgreSQL schema (Date not int, bool not 0/1)

Browse source
This commit is contained in:
Marko Djordjevic 2026-02-17 22:50:31 +01:00
parent e00bd4d804
commit 69634909d1
9 changed files with 75 additions and 131 deletions
Download patch file Download diff file Expand all files Collapse all files

2
next-env.d.ts vendored
View file

@ -1,6 +1,6 @@
/// <reference types="next" />
/// <reference types="next/image-types/global" />
import "./.next/dev/types/routes.d.ts";
import "./.next/types/routes.d.ts";
// NOTE: This file should not be edited
// see https://nextjs.org/docs/app/api-reference/config/typescript for more information.

10
scripts/import_talib_annotations.ts
View file

@ -93,9 +93,9 @@ async function ensureLabelTypes(labels: string[]): Promise<LabelTypeMap> {
display_name: label,
color,
hotkey: null,
is_active: 1,
is_active: true,
sort_order: sortOrder++,
created_at: Math.floor(Date.now() / 1000),
created_at: new Date(),
}).returning();
labelMap[label] = {
@ -135,8 +135,8 @@ async function importAnnotations(
try {
await db.insert(spanAnnotations).values({
chart_id: chartId,
start_time: ann.start_time,
end_time: ann.end_time,
start_time: new Date(ann.start_time * 1000),
end_time: new Date(ann.end_time * 1000),
label: ann.label,
confidence: ann.confidence || null,
outcome: null,
@ -145,7 +145,7 @@ async function importAnnotations(
color: labelInfo.color,
source: ann.source || 'programmatic',
model_prediction: null,
created_at: Math.floor(Date.now() / 1000),
created_at: new Date(),
});
imported++;

2
scripts/list_charts.ts
View file

@ -19,7 +19,7 @@ async function main() {
} else {
console.log(`Found ${allCharts.length} chart(s):\n`);
for (const chart of allCharts) {
const date = new Date(chart.created_at * 1000).toISOString();
const date = chart.created_at.toISOString();
console.log(` ID: ${chart.id}`);
console.log(` Name: ${chart.name}`);
console.log(` Created: ${date}`);

27
scripts/migrate-sqlite-to-postgres.ts
View file

@ -79,8 +79,13 @@ const stats: MigrationStats[] = [];
/**
* Convert SQLite integer timestamp (Unix seconds) to JavaScript Date
*/
function sqliteTimestampToDate(timestamp: number | null): Date | null {
if (!timestamp) return null;
function sqliteTimestampToDate(timestamp: number | null): Date | undefined {
if (!timestamp) return undefined;
return new Date(timestamp * 1000);
}
function sqliteTimestampToDateRequired(timestamp: number | null): Date {
if (!timestamp) throw new Error(`Required timestamp is null/zero: ${timestamp}`);
return new Date(timestamp * 1000);
}
@ -157,7 +162,7 @@ async function migrateCharts() {
await pg.insert(schema.charts).values({
id: row.id,
name: row.name,
created_at: sqliteTimestampToDate(row.created_at),
created_at: sqliteTimestampToDateRequired(row.created_at),
});
migrated++;
@ -201,7 +206,7 @@ async function migrateCandles() {
await pg.insert(schema.candles).values({
id: row.id,
chart_id: row.chart_id,
time: sqliteTimestampToDate(row.time),
time: sqliteTimestampToDateRequired(row.time),
open: row.open,
high: row.high,
low: row.low,
@ -254,7 +259,7 @@ async function migrateAnnotationTypes() {
category: row.category,
icon: row.icon,
is_active: sqliteBooleanToBoolean(row.is_active),
created_at: sqliteTimestampToDate(row.created_at),
created_at: sqliteTimestampToDateRequired(row.created_at),
});
migrated++;
@ -298,11 +303,11 @@ async function migrateAnnotations() {
await pg.insert(schema.annotations).values({
id: row.id,
chart_id: row.chart_id,
timestamp: sqliteTimestampToDate(row.timestamp),
timestamp: sqliteTimestampToDateRequired(row.timestamp),
label_type: row.label_type,
geometry: sqliteJsonToObject(row.geometry),
color: row.color || '#3b82f6',
created_at: sqliteTimestampToDate(row.created_at),
created_at: sqliteTimestampToDateRequired(row.created_at),
});
migrated++;
@ -351,7 +356,7 @@ async function migrateSpanLabelTypes() {
hotkey: row.hotkey,
is_active: sqliteBooleanToBoolean(row.is_active),
sort_order: row.sort_order || 0,
created_at: sqliteTimestampToDate(row.created_at),
created_at: sqliteTimestampToDateRequired(row.created_at),
});
migrated++;
@ -395,8 +400,8 @@ async function migrateSpanAnnotations() {
await pg.insert(schema.spanAnnotations).values({
id: row.id,
chart_id: row.chart_id,
start_time: sqliteTimestampToDate(row.start_time),
end_time: sqliteTimestampToDate(row.end_time),
start_time: sqliteTimestampToDateRequired(row.start_time),
end_time: sqliteTimestampToDateRequired(row.end_time),
label: row.label,
confidence: row.confidence,
outcome: row.outcome,
@ -405,7 +410,7 @@ async function migrateSpanAnnotations() {
color: row.color || '#2196F3',
source: row.source || 'human',
model_prediction: sqliteJsonToObject(row.model_prediction),
created_at: sqliteTimestampToDate(row.created_at),
created_at: sqliteTimestampToDateRequired(row.created_at),
});
migrated++;

BIN
services/ml/models/2faa1bc8-819a-4310-b08c-a901f0d74e98.pkl Normal file
View file

Binary file not shown.

73
src/app/api/export/spans/route.ts
View file

@ -1,7 +1,11 @@
import { NextRequest, NextResponse } from 'next/server';
import { db } from '@/lib/db';
import { spanAnnotations, candles } from '@/lib/db/schema';
import { eq, and } from 'drizzle-orm';
import { eq } from 'drizzle-orm';
function toUnix(d: Date): number {
return Math.floor(d.getTime() / 1000);
}
export async function GET(request: NextRequest) {
try {
@ -38,15 +42,15 @@ export async function GET(request: NextRequest) {
total_spans: spans.length,
spans: spans.map((span) => ({
id: span.id,
start_time: span.start_time,
end_time: span.end_time,
start_time: toUnix(span.start_time),
end_time: toUnix(span.end_time),
label: span.label,
confidence: span.confidence,
outcome: span.outcome,
notes: span.notes,
sub_spans: span.sub_spans,
color: span.color,
created_at: span.created_at,
created_at: toUnix(span.created_at),
})),
summary: {
labels: spans.reduce((acc, span) => {
@ -69,25 +73,23 @@ export async function GET(request: NextRequest) {
];
for (const span of spans) {
// Find candles in the span range
const spanCandles = chartCandles.filter(
(c) => c.time >= span.start_time && c.time <= span.end_time
);
const spanStart = toUnix(span.start_time);
const spanEnd = toUnix(span.end_time);
// Add context padding
const contextStart = span.start_time - contextPadding * 60; // Assuming 1-minute candles
const contextEnd = span.end_time + contextPadding * 60;
// Add context padding (assuming 1-minute candles)
const contextStart = spanStart - contextPadding * 60;
const contextEnd = spanEnd + contextPadding * 60;
const contextCandles = chartCandles.filter(
(c) => c.time >= contextStart && c.time <= contextEnd
);
const contextCandles = chartCandles.filter((c) => {
const t = toUnix(c.time);
return t >= contextStart && t <= contextEnd;
});
// Create one row per candle in the context window
contextCandles.forEach((candle, idx) => {
let position = 'context';
if (candle.time >= span.start_time && candle.time <= span.end_time) {
position = 'span';
}
contextCandles.forEach((candle) => {
const candleTime = toUnix(candle.time);
const position =
candleTime >= spanStart && candleTime <= spanEnd ? 'span' : 'context';
csvRows.push(
[
@ -95,11 +97,11 @@ export async function GET(request: NextRequest) {
span.label,
span.confidence || '',
span.outcome || '',
span.start_time,
span.end_time,
spanStart,
spanEnd,
contextStart,
contextEnd,
candle.time,
candleTime,
candle.open,
candle.high,
candle.low,
@ -119,24 +121,29 @@ export async function GET(request: NextRequest) {
},
});
} else if (format === 'bio') {
// BIO-tagged CSV
// One row per candle, with BIO tags for each label type
// BIO-tagged CSV — one row per candle, with BIO tags for each label type
// Get all unique labels
const labelTypes = Array.from(new Set(spans.map((s) => s.label)));
// Pre-compute unix times for spans
const spansUnix = spans.map((s) => ({
...s,
startUnix: toUnix(s.start_time),
endUnix: toUnix(s.end_time),
}));
// Create header
const header = ['time', 'open', 'high', 'low', 'close'];
labelTypes.forEach((label) => {
header.push(`bio_${label}`);
});
labelTypes.forEach((label) => header.push(`bio_${label}`));
const csvRows: string[] = [header.join(',')];
// Process each candle
chartCandles.forEach((candle) => {
const candleTime = toUnix(candle.time);
const row = [
candle.time.toString(),
candleTime.toString(),
candle.open.toString(),
candle.high.toString(),
candle.low.toString(),
@ -145,16 +152,14 @@ export async function GET(request: NextRequest) {
// For each label type, determine BIO tag
labelTypes.forEach((label) => {
const spansWithLabel = spans.filter((s) => s.label === label);
const spansWithLabel = spansUnix.filter((s) => s.label === label);
let bioTag = 'O'; // Outside by default
for (const span of spansWithLabel) {
if (candle.time >= span.start_time && candle.time <= span.end_time) {
// Check if this is the first candle in the span
const isFirst = candle.time === span.start_time;
bioTag = isFirst ? `B-${label}` : `I-${label}`;
break; // Use first matching span
if (candleTime >= span.startUnix && candleTime <= span.endUnix) {
bioTag = candleTime === span.startUnix ? `B-${label}` : `I-${label}`;
break;
}
}

4
src/app/api/upload/route.ts
View file

@ -89,7 +89,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
// Parse and prepare candle data
const candleData = rows.map((row) => {
let timestamp: number;
let timestamp: Date;
// Handle both date strings and Unix timestamps
if (typeof row.time === 'string') {
@ -98,7 +98,7 @@ export async function POST(request: NextRequest): Promise<NextResponse> {
if (isNaN(date.getTime())) {
throw new Error(`Invalid date format: ${row.time}`);
}
timestamp = date; // PostgreSQL timestamp type expects Date object or ISO string
timestamp = date;
} else if (typeof row.time === 'number') {
// If Unix timestamp (seconds), convert to Date
timestamp = new Date(row.time * 1000);

8
src/lib/db/seed-annotation-types.ts
View file

@ -2,8 +2,6 @@ import { db } from './index';
import { annotationTypes } from './schema';
export async function seedAnnotationTypes() {
const now = Math.floor(Date.now() / 1000);
const defaultTypes = [
{
name: 'break_up',
@ -11,8 +9,6 @@ export async function seedAnnotationTypes() {
color: '#10b981',
category: 'marker',
icon: 'arrowUp',
is_active: 1,
created_at: now,
},
{
name: 'break_down',
@ -20,8 +16,6 @@ export async function seedAnnotationTypes() {
color: '#ef4444',
category: 'marker',
icon: 'arrowDown',
is_active: 1,
created_at: now,
},
{
name: 'line',
@ -29,8 +23,6 @@ export async function seedAnnotationTypes() {
color: '#3b82f6',
category: 'line',
icon: 'line',
is_active: 1,
created_at: now,
},
];

72
src/lib/db/seed-span-label-types.ts
View file

@ -2,72 +2,14 @@ import { db } from './index';
import { spanLabelTypes } from './schema';
export async function seedSpanLabelTypes() {
const now = Math.floor(Date.now() / 1000);
const defaultTypes = [
{
name: 'bull_flag',
display_name: 'Bull Flag',
color: '#4CAF50',
hotkey: '1',
is_active: 1,
sort_order: 1,
created_at: now,
},
{
name: 'bear_flag',
display_name: 'Bear Flag',
color: '#F44336',
hotkey: '2',
is_active: 1,
sort_order: 2,
created_at: now,
},
{
name: 'head_and_shoulders',
display_name: 'Head and Shoulders',
color: '#9C27B0',
hotkey: '3',
is_active: 1,
sort_order: 3,
created_at: now,
},
{
name: 'double_bottom',
display_name: 'Double Bottom',
color: '#2196F3',
hotkey: '4',
is_active: 1,
sort_order: 4,
created_at: now,
},
{
name: 'wedge_up',
display_name: 'Wedge Up',
color: '#FF9800',
hotkey: '5',
is_active: 1,
sort_order: 5,
created_at: now,
},
{
name: 'wedge_down',
display_name: 'Wedge Down',
color: '#FF5722',
hotkey: '6',
is_active: 1,
sort_order: 6,
created_at: now,
},
{
name: 'custom',
display_name: 'Custom',
color: '#607D8B',
hotkey: '7',
is_active: 1,
sort_order: 7,
created_at: now,
},
{ name: 'bull_flag', display_name: 'Bull Flag', color: '#4CAF50', hotkey: '1', sort_order: 1 },
{ name: 'bear_flag', display_name: 'Bear Flag', color: '#F44336', hotkey: '2', sort_order: 2 },
{ name: 'head_and_shoulders', display_name: 'Head and Shoulders', color: '#9C27B0', hotkey: '3', sort_order: 3 },
{ name: 'double_bottom', display_name: 'Double Bottom', color: '#2196F3', hotkey: '4', sort_order: 4 },
{ name: 'wedge_up', display_name: 'Wedge Up', color: '#FF9800', hotkey: '5', sort_order: 5 },
{ name: 'wedge_down', display_name: 'Wedge Down', color: '#FF5722', hotkey: '6', sort_order: 6 },
{ name: 'custom', display_name: 'Custom', color: '#607D8B', hotkey: '7', sort_order: 7 },
];
// Check if types already exist