diff --git a/.env.example b/.env.example index 1bc757e..d8cef2c 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,6 @@ NODE_ENV=production PORT=3000 -DATABASE_PATH=/app/data/candles.db +DATABASE_URL=postgresql://ml_user:ml_password@postgres:5432/candle_annotator # ML Inference Service Configuration INFERENCE_API_URL=http://localhost:8001 diff --git a/CLAUDE.md b/CLAUDE.md index ce95e91..70e9424 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -8,8 +8,6 @@ use context7, lightweight charts Always use Context7 MCP when I need library/API documentation, code generation, setup or configuration steps without me having to explicitly ask. -## RTK Known Issues -- **npm run bug**: Instead of `rtk npm run build`, use `rtk npm build` (RTK has a bug with the `run` subcommand) commit after every task. pause after every section. diff --git a/drizzle.config.ts b/drizzle.config.ts index 26c6900..1da8564 100644 --- a/drizzle.config.ts +++ b/drizzle.config.ts @@ -3,8 +3,8 @@ import { defineConfig } from 'drizzle-kit'; export default defineConfig({ schema: './src/lib/db/schema.ts', out: './drizzle', - dialect: 'sqlite', + dialect: 'postgresql', dbCredentials: { - url: './data/candles.db', + url: process.env.DATABASE_URL!, }, }); diff --git a/drizzle/0000_goofy_captain_midlands.sql b/drizzle/0000_goofy_captain_midlands.sql deleted file mode 100644 index 62b2b21..0000000 --- a/drizzle/0000_goofy_captain_midlands.sql +++ /dev/null @@ -1,19 +0,0 @@ -CREATE TABLE `annotations` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `timestamp` integer NOT NULL, - `label_type` text NOT NULL, - `geometry` text, - `color` text DEFAULT '#3b82f6', - `created_at` integer NOT NULL -); ---> statement-breakpoint -CREATE TABLE `candles` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `time` integer NOT NULL, - `open` real NOT NULL, - `high` real NOT NULL, - `low` real NOT NULL, - `close` real NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `candles_time_unique` ON `candles` (`time`); \ No newline at end of file diff --git a/drizzle/0000_nifty_gauntlet.sql b/drizzle/0000_nifty_gauntlet.sql new file mode 100644 index 0000000..1f52cae --- /dev/null +++ b/drizzle/0000_nifty_gauntlet.sql @@ -0,0 +1,71 @@ +CREATE TABLE "annotation_types" ( + "id" serial PRIMARY KEY NOT NULL, + "name" text NOT NULL, + "display_name" text NOT NULL, + "color" text NOT NULL, + "category" text NOT NULL, + "icon" text, + "is_active" boolean DEFAULT true NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "annotation_types_name_unique" UNIQUE("name") +); +--> statement-breakpoint +CREATE TABLE "annotations" ( + "id" serial PRIMARY KEY NOT NULL, + "chart_id" integer NOT NULL, + "timestamp" timestamp NOT NULL, + "label_type" text NOT NULL, + "geometry" jsonb, + "color" text DEFAULT '#3b82f6', + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "candles" ( + "id" serial PRIMARY KEY NOT NULL, + "chart_id" integer NOT NULL, + "time" timestamp NOT NULL, + "open" double precision NOT NULL, + "high" double precision NOT NULL, + "low" double precision NOT NULL, + "close" double precision NOT NULL +); +--> statement-breakpoint +CREATE TABLE "charts" ( + "id" serial PRIMARY KEY NOT NULL, + "name" text NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "charts_name_unique" UNIQUE("name") +); +--> statement-breakpoint +CREATE TABLE "span_annotations" ( + "id" serial PRIMARY KEY NOT NULL, + "chart_id" integer NOT NULL, + "start_time" timestamp NOT NULL, + "end_time" timestamp NOT NULL, + "label" text NOT NULL, + "confidence" integer, + "outcome" text, + "notes" text, + "sub_spans" jsonb, + "color" text DEFAULT '#2196F3' NOT NULL, + "source" text DEFAULT 'human' NOT NULL, + "model_prediction" jsonb, + "created_at" timestamp DEFAULT now() NOT NULL +); +--> statement-breakpoint +CREATE TABLE "span_label_types" ( + "id" serial PRIMARY KEY NOT NULL, + "name" text NOT NULL, + "display_name" text NOT NULL, + "color" text NOT NULL, + "hotkey" text, + "is_active" boolean DEFAULT true NOT NULL, + "sort_order" integer DEFAULT 0 NOT NULL, + "created_at" timestamp DEFAULT now() NOT NULL, + CONSTRAINT "span_label_types_name_unique" UNIQUE("name") +); +--> statement-breakpoint +ALTER TABLE "annotations" ADD CONSTRAINT "annotations_chart_id_charts_id_fk" FOREIGN KEY ("chart_id") REFERENCES "public"."charts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "candles" ADD CONSTRAINT "candles_chart_id_charts_id_fk" FOREIGN KEY ("chart_id") REFERENCES "public"."charts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +ALTER TABLE "span_annotations" ADD CONSTRAINT "span_annotations_chart_id_charts_id_fk" FOREIGN KEY ("chart_id") REFERENCES "public"."charts"("id") ON DELETE no action ON UPDATE no action;--> statement-breakpoint +CREATE UNIQUE INDEX "candles_chart_time_unique" ON "candles" USING btree ("chart_id","time"); \ No newline at end of file diff --git a/drizzle/0001_sticky_shinko_yamashiro.sql b/drizzle/0001_sticky_shinko_yamashiro.sql deleted file mode 100644 index 5b071dc..0000000 --- a/drizzle/0001_sticky_shinko_yamashiro.sql +++ /dev/null @@ -1,12 +0,0 @@ -CREATE TABLE `annotation_types` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text NOT NULL, - `display_name` text NOT NULL, - `color` text NOT NULL, - `category` text NOT NULL, - `icon` text, - `is_active` integer DEFAULT 1 NOT NULL, - `created_at` integer NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `annotation_types_name_unique` ON `annotation_types` (`name`); \ No newline at end of file diff --git a/drizzle/0002_careful_synch.sql b/drizzle/0002_careful_synch.sql deleted file mode 100644 index a537ff3..0000000 --- a/drizzle/0002_careful_synch.sql +++ /dev/null @@ -1,74 +0,0 @@ --- Create charts table -CREATE TABLE `charts` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text NOT NULL, - `created_at` integer NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `charts_name_unique` ON `charts` (`name`); ---> statement-breakpoint - --- Insert default chart if candles exist -INSERT INTO `charts` (`name`, `created_at`) -SELECT 'Imported Data', CAST(strftime('%s', 'now') AS INTEGER) -WHERE EXISTS (SELECT 1 FROM `candles` LIMIT 1); ---> statement-breakpoint - --- Drop old unique index on candles.time -DROP INDEX IF EXISTS `candles_time_unique`; ---> statement-breakpoint - --- Add chart_id column to candles (nullable first for backfill) -ALTER TABLE `candles` ADD `chart_id` integer REFERENCES charts(id); ---> statement-breakpoint - --- Backfill existing candles with the default chart id -UPDATE `candles` SET `chart_id` = (SELECT `id` FROM `charts` WHERE `name` = 'Imported Data') WHERE `chart_id` IS NULL; ---> statement-breakpoint - --- Add chart_id column to annotations (nullable first for backfill) -ALTER TABLE `annotations` ADD `chart_id` integer REFERENCES charts(id); ---> statement-breakpoint - --- Backfill existing annotations with the default chart id -UPDATE `annotations` SET `chart_id` = (SELECT `id` FROM `charts` WHERE `name` = 'Imported Data') WHERE `chart_id` IS NULL; ---> statement-breakpoint - --- Recreate candles table with NOT NULL constraint and composite unique index -CREATE TABLE `candles_new` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `chart_id` integer NOT NULL REFERENCES charts(id), - `time` integer NOT NULL, - `open` real NOT NULL, - `high` real NOT NULL, - `low` real NOT NULL, - `close` real NOT NULL -); ---> statement-breakpoint -INSERT INTO `candles_new` (`id`, `chart_id`, `time`, `open`, `high`, `low`, `close`) -SELECT `id`, `chart_id`, `time`, `open`, `high`, `low`, `close` FROM `candles`; ---> statement-breakpoint -DROP TABLE `candles`; ---> statement-breakpoint -ALTER TABLE `candles_new` RENAME TO `candles`; ---> statement-breakpoint -CREATE UNIQUE INDEX `candles_chart_time_unique` ON `candles` (`chart_id`, `time`); ---> statement-breakpoint - --- Recreate annotations table with NOT NULL constraint -CREATE TABLE `annotations_new` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `chart_id` integer NOT NULL REFERENCES charts(id), - `timestamp` integer NOT NULL, - `label_type` text NOT NULL, - `geometry` text, - `color` text DEFAULT '#3b82f6', - `created_at` integer NOT NULL -); ---> statement-breakpoint -INSERT INTO `annotations_new` (`id`, `chart_id`, `timestamp`, `label_type`, `geometry`, `color`, `created_at`) -SELECT `id`, `chart_id`, `timestamp`, `label_type`, `geometry`, `color`, `created_at` FROM `annotations`; ---> statement-breakpoint -DROP TABLE `annotations`; ---> statement-breakpoint -ALTER TABLE `annotations_new` RENAME TO `annotations`; diff --git a/drizzle/0003_demonic_captain_flint.sql b/drizzle/0003_demonic_captain_flint.sql deleted file mode 100644 index e62a1a8..0000000 --- a/drizzle/0003_demonic_captain_flint.sql +++ /dev/null @@ -1,27 +0,0 @@ -CREATE TABLE `span_annotations` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `chart_id` integer NOT NULL, - `start_time` integer NOT NULL, - `end_time` integer NOT NULL, - `label` text NOT NULL, - `confidence` integer, - `outcome` text, - `notes` text, - `sub_spans` text, - `color` text DEFAULT '#2196F3' NOT NULL, - `created_at` integer NOT NULL, - FOREIGN KEY (`chart_id`) REFERENCES `charts`(`id`) ON UPDATE no action ON DELETE no action -); ---> statement-breakpoint -CREATE TABLE `span_label_types` ( - `id` integer PRIMARY KEY AUTOINCREMENT NOT NULL, - `name` text NOT NULL, - `display_name` text NOT NULL, - `color` text NOT NULL, - `hotkey` text, - `is_active` integer DEFAULT 1 NOT NULL, - `sort_order` integer DEFAULT 0 NOT NULL, - `created_at` integer NOT NULL -); ---> statement-breakpoint -CREATE UNIQUE INDEX `span_label_types_name_unique` ON `span_label_types` (`name`); \ No newline at end of file diff --git a/drizzle/0004_add_default_span_label_types.sql b/drizzle/0004_add_default_span_label_types.sql deleted file mode 100644 index 487ce29..0000000 --- a/drizzle/0004_add_default_span_label_types.sql +++ /dev/null @@ -1,8 +0,0 @@ --- Insert default span label types (idempotent - skip if already exists) -INSERT OR IGNORE INTO `span_label_types` (`name`, `display_name`, `color`, `hotkey`, `is_active`, `sort_order`, `created_at`) VALUES -('bull_flag', 'Bull Flag', '#4CAF50', '1', 1, 1, strftime('%s', 'now')), -('bear_flag', 'Bear Flag', '#F44336', '2', 1, 2, strftime('%s', 'now')), -('triangle', 'Triangle', '#FF9800', '3', 1, 3, strftime('%s', 'now')), -('wedge', 'Wedge', '#9C27B0', '4', 1, 4, strftime('%s', 'now')), -('channel', 'Channel', '#2196F3', '5', 1, 5, strftime('%s', 'now')), -('consolidation', 'Consolidation', '#607D8B', '6', 1, 6, strftime('%s', 'now')); diff --git a/drizzle/0005_add_source_and_model_prediction_to_span_annotations.sql b/drizzle/0005_add_source_and_model_prediction_to_span_annotations.sql deleted file mode 100644 index fe50439..0000000 --- a/drizzle/0005_add_source_and_model_prediction_to_span_annotations.sql +++ /dev/null @@ -1,11 +0,0 @@ --- Migration: Add source and model_prediction fields to span_annotations --- Supports tracking annotation origin and model feedback loop - --- Add source field (defaults to 'human') --- Possible values: 'human', 'model', 'human_correction' -ALTER TABLE `span_annotations` ADD COLUMN `source` TEXT NOT NULL DEFAULT 'human'; ---> statement-breakpoint --- Add model_prediction field (nullable JSON) --- Stores model prediction metadata when user confirms/corrects a prediction --- Example: {"label": "bull_flag", "confidence": 0.85, "model_version": "xgb_v1"} -ALTER TABLE `span_annotations` ADD COLUMN `model_prediction` TEXT; diff --git a/drizzle/meta/0000_snapshot.json b/drizzle/meta/0000_snapshot.json index 708320d..a2d7835 100644 --- a/drizzle/meta/0000_snapshot.json +++ b/drizzle/meta/0000_snapshot.json @@ -1,131 +1,472 @@ { - "version": "6", - "dialect": "sqlite", - "id": "8c883f15-4c03-4681-bf49-aac5b35c6610", + "id": "2ac019ff-95bf-4bc3-9aa5-456fe6213f25", "prevId": "00000000-0000-0000-0000-000000000000", + "version": "7", + "dialect": "postgresql", "tables": { - "annotations": { - "name": "annotations", + "public.annotation_types": { + "name": "annotation_types", + "schema": "", "columns": { "id": { "name": "id", - "type": "integer", + "type": "serial", "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "category": { + "name": "category", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "icon": { + "name": "icon", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, "notNull": true, - "autoincrement": true + "default": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "annotation_types_name_unique": { + "name": "annotation_types_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.annotations": { + "name": "annotations", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "chart_id": { + "name": "chart_id", + "type": "integer", + "primaryKey": false, + "notNull": true }, "timestamp": { "name": "timestamp", - "type": "integer", + "type": "timestamp", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "label_type": { "name": "label_type", "type": "text", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, "geometry": { "name": "geometry", - "type": "text", + "type": "jsonb", "primaryKey": false, - "notNull": false, - "autoincrement": false + "notNull": false }, "color": { "name": "color", "type": "text", "primaryKey": false, "notNull": false, - "autoincrement": false, "default": "'#3b82f6'" }, "created_at": { "name": "created_at", - "type": "integer", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": { + "annotations_chart_id_charts_id_fk": { + "name": "annotations_chart_id_charts_id_fk", + "tableFrom": "annotations", + "tableTo": "charts", + "columnsFrom": [ + "chart_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.candles": { + "name": "candles", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "chart_id": { + "name": "chart_id", + "type": "integer", + "primaryKey": false, + "notNull": true + }, + "time": { + "name": "time", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "open": { + "name": "open", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "high": { + "name": "high", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "low": { + "name": "low", + "type": "double precision", + "primaryKey": false, + "notNull": true + }, + "close": { + "name": "close", + "type": "double precision", + "primaryKey": false, + "notNull": true + } + }, + "indexes": { + "candles_chart_time_unique": { + "name": "candles_chart_time_unique", + "columns": [ + { + "expression": "chart_id", + "isExpression": false, + "asc": true, + "nulls": "last" + }, + { + "expression": "time", + "isExpression": false, + "asc": true, + "nulls": "last" + } + ], + "isUnique": true, + "concurrently": false, + "method": "btree", + "with": {} + } + }, + "foreignKeys": { + "candles_chart_id_charts_id_fk": { + "name": "candles_chart_id_charts_id_fk", + "tableFrom": "candles", + "tableTo": "charts", + "columnsFrom": [ + "chart_id" + ], + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" + } + }, + "compositePrimaryKeys": {}, + "uniqueConstraints": {}, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.charts": { + "name": "charts", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" } }, "indexes": {}, "foreignKeys": {}, "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} + "uniqueConstraints": { + "charts_name_unique": { + "name": "charts_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false }, - "candles": { - "name": "candles", + "public.span_annotations": { + "name": "span_annotations", + "schema": "", "columns": { "id": { "name": "id", - "type": "integer", + "type": "serial", "primaryKey": true, - "notNull": true, - "autoincrement": true + "notNull": true }, - "time": { - "name": "time", + "chart_id": { + "name": "chart_id", "type": "integer", "primaryKey": false, - "notNull": true, - "autoincrement": false + "notNull": true }, - "open": { - "name": "open", - "type": "real", + "start_time": { + "name": "start_time", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "end_time": { + "name": "end_time", + "type": "timestamp", + "primaryKey": false, + "notNull": true + }, + "label": { + "name": "label", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "confidence": { + "name": "confidence", + "type": "integer", + "primaryKey": false, + "notNull": false + }, + "outcome": { + "name": "outcome", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "notes": { + "name": "notes", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "sub_spans": { + "name": "sub_spans", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "color": { + "name": "color", + "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false + "default": "'#2196F3'" }, - "high": { - "name": "high", - "type": "real", + "source": { + "name": "source", + "type": "text", "primaryKey": false, "notNull": true, - "autoincrement": false + "default": "'human'" }, - "low": { - "name": "low", - "type": "real", + "model_prediction": { + "name": "model_prediction", + "type": "jsonb", + "primaryKey": false, + "notNull": false + }, + "created_at": { + "name": "created_at", + "type": "timestamp", "primaryKey": false, "notNull": true, - "autoincrement": false - }, - "close": { - "name": "close", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false + "default": "now()" } }, - "indexes": { - "candles_time_unique": { - "name": "candles_time_unique", - "columns": [ - "time" + "indexes": {}, + "foreignKeys": { + "span_annotations_chart_id_charts_id_fk": { + "name": "span_annotations_chart_id_charts_id_fk", + "tableFrom": "span_annotations", + "tableTo": "charts", + "columnsFrom": [ + "chart_id" ], - "isUnique": true + "columnsTo": [ + "id" + ], + "onDelete": "no action", + "onUpdate": "no action" } }, - "foreignKeys": {}, "compositePrimaryKeys": {}, "uniqueConstraints": {}, - "checkConstraints": {} + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false + }, + "public.span_label_types": { + "name": "span_label_types", + "schema": "", + "columns": { + "id": { + "name": "id", + "type": "serial", + "primaryKey": true, + "notNull": true + }, + "name": { + "name": "name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "display_name": { + "name": "display_name", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "color": { + "name": "color", + "type": "text", + "primaryKey": false, + "notNull": true + }, + "hotkey": { + "name": "hotkey", + "type": "text", + "primaryKey": false, + "notNull": false + }, + "is_active": { + "name": "is_active", + "type": "boolean", + "primaryKey": false, + "notNull": true, + "default": true + }, + "sort_order": { + "name": "sort_order", + "type": "integer", + "primaryKey": false, + "notNull": true, + "default": 0 + }, + "created_at": { + "name": "created_at", + "type": "timestamp", + "primaryKey": false, + "notNull": true, + "default": "now()" + } + }, + "indexes": {}, + "foreignKeys": {}, + "compositePrimaryKeys": {}, + "uniqueConstraints": { + "span_label_types_name_unique": { + "name": "span_label_types_name_unique", + "nullsNotDistinct": false, + "columns": [ + "name" + ] + } + }, + "policies": {}, + "checkConstraints": {}, + "isRLSEnabled": false } }, - "views": {}, "enums": {}, + "schemas": {}, + "sequences": {}, + "roles": {}, + "policies": {}, + "views": {}, "_meta": { + "columns": {}, "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} + "tables": {} } } \ No newline at end of file diff --git a/drizzle/meta/0001_snapshot.json b/drizzle/meta/0001_snapshot.json deleted file mode 100644 index c042ce0..0000000 --- a/drizzle/meta/0001_snapshot.json +++ /dev/null @@ -1,206 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "111e1b91-6d7b-45e4-aeb9-9762725d6905", - "prevId": "9a43200c-01b1-41fc-ac10-8071afa36f6f", - "tables": { - "annotation_types": { - "name": "annotation_types", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "category": { - "name": "category", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "icon": { - "name": "icon", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "is_active": { - "name": "is_active", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "annotation_types_name_unique": { - "name": "annotation_types_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "annotations": { - "name": "annotations", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "timestamp": { - "name": "timestamp", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "label_type": { - "name": "label_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "geometry": { - "name": "geometry", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "'#3b82f6'" - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "candles": { - "name": "candles", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "time": { - "name": "time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "open": { - "name": "open", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "high": { - "name": "high", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "low": { - "name": "low", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "close": { - "name": "close", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "candles_time_unique": { - "name": "candles_time_unique", - "columns": [ - "time" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/drizzle/meta/0002_snapshot.json b/drizzle/meta/0002_snapshot.json deleted file mode 100644 index 840cb54..0000000 --- a/drizzle/meta/0002_snapshot.json +++ /dev/null @@ -1,288 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "8eb771d5-6d44-473e-8bce-144d195ae2b5", - "prevId": "111e1b91-6d7b-45e4-aeb9-9762725d6905", - "tables": { - "annotation_types": { - "name": "annotation_types", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "category": { - "name": "category", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "icon": { - "name": "icon", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "is_active": { - "name": "is_active", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "annotation_types_name_unique": { - "name": "annotation_types_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "annotations": { - "name": "annotations", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "timestamp": { - "name": "timestamp", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "label_type": { - "name": "label_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "geometry": { - "name": "geometry", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "'#3b82f6'" - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "annotations_chart_id_charts_id_fk": { - "name": "annotations_chart_id_charts_id_fk", - "tableFrom": "annotations", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "candles": { - "name": "candles", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "time": { - "name": "time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "open": { - "name": "open", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "high": { - "name": "high", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "low": { - "name": "low", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "close": { - "name": "close", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "candles_chart_time_unique": { - "name": "candles_chart_time_unique", - "columns": [ - "chart_id", - "time" - ], - "isUnique": true - } - }, - "foreignKeys": { - "candles_chart_id_charts_id_fk": { - "name": "candles_chart_id_charts_id_fk", - "tableFrom": "candles", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "charts": { - "name": "charts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "charts_name_unique": { - "name": "charts_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/drizzle/meta/0003_snapshot.json b/drizzle/meta/0003_snapshot.json deleted file mode 100644 index fffe38a..0000000 --- a/drizzle/meta/0003_snapshot.json +++ /dev/null @@ -1,466 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "17f9ee96-d37f-4c25-97d8-45f8f1a4c868", - "prevId": "8eb771d5-6d44-473e-8bce-144d195ae2b5", - "tables": { - "annotation_types": { - "name": "annotation_types", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "category": { - "name": "category", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "icon": { - "name": "icon", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "is_active": { - "name": "is_active", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "annotation_types_name_unique": { - "name": "annotation_types_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "annotations": { - "name": "annotations", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "timestamp": { - "name": "timestamp", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "label_type": { - "name": "label_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "geometry": { - "name": "geometry", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "'#3b82f6'" - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "annotations_chart_id_charts_id_fk": { - "name": "annotations_chart_id_charts_id_fk", - "tableFrom": "annotations", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "candles": { - "name": "candles", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "time": { - "name": "time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "open": { - "name": "open", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "high": { - "name": "high", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "low": { - "name": "low", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "close": { - "name": "close", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "candles_chart_time_unique": { - "name": "candles_chart_time_unique", - "columns": [ - "chart_id", - "time" - ], - "isUnique": true - } - }, - "foreignKeys": { - "candles_chart_id_charts_id_fk": { - "name": "candles_chart_id_charts_id_fk", - "tableFrom": "candles", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "charts": { - "name": "charts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "charts_name_unique": { - "name": "charts_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "span_annotations": { - "name": "span_annotations", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "start_time": { - "name": "start_time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "end_time": { - "name": "end_time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "label": { - "name": "label", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "confidence": { - "name": "confidence", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "outcome": { - "name": "outcome", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "notes": { - "name": "notes", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "sub_spans": { - "name": "sub_spans", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'#2196F3'" - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "span_annotations_chart_id_charts_id_fk": { - "name": "span_annotations_chart_id_charts_id_fk", - "tableFrom": "span_annotations", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "span_label_types": { - "name": "span_label_types", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "hotkey": { - "name": "hotkey", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "is_active": { - "name": "is_active", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "sort_order": { - "name": "sort_order", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "span_label_types_name_unique": { - "name": "span_label_types_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} \ No newline at end of file diff --git a/drizzle/meta/0004_snapshot.json b/drizzle/meta/0004_snapshot.json deleted file mode 100644 index 2acf754..0000000 --- a/drizzle/meta/0004_snapshot.json +++ /dev/null @@ -1,466 +0,0 @@ -{ - "version": "6", - "dialect": "sqlite", - "id": "a8c3f2d1-e5b7-4a9f-b2d3-6c8e9f1a3b5c", - "prevId": "17f9ee96-d37f-4c25-97d8-45f8f1a4c868", - "tables": { - "annotation_types": { - "name": "annotation_types", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "category": { - "name": "category", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "icon": { - "name": "icon", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "is_active": { - "name": "is_active", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "annotation_types_name_unique": { - "name": "annotation_types_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "annotations": { - "name": "annotations", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "timestamp": { - "name": "timestamp", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "label_type": { - "name": "label_type", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "geometry": { - "name": "geometry", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false, - "default": "'#3b82f6'" - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "annotations_chart_id_charts_id_fk": { - "name": "annotations_chart_id_charts_id_fk", - "tableFrom": "annotations", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "candles": { - "name": "candles", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "time": { - "name": "time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "open": { - "name": "open", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "high": { - "name": "high", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "low": { - "name": "low", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "close": { - "name": "close", - "type": "real", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "candles_chart_time_unique": { - "name": "candles_chart_time_unique", - "columns": [ - "chart_id", - "time" - ], - "isUnique": true - } - }, - "foreignKeys": { - "candles_chart_id_charts_id_fk": { - "name": "candles_chart_id_charts_id_fk", - "tableFrom": "candles", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "charts": { - "name": "charts", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "charts_name_unique": { - "name": "charts_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "span_annotations": { - "name": "span_annotations", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "chart_id": { - "name": "chart_id", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "start_time": { - "name": "start_time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "end_time": { - "name": "end_time", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "label": { - "name": "label", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "confidence": { - "name": "confidence", - "type": "integer", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "outcome": { - "name": "outcome", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "notes": { - "name": "notes", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "sub_spans": { - "name": "sub_spans", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": "'#2196F3'" - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": {}, - "foreignKeys": { - "span_annotations_chart_id_charts_id_fk": { - "name": "span_annotations_chart_id_charts_id_fk", - "tableFrom": "span_annotations", - "tableTo": "charts", - "columnsFrom": [ - "chart_id" - ], - "columnsTo": [ - "id" - ], - "onDelete": "no action", - "onUpdate": "no action" - } - }, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - }, - "span_label_types": { - "name": "span_label_types", - "columns": { - "id": { - "name": "id", - "type": "integer", - "primaryKey": true, - "notNull": true, - "autoincrement": true - }, - "name": { - "name": "name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "display_name": { - "name": "display_name", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "color": { - "name": "color", - "type": "text", - "primaryKey": false, - "notNull": true, - "autoincrement": false - }, - "hotkey": { - "name": "hotkey", - "type": "text", - "primaryKey": false, - "notNull": false, - "autoincrement": false - }, - "is_active": { - "name": "is_active", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 1 - }, - "sort_order": { - "name": "sort_order", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false, - "default": 0 - }, - "created_at": { - "name": "created_at", - "type": "integer", - "primaryKey": false, - "notNull": true, - "autoincrement": false - } - }, - "indexes": { - "span_label_types_name_unique": { - "name": "span_label_types_name_unique", - "columns": [ - "name" - ], - "isUnique": true - } - }, - "foreignKeys": {}, - "compositePrimaryKeys": {}, - "uniqueConstraints": {}, - "checkConstraints": {} - } - }, - "views": {}, - "enums": {}, - "_meta": { - "schemas": {}, - "tables": {}, - "columns": {} - }, - "internal": { - "indexes": {} - } -} diff --git a/drizzle/meta/_journal.json b/drizzle/meta/_journal.json index 7687ef6..c3cbf8d 100644 --- a/drizzle/meta/_journal.json +++ b/drizzle/meta/_journal.json @@ -1,47 +1,12 @@ { "version": "7", - "dialect": "sqlite", + "dialect": "postgresql", "entries": [ { "idx": 0, - "version": "6", - "when": 1770907611962, - "tag": "0000_goofy_captain_midlands", - "breakpoints": true - }, - { - "idx": 1, - "version": "6", - "when": 1770915891699, - "tag": "0001_sticky_shinko_yamashiro", - "breakpoints": true - }, - { - "idx": 2, - "version": "6", - "when": 1770937855462, - "tag": "0002_careful_synch", - "breakpoints": true - }, - { - "idx": 3, - "version": "6", - "when": 1771044740273, - "tag": "0003_demonic_captain_flint", - "breakpoints": true - }, - { - "idx": 4, - "version": "6", - "when": 1771055000000, - "tag": "0004_add_default_span_label_types", - "breakpoints": true - }, - { - "idx": 5, - "version": "6", - "when": 1771156000000, - "tag": "0005_add_source_and_model_prediction_to_span_annotations", + "version": "7", + "when": 1771332001387, + "tag": "0000_nifty_gauntlet", "breakpoints": true } ] diff --git a/openspec/changes/ml-db-consolidation/.openspec.yaml b/openspec/changes/ml-db-consolidation/.openspec.yaml new file mode 100644 index 0000000..c8d3976 --- /dev/null +++ b/openspec/changes/ml-db-consolidation/.openspec.yaml @@ -0,0 +1,2 @@ +schema: spec-driven +created: 2026-02-17 diff --git a/openspec/changes/ml-db-consolidation/design.md b/openspec/changes/ml-db-consolidation/design.md new file mode 100644 index 0000000..204891f --- /dev/null +++ b/openspec/changes/ml-db-consolidation/design.md @@ -0,0 +1,110 @@ +## Context + +The candle annotator runs two databases: + +1. **SQLite** (`data/candles.db`) — serves the Next.js frontend via Drizzle ORM (`better-sqlite3` driver). Contains 6 tables: charts, candles, annotations, annotation_types, span_annotations, span_label_types. +2. **PostgreSQL** (`postgres:5432/ml_db`) — serves the Python ML service via SQLAlchemy. Contains 1 table: training_runs. + +The ML service cannot directly query annotation/candle data. Data flows through CSV/JSON file exports. PostgreSQL already runs in Docker for the ML service, so consolidating means adding frontend tables there — not introducing a new service. + +## Goals / Non-Goals + +**Goals:** +- Single PostgreSQL instance for all application data +- Drizzle ORM continues to manage frontend schema (just switches dialect) +- ML service gains direct read access to candle/annotation tables +- Simplified Docker setup (one fewer volume, one database to back up) +- One-time data migration path from SQLite to PostgreSQL + +**Non-Goals:** +- Changing the ML service ORM (SQLAlchemy stays) +- Merging Drizzle and SQLAlchemy migration systems (each manages its own tables) +- Changing API route logic or query patterns beyond what's needed for the dialect switch +- Multi-tenant or schema separation (all tables go in the `public` schema) +- Migrating away from Drizzle ORM + +## Decisions + +### 1. Drizzle PostgreSQL driver: `drizzle-orm/node-postgres` with `pg` + +**Choice**: Use `pg` (node-postgres) as the driver. + +**Why**: `pg` is the most mature PostgreSQL driver for Node.js. Drizzle supports it natively via `drizzle-orm/node-postgres`. The `postgres` (postgres.js) driver is also an option but `pg` has broader ecosystem support and is easier to debug. + +**Alternative considered**: `postgres` (postgres.js) — lighter, promise-native, but less battle-tested with Drizzle migrations. + +### 2. Shared database, single `public` schema + +**Choice**: All tables (frontend + ML) live in the same database (`ml_db`) and the default `public` schema. + +**Why**: The table sets don't overlap (frontend has charts/candles/annotations, ML has training_runs). Separate schemas add complexity with no benefit for 7 total tables. The ML service already connects to `ml_db`. + +**Alternative considered**: Separate PostgreSQL schemas (`app` and `ml`) — cleaner isolation but adds schema-prefix complexity to queries and cross-schema references. Not worth it at this scale. + +### 3. Rename database from `ml_db` to `candle_annotator` + +**Choice**: Rename the PostgreSQL database to `candle_annotator` since it now serves the whole application, not just ML. + +**Why**: `ml_db` is misleading when the database holds frontend data too. Renaming during consolidation is the natural time to do it. + +**Alternative considered**: Keep `ml_db` — avoids a rename step but creates lasting confusion. + +### 4. Fresh Drizzle migrations (drop SQLite migrations) + +**Choice**: Delete all existing SQLite migrations in `drizzle/`, rewrite the schema file with `pgTable` equivalents, and run `drizzle-kit generate` to produce a fresh initial PostgreSQL migration. + +**Why**: SQLite migrations are dialect-specific (e.g., `integer` for booleans, no native timestamps). Converting them one-by-one is fragile. A clean start from the PostgreSQL schema is simpler and produces idiomatic SQL. + +**Alternative considered**: Manually converting each SQLite migration to PostgreSQL — error-prone and provides no benefit since there's no production data that needs incremental migration history. + +### 5. Type mappings: SQLite → PostgreSQL + +| SQLite type | PostgreSQL type | Notes | +|---|---|---| +| `integer` (PK, autoIncrement) | `serial` | Auto-incrementing integer | +| `integer` (timestamps) | `timestamp` | Use `defaultNow()` where applicable | +| `integer` (booleans like `is_active`) | `boolean` | True PostgreSQL booleans | +| `real` | `doublePrecision` | OHLC price data | +| `text` | `text` | No change | +| `text` (JSON strings) | `jsonb` | For `geometry`, `sub_spans`, `model_prediction` | + +### 6. Connection management for Next.js + +**Choice**: Use a connection pool via `pg.Pool` with `max: 10` connections. Connection string from `DATABASE_URL` env var. + +**Why**: SQLite was single-file, no pooling needed. PostgreSQL requires connection pooling for concurrent API requests. 10 connections is reasonable for the frontend workload. + +### 7. ML service direct access to frontend tables + +**Choice**: The ML service reads frontend tables (candles, annotations, span_annotations) directly via SQLAlchemy using its existing connection. No new SQLAlchemy models needed — raw SQL queries or lightweight table reflections are sufficient for read-only access. + +**Why**: The ML service only needs to read training data. Adding full SQLAlchemy models for tables owned by Drizzle creates a dual-ownership problem. Raw queries or `Table` reflections keep it simple. + +## Risks / Trade-offs + +**[Schema drift between Drizzle and SQLAlchemy]** → Both ORMs manage tables in the same database. Drizzle owns frontend tables, SQLAlchemy owns ML tables. Neither should modify the other's tables. This is enforced by convention, not tooling. + +**[Connection pool exhaustion]** → Adding the frontend's database traffic to the same PostgreSQL instance increases load. Mitigation: PostgreSQL 16 handles far more concurrent connections than SQLite. The `pg.Pool` max of 10 plus SQLAlchemy's pool of 5 is well within PostgreSQL's default `max_connections` of 100. + +**[Data loss during migration]** → SQLite data must be migrated before switching. Mitigation: Write a migration script that exports SQLite data and imports to PostgreSQL. Run before deploying the new code. Keep the SQLite file as backup. + +**[Drizzle push/generate differences]** → PostgreSQL dialect may generate slightly different migration SQL than expected. Mitigation: Review generated migrations before applying. Use `drizzle-kit push` for development, `drizzle-kit generate` + `drizzle-kit migrate` for production. + +**[Boolean conversion]** → SQLite uses `0/1` for booleans, PostgreSQL uses `true/false`. Mitigation: The migration script handles conversion. Drizzle's `boolean()` type handles this transparently at the ORM level going forward. + +## Migration Plan + +1. **Update schema and dependencies** — Rewrite Drizzle schema for PostgreSQL, swap npm packages +2. **Generate fresh migrations** — `drizzle-kit generate` from the new PostgreSQL schema +3. **Update docker-compose.yml** — Rename database, add frontend dependency on postgres, remove `candle-data` volume +4. **Update environment variables** — `DATABASE_URL` for the frontend service +5. **Write data migration script** — `scripts/migrate-sqlite-to-postgres.ts` that reads SQLite and inserts into PostgreSQL with type conversions +6. **Update db/index.ts** — Switch from `better-sqlite3` to `pg` pool, update migration runner +7. **Test locally** — Run migrations, migrate data, verify API routes work +8. **Deploy** — Stop current services, run PostgreSQL migrations, run data migration, deploy new code +9. **Rollback** — If issues arise, revert docker-compose and code, restore SQLite volume. The SQLite file is kept as backup for 1 week post-migration. + +## Open Questions + +- Should the ML service user (`ml_user`) have write access to frontend tables, or should we create a separate read-only role? (Recommendation: keep `ml_user` with full access for simplicity, revisit if the team grows.) +- Do we need to preserve SQLite migration history in git for reference, or delete the `drizzle/` folder contents entirely? (Recommendation: delete and start fresh.) diff --git a/openspec/changes/ml-db-consolidation/proposal.md b/openspec/changes/ml-db-consolidation/proposal.md new file mode 100644 index 0000000..869ea89 --- /dev/null +++ b/openspec/changes/ml-db-consolidation/proposal.md @@ -0,0 +1,35 @@ +## Why + +The project currently runs two separate database servers: SQLite (via Drizzle ORM) for the Next.js frontend and PostgreSQL for the ML service. This creates unnecessary operational complexity — two different ORMs, two migration systems, two backup strategies, and no ability for the ML service to directly query annotation/candle data. Consolidating to PostgreSQL as the single database simplifies deployment, enables direct cross-service data access, and reduces the infrastructure footprint. + +## What Changes + +- **BREAKING**: Replace SQLite/better-sqlite3/Drizzle with PostgreSQL/Drizzle (pg driver) for the Next.js frontend +- Remove the `candle-data` Docker volume (SQLite file storage) and `DATABASE_PATH` env var +- Migrate all frontend tables (charts, candles, annotations, annotation_types, span_annotations, span_label_types) into the existing PostgreSQL instance +- Update Drizzle schema and config to target PostgreSQL instead of SQLite +- Regenerate Drizzle migrations for PostgreSQL dialect (column types change: `integer` → `serial`, `real` → `double precision`, timestamps as proper `timestamp` types, etc.) +- Update the ML service to share the same PostgreSQL database (or a separate schema within it) so it can directly query candle/annotation data instead of relying on CSV/JSON exports +- Update docker-compose.yml to remove SQLite volume dependency and point the frontend at PostgreSQL +- Update environment variables: frontend gets `DATABASE_URL` pointing to PostgreSQL + +## Capabilities + +### New Capabilities +- `postgres-data-layer`: Unified PostgreSQL data access layer for the Next.js frontend, replacing the SQLite/better-sqlite3 setup with Drizzle's PostgreSQL driver + +### Modified Capabilities +- `docker-deployment`: Container configuration changes — remove SQLite volume, add PostgreSQL dependency for the frontend service, update environment variables +- `ml-training`: ML service can now query annotations and candle data directly from PostgreSQL instead of requiring CSV/JSON file exports + +## Impact + +- **Database schema**: All 6 frontend tables move to PostgreSQL with type adaptations (SQLite integers → PostgreSQL serial/integer/timestamp) +- **ORM layer**: `src/lib/db/index.ts` switches from `better-sqlite3` to `postgres` driver; schema types in `src/lib/db/schema.ts` change to PostgreSQL equivalents +- **Dependencies**: Remove `better-sqlite3`, add `postgres` (or `pg`) npm package for Drizzle's PostgreSQL adapter +- **Migrations**: Existing SQLite migrations become obsolete; new PostgreSQL migrations needed +- **Docker**: `candle-annotator` service gains `depends_on: postgres`, loses `candle-data` volume mount +- **Environment**: `.env` and `.env.example` updated with PostgreSQL connection string for frontend +- **ML service**: `services/ml/app/db.py` gains access to frontend tables (candles, annotations) for direct querying +- **Data migration**: Existing SQLite data needs a one-time migration script to PostgreSQL +- **API routes**: All Next.js API routes using `db` from `src/lib/db` continue working (Drizzle abstracts the driver change), but queries using SQLite-specific syntax may need adjustment diff --git a/openspec/changes/ml-db-consolidation/specs/docker-deployment/spec.md b/openspec/changes/ml-db-consolidation/specs/docker-deployment/spec.md new file mode 100644 index 0000000..1092578 --- /dev/null +++ b/openspec/changes/ml-db-consolidation/specs/docker-deployment/spec.md @@ -0,0 +1,81 @@ +## MODIFIED Requirements + +### Requirement: Docker Compose configuration +The project SHALL include docker-compose.yml for simplified deployment orchestration. + +#### Scenario: Service definition +- **WHEN** docker-compose.yml is parsed +- **THEN** defines service named 'candle-annotator' using Dockerfile from current directory + +#### Scenario: Port mapping +- **WHEN** docker-compose up runs +- **THEN** maps host port 3000 to container port 3000 + +#### Scenario: Volume mounting for ML data +- **WHEN** docker-compose up runs +- **THEN** mounts named volume 'ml-data' to /app/ml-data in the candle-annotator container + +#### Scenario: Frontend depends on PostgreSQL +- **WHEN** docker-compose up runs +- **THEN** the candle-annotator service starts only after the postgres service is healthy (`depends_on: postgres: condition: service_healthy`) + +#### Scenario: Frontend DATABASE_URL +- **WHEN** the candle-annotator service starts +- **THEN** the `DATABASE_URL` environment variable is set to `postgresql://ml_user:ml_password@postgres:5432/candle_annotator` + +#### Scenario: Restart policy +- **WHEN** container crashes or stops +- **THEN** docker-compose automatically restarts container unless explicitly stopped (restart: unless-stopped) + +#### Scenario: No SQLite volume +- **WHEN** docker-compose.yml is parsed +- **THEN** there is no `candle-data` volume defined or mounted + +### Requirement: Environment variable configuration +The project SHALL use environment variables for runtime configuration. + +#### Scenario: .env.example file +- **WHEN** repository is cloned +- **THEN** includes .env.example file documenting all configurable environment variables with example values + +#### Scenario: DATABASE_URL configuration +- **WHEN** `DATABASE_URL` environment variable is set +- **THEN** the Next.js application connects to the PostgreSQL database at the specified URL + +#### Scenario: No DATABASE_PATH variable +- **WHEN** environment variables are inspected +- **THEN** there is no `DATABASE_PATH` variable (SQLite path is removed) + +#### Scenario: PORT configuration +- **WHEN** PORT environment variable is set +- **THEN** Next.js server listens on specified port (default: 3000) + +#### Scenario: NODE_ENV configuration +- **WHEN** NODE_ENV environment variable is set to 'production' +- **THEN** Next.js runs in production mode with optimizations enabled + +### Requirement: Database persistence +The deployment SHALL ensure PostgreSQL data persists across container restarts. + +#### Scenario: PostgreSQL volume +- **WHEN** docker-compose up runs +- **THEN** the `postgres-data` named volume is mounted to `/var/lib/postgresql/data` in the postgres container + +#### Scenario: Container restart preserves data +- **WHEN** the postgres container is stopped and restarted +- **THEN** all database tables and data remain intact + +#### Scenario: PostgreSQL database name +- **WHEN** the postgres service starts +- **THEN** the `POSTGRES_DB` environment variable is set to `candle_annotator` + +### Requirement: Health check endpoint +The API SHALL provide a health check endpoint for container orchestration. + +#### Scenario: Health check endpoint responds +- **WHEN** GET request sent to `/api/health` +- **THEN** system returns 200 status with JSON `{ status: 'ok', timestamp: }` + +#### Scenario: Database connection check +- **WHEN** GET request sent to `/api/health?check=db` +- **THEN** system attempts a PostgreSQL query and returns 200 if successful, 503 if database unavailable diff --git a/openspec/changes/ml-db-consolidation/specs/ml-training/spec.md b/openspec/changes/ml-db-consolidation/specs/ml-training/spec.md new file mode 100644 index 0000000..ef954e6 --- /dev/null +++ b/openspec/changes/ml-db-consolidation/specs/ml-training/spec.md @@ -0,0 +1,37 @@ +## MODIFIED Requirements + +### Requirement: PostgreSQL training metadata storage +The system SHALL store training run metadata in the PostgreSQL database. Each training run record SHALL include: run_id (MLflow run ID), model_type, experiment_name, pipeline_config_hash, dataset_version, metrics summary (JSON), status, and timestamps (created_at, completed_at). + +#### Scenario: Store training run record +- **WHEN** a training run completes successfully +- **THEN** the system inserts a record into the PostgreSQL `training_runs` table with the run metadata + +#### Scenario: Query training history +- **WHEN** the system queries training runs +- **THEN** it returns records from PostgreSQL ordered by created_at descending + +#### Scenario: Database name updated +- **WHEN** the ML service connects to PostgreSQL +- **THEN** it connects to the `candle_annotator` database (not `ml_db`) + +## ADDED Requirements + +### Requirement: Direct annotation data access +The ML service SHALL read candle and annotation data directly from PostgreSQL instead of requiring CSV/JSON file exports. The ML service SHALL query the `candles`, `annotations`, `span_annotations`, and `charts` tables for training data. + +#### Scenario: Query candle data for training +- **WHEN** the ML training pipeline needs OHLC data for a chart +- **THEN** it queries the `candles` table in PostgreSQL filtered by `chart_id`, ordered by `time` + +#### Scenario: Query span annotations for labels +- **WHEN** the ML training pipeline needs labeled spans for training +- **THEN** it queries the `span_annotations` table in PostgreSQL filtered by `chart_id` and optionally by `source` + +#### Scenario: No CSV/JSON export required +- **WHEN** the ML training pipeline starts +- **THEN** it does not require pre-exported CSV or JSON files — all data is read from PostgreSQL + +#### Scenario: Shared database connection +- **WHEN** the ML service reads candle/annotation data +- **THEN** it uses the same PostgreSQL connection (same database, same credentials) as for `training_runs` diff --git a/openspec/changes/ml-db-consolidation/specs/postgres-data-layer/spec.md b/openspec/changes/ml-db-consolidation/specs/postgres-data-layer/spec.md new file mode 100644 index 0000000..3e2294f --- /dev/null +++ b/openspec/changes/ml-db-consolidation/specs/postgres-data-layer/spec.md @@ -0,0 +1,80 @@ +## ADDED Requirements + +### Requirement: PostgreSQL connection via Drizzle ORM +The Next.js application SHALL connect to PostgreSQL using Drizzle ORM with the `node-postgres` (`pg`) driver. The connection SHALL use a pool with a configurable maximum number of connections (default: 10). The connection string SHALL be read from the `DATABASE_URL` environment variable. + +#### Scenario: Successful connection +- **WHEN** the application starts with a valid `DATABASE_URL` pointing to a running PostgreSQL instance +- **THEN** Drizzle ORM establishes a connection pool and the `db` export is ready for queries + +#### Scenario: Missing DATABASE_URL +- **WHEN** the `DATABASE_URL` environment variable is not set +- **THEN** the application SHALL fail to start with an error message indicating the missing variable + +#### Scenario: Database unreachable +- **WHEN** the PostgreSQL instance is not reachable at the configured URL +- **THEN** the application SHALL fail to start with a connection error + +### Requirement: PostgreSQL schema definitions +The Drizzle schema SHALL define all frontend tables using `pgTable` from `drizzle-orm/pg-core`. The following tables SHALL be defined: `charts`, `candles`, `annotation_types`, `annotations`, `span_label_types`, `span_annotations`. + +#### Scenario: Charts table schema +- **WHEN** the schema is loaded +- **THEN** the `charts` table has columns: `id` (serial, primary key), `name` (text, unique, not null), `created_at` (timestamp, not null, default now) + +#### Scenario: Candles table schema +- **WHEN** the schema is loaded +- **THEN** the `candles` table has columns: `id` (serial, primary key), `chart_id` (integer, foreign key to charts.id, not null), `time` (timestamp, not null), `open` (double precision, not null), `high` (double precision, not null), `low` (double precision, not null), `close` (double precision, not null), with a unique index on `(chart_id, time)` + +#### Scenario: Annotation types table schema +- **WHEN** the schema is loaded +- **THEN** the `annotation_types` table has columns: `id` (serial, primary key), `name` (text, unique, not null), `display_name` (text, not null), `color` (text, not null), `category` (text, not null), `icon` (text, nullable), `is_active` (boolean, not null, default true), `created_at` (timestamp, not null, default now) + +#### Scenario: Annotations table schema +- **WHEN** the schema is loaded +- **THEN** the `annotations` table has columns: `id` (serial, primary key), `chart_id` (integer, foreign key to charts.id, not null), `timestamp` (timestamp, not null), `label_type` (text, not null), `geometry` (jsonb, nullable), `color` (text, default '#3b82f6'), `created_at` (timestamp, not null, default now) + +#### Scenario: Span label types table schema +- **WHEN** the schema is loaded +- **THEN** the `span_label_types` table has columns: `id` (serial, primary key), `name` (text, unique, not null), `display_name` (text, not null), `color` (text, not null), `hotkey` (text, nullable), `is_active` (boolean, not null, default true), `sort_order` (integer, not null, default 0), `created_at` (timestamp, not null, default now) + +#### Scenario: Span annotations table schema +- **WHEN** the schema is loaded +- **THEN** the `span_annotations` table has columns: `id` (serial, primary key), `chart_id` (integer, foreign key to charts.id, not null), `start_time` (timestamp, not null), `end_time` (timestamp, not null), `label` (text, not null), `confidence` (integer, nullable), `outcome` (text, nullable), `notes` (text, nullable), `sub_spans` (jsonb, nullable), `color` (text, not null, default '#2196F3'), `source` (text, not null, default 'human'), `model_prediction` (jsonb, nullable), `created_at` (timestamp, not null, default now) + +### Requirement: PostgreSQL migrations via Drizzle Kit +The project SHALL use Drizzle Kit to generate and apply PostgreSQL migrations. The `drizzle.config.ts` SHALL target the `postgresql` dialect. Existing SQLite migrations SHALL be removed. + +#### Scenario: Generate migrations +- **WHEN** `drizzle-kit generate` is executed +- **THEN** a new SQL migration file is created in the `drizzle/` directory with PostgreSQL-dialect DDL + +#### Scenario: Apply migrations at startup +- **WHEN** the application starts (not during build phase) +- **THEN** Drizzle runs pending migrations against the PostgreSQL database + +#### Scenario: Skip migrations during build +- **WHEN** `NEXT_PHASE` is `phase-production-build` or `phase-development-build` +- **THEN** migration execution is skipped + +### Requirement: npm dependency changes +The project SHALL remove `better-sqlite3` and `@types/better-sqlite3` from dependencies and add `pg` and `@types/pg`. + +#### Scenario: Dependencies updated +- **WHEN** `package.json` is inspected +- **THEN** `better-sqlite3` and `@types/better-sqlite3` are absent, and `pg` and `@types/pg` are present in dependencies + +### Requirement: Data migration from SQLite to PostgreSQL +The project SHALL include a one-time migration script at `scripts/migrate-sqlite-to-postgres.ts` that reads all data from the SQLite database and inserts it into PostgreSQL with appropriate type conversions. + +#### Scenario: Migrate all tables +- **WHEN** the migration script is executed with both databases accessible +- **THEN** all rows from charts, candles, annotation_types, annotations, span_label_types, and span_annotations are transferred to PostgreSQL + +#### Scenario: Type conversions applied +- **WHEN** data is migrated +- **THEN** SQLite integer timestamps are converted to PostgreSQL timestamps, integer booleans (0/1) are converted to PostgreSQL booleans, and text JSON fields are inserted as jsonb + +#### Scenario: Idempotent execution +- **WHEN** the migration script is run a second time on an already-migrated database +- **THEN** the script either skips existing data or clears and re-inserts (with a flag), without creating duplicates diff --git a/openspec/changes/ml-db-consolidation/tasks.md b/openspec/changes/ml-db-consolidation/tasks.md new file mode 100644 index 0000000..cb64d20 --- /dev/null +++ b/openspec/changes/ml-db-consolidation/tasks.md @@ -0,0 +1,53 @@ +## 1. Dependencies and Configuration + +- [x] 1.1 Remove `better-sqlite3` and `@types/better-sqlite3` from package.json +- [x] 1.2 Add `pg` and `@types/pg` to package.json dependencies +- [x] 1.3 Run `npm install` to update node_modules and lockfile +- [x] 1.4 Update `drizzle.config.ts` to target `postgresql` dialect with `DATABASE_URL` env var +- [x] 1.5 Update `.env.example` — replace `DATABASE_PATH` with `DATABASE_URL=postgresql://ml_user:ml_password@postgres:5432/candle_annotator` + +## 2. Drizzle Schema Migration (SQLite → PostgreSQL) + +- [x] 2.1 Rewrite `src/lib/db/schema.ts` — replace all `sqliteTable` with `pgTable`, apply type mappings (integer→serial, integer→timestamp, integer→boolean, real→doublePrecision, text JSON→jsonb) +- [x] 2.2 Delete all existing SQLite migration files in `drizzle/` directory +- [x] 2.3 Run `drizzle-kit generate` to produce fresh PostgreSQL migration SQL +- [x] 2.4 Review generated migration SQL for correctness + +## 3. Database Connection Layer + +- [x] 3.1 Rewrite `src/lib/db/index.ts` — replace `better-sqlite3` driver with `pg.Pool` (max: 10), read `DATABASE_URL` from env, fail if missing +- [x] 3.2 Update migration runner to use PostgreSQL-compatible execution (skip during build phase via `NEXT_PHASE` check) +- [x] 3.3 Update all imports if any changed (verify `db` export still works for API routes) + +## 4. API Route Adjustments + +- [x] 4.1 Audit all Next.js API routes using `db` for SQLite-specific syntax (e.g., integer booleans, raw SQL fragments) +- [x] 4.2 Fix any SQLite-specific query patterns to work with PostgreSQL (boolean handling, timestamp handling, jsonb operations) +- [x] 4.3 Update health check endpoint (`/api/health`) to verify PostgreSQL connectivity + +## 5. Docker and Deployment + +- [ ] 5.1 Update `docker-compose.yml` — rename `POSTGRES_DB` to `candle_annotator`, add `DATABASE_URL` env to candle-annotator service, add `depends_on: postgres` with health check condition +- [ ] 5.2 Remove `candle-data` volume from `docker-compose.yml` (SQLite volume) +- [ ] 5.3 Update `Dockerfile` if it references SQLite or `DATABASE_PATH` +- [ ] 5.4 Update ML service database connection — change database name from `ml_db` to `candle_annotator` in environment config + +## 6. ML Service Direct Data Access + +- [ ] 6.1 Add SQLAlchemy table reflections or raw queries in the ML service for reading `candles`, `annotations`, `span_annotations`, `charts` tables +- [ ] 6.2 Update ML training pipeline to query candle/annotation data from PostgreSQL instead of CSV/JSON exports +- [ ] 6.3 Remove or deprecate any CSV/JSON export code paths that are no longer needed + +## 7. Data Migration Script + +- [ ] 7.1 Create `scripts/migrate-sqlite-to-postgres.ts` — read all 6 tables from SQLite, apply type conversions (timestamps, booleans, JSON→jsonb), insert into PostgreSQL +- [ ] 7.2 Make the script idempotent (skip or clear+re-insert with flag) +- [ ] 7.3 Test migration script with existing SQLite data + +## 8. Testing and Verification + +- [ ] 8.1 Run the full application locally with PostgreSQL — verify all API routes work +- [ ] 8.2 Verify ML service can query candle/annotation data from shared database +- [ ] 8.3 Run `docker compose up` and verify all services start correctly with new configuration +- [ ] 8.4 Update `DEPLOYMENT.md` with new deployment steps (PostgreSQL migration, data migration script, rollback procedure) +- [ ] 8.5 Update `README.md` and `CLAUDE_DESCRIPTION.md` with database architecture changes diff --git a/package-lock.json b/package-lock.json index 48986fd..0f54028 100644 --- a/package-lock.json +++ b/package-lock.json @@ -13,13 +13,12 @@ "@radix-ui/react-label": "^2.1.8", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slider": "^1.3.6", - "@types/better-sqlite3": "^7.6.13", "@types/node": "^25.2.3", "@types/papaparse": "^5.5.2", + "@types/pg": "^8.11.10", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "autoprefixer": "^10.4.24", - "better-sqlite3": "^12.6.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "drizzle-orm": "^0.45.1", @@ -30,6 +29,7 @@ "next": "^16.1.6", "next-themes": "^0.4.6", "papaparse": "^5.5.3", + "pg": "^8.13.1", "postcss": "^8.5.6", "react": "^19.2.4", "react-dom": "^19.2.4", @@ -2694,14 +2694,6 @@ "tslib": "^2.4.0" } }, - "node_modules/@types/better-sqlite3": { - "version": "7.6.13", - "license": "MIT", - "peer": true, - "dependencies": { - "@types/node": "*" - } - }, "node_modules/@types/estree": { "version": "1.0.8", "license": "MIT" @@ -2728,6 +2720,18 @@ "@types/node": "*" } }, + "node_modules/@types/pg": { + "version": "8.16.0", + "resolved": "https://registry.npmjs.org/@types/pg/-/pg-8.16.0.tgz", + "integrity": "sha512-RmhMd/wD+CF8Dfo+cVIy3RR5cl8CyfXQ0tGgW6XBL8L4LM/UTEbNXYRbLwU6w+CgrKBNbrQWt4FUtTfaU5jSYQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "@types/node": "*", + "pg-protocol": "*", + "pg-types": "^2.2.0" + } + }, "node_modules/@types/react": { "version": "19.2.14", "license": "MIT", @@ -3544,7 +3548,8 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/baseline-browser-mapping": { "version": "2.9.19", @@ -3553,21 +3558,6 @@ "baseline-browser-mapping": "dist/cli.js" } }, - "node_modules/better-sqlite3": { - "version": "12.6.2", - "resolved": "https://registry.npmjs.org/better-sqlite3/-/better-sqlite3-12.6.2.tgz", - "integrity": "sha512-8VYKM3MjCa9WcaSAI3hzwhmyHVlH8tiGFwf0RlTsZPWJ1I5MkzjiudCo4KC4DxOaL/53A5B1sI/IbldNFDbsKA==", - "hasInstallScript": true, - "license": "MIT", - "peer": true, - "dependencies": { - "bindings": "^1.5.0", - "prebuild-install": "^7.1.1" - }, - "engines": { - "node": "20.x || 22.x || 23.x || 24.x || 25.x" - } - }, "node_modules/binary-extensions": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/binary-extensions/-/binary-extensions-2.3.0.tgz", @@ -3584,6 +3574,7 @@ "node_modules/bindings": { "version": "1.5.0", "license": "MIT", + "optional": true, "dependencies": { "file-uri-to-path": "1.0.0" } @@ -3591,6 +3582,7 @@ "node_modules/bl": { "version": "4.1.0", "license": "MIT", + "optional": true, "dependencies": { "buffer": "^5.5.0", "inherits": "^2.0.4", @@ -3664,6 +3656,7 @@ } ], "license": "MIT", + "optional": true, "dependencies": { "base64-js": "^1.3.1", "ieee754": "^1.1.13" @@ -3804,7 +3797,8 @@ }, "node_modules/chownr": { "version": "1.1.4", - "license": "ISC" + "license": "ISC", + "optional": true }, "node_modules/class-variance-authority": { "version": "0.7.1", @@ -3955,6 +3949,7 @@ "node_modules/decompress-response": { "version": "6.0.0", "license": "MIT", + "optional": true, "dependencies": { "mimic-response": "^3.1.0" }, @@ -3968,6 +3963,7 @@ "node_modules/deep-extend": { "version": "0.6.0", "license": "MIT", + "optional": true, "engines": { "node": ">=4.0.0" } @@ -4009,6 +4005,7 @@ "node_modules/detect-libc": { "version": "2.1.2", "license": "Apache-2.0", + "optional": true, "engines": { "node": ">=8" } @@ -4203,6 +4200,7 @@ "node_modules/end-of-stream": { "version": "1.4.5", "license": "MIT", + "optional": true, "dependencies": { "once": "^1.4.0" } @@ -4795,6 +4793,7 @@ "node_modules/expand-template": { "version": "2.0.3", "license": "(MIT OR WTFPL)", + "optional": true, "engines": { "node": ">=6" } @@ -4858,7 +4857,8 @@ }, "node_modules/file-uri-to-path": { "version": "1.0.0", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/fill-range": { "version": "7.1.1", @@ -4925,7 +4925,8 @@ }, "node_modules/fs-constants": { "version": "1.0.0", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/fsevents": { "version": "2.3.3", @@ -5057,7 +5058,8 @@ }, "node_modules/github-from-package": { "version": "0.0.0", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/glob-parent": { "version": "6.0.2", @@ -5203,7 +5205,8 @@ "url": "https://feross.org/support" } ], - "license": "BSD-3-Clause" + "license": "BSD-3-Clause", + "optional": true }, "node_modules/ignore": { "version": "5.3.2", @@ -5235,11 +5238,13 @@ }, "node_modules/inherits": { "version": "2.0.4", - "license": "ISC" + "license": "ISC", + "optional": true }, "node_modules/ini": { "version": "1.3.8", - "license": "ISC" + "license": "ISC", + "optional": true }, "node_modules/internal-slot": { "version": "1.1.0", @@ -5823,6 +5828,7 @@ "node_modules/mimic-response": { "version": "3.1.0", "license": "MIT", + "optional": true, "engines": { "node": ">=10" }, @@ -5849,7 +5855,8 @@ }, "node_modules/mkdirp-classic": { "version": "0.5.3", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/ms": { "version": "2.1.3", @@ -5885,7 +5892,8 @@ }, "node_modules/napi-build-utils": { "version": "2.0.0", - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/napi-postinstall": { "version": "0.3.4", @@ -5994,6 +6002,7 @@ "node_modules/node-abi": { "version": "3.87.0", "license": "MIT", + "optional": true, "dependencies": { "semver": "^7.3.5" }, @@ -6004,6 +6013,7 @@ "node_modules/node-abi/node_modules/semver": { "version": "7.7.4", "license": "ISC", + "optional": true, "bin": { "semver": "bin/semver.js" }, @@ -6137,6 +6147,7 @@ "node_modules/once": { "version": "1.4.0", "license": "ISC", + "optional": true, "dependencies": { "wrappy": "1" } @@ -6229,6 +6240,96 @@ "version": "1.0.7", "license": "MIT" }, + "node_modules/pg": { + "version": "8.18.0", + "resolved": "https://registry.npmjs.org/pg/-/pg-8.18.0.tgz", + "integrity": "sha512-xqrUDL1b9MbkydY/s+VZ6v+xiMUmOUk7SS9d/1kpyQxoJ6U9AO1oIJyUWVZojbfe5Cc/oluutcgFG4L9RDP1iQ==", + "license": "MIT", + "peer": true, + "dependencies": { + "pg-connection-string": "^2.11.0", + "pg-pool": "^3.11.0", + "pg-protocol": "^1.11.0", + "pg-types": "2.2.0", + "pgpass": "1.0.5" + }, + "engines": { + "node": ">= 16.0.0" + }, + "optionalDependencies": { + "pg-cloudflare": "^1.3.0" + }, + "peerDependencies": { + "pg-native": ">=3.0.1" + }, + "peerDependenciesMeta": { + "pg-native": { + "optional": true + } + } + }, + "node_modules/pg-cloudflare": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/pg-cloudflare/-/pg-cloudflare-1.3.0.tgz", + "integrity": "sha512-6lswVVSztmHiRtD6I8hw4qP/nDm1EJbKMRhf3HCYaqud7frGysPv7FYJ5noZQdhQtN2xJnimfMtvQq21pdbzyQ==", + "license": "MIT", + "optional": true + }, + "node_modules/pg-connection-string": { + "version": "2.11.0", + "resolved": "https://registry.npmjs.org/pg-connection-string/-/pg-connection-string-2.11.0.tgz", + "integrity": "sha512-kecgoJwhOpxYU21rZjULrmrBJ698U2RxXofKVzOn5UDj61BPj/qMb7diYUR1nLScCDbrztQFl1TaQZT0t1EtzQ==", + "license": "MIT" + }, + "node_modules/pg-int8": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/pg-int8/-/pg-int8-1.0.1.tgz", + "integrity": "sha512-WCtabS6t3c8SkpDBUlb1kjOs7l66xsGdKpIPZsg4wR+B3+u9UAum2odSsF9tnvxg80h4ZxLWMy4pRjOsFIqQpw==", + "license": "ISC", + "engines": { + "node": ">=4.0.0" + } + }, + "node_modules/pg-pool": { + "version": "3.11.0", + "resolved": "https://registry.npmjs.org/pg-pool/-/pg-pool-3.11.0.tgz", + "integrity": "sha512-MJYfvHwtGp870aeusDh+hg9apvOe2zmpZJpyt+BMtzUWlVqbhFmMK6bOBXLBUPd7iRtIF9fZplDc7KrPN3PN7w==", + "license": "MIT", + "peerDependencies": { + "pg": ">=8.0" + } + }, + "node_modules/pg-protocol": { + "version": "1.11.0", + "resolved": "https://registry.npmjs.org/pg-protocol/-/pg-protocol-1.11.0.tgz", + "integrity": "sha512-pfsxk2M9M3BuGgDOfuy37VNRRX3jmKgMjcvAcWqNDpZSf4cUmv8HSOl5ViRQFsfARFn0KuUQTgLxVMbNq5NW3g==", + "license": "MIT" + }, + "node_modules/pg-types": { + "version": "2.2.0", + "resolved": "https://registry.npmjs.org/pg-types/-/pg-types-2.2.0.tgz", + "integrity": "sha512-qTAAlrEsl8s4OiEQY69wDvcMIdQN6wdz5ojQiOy6YRMuynxenON0O5oCpJI6lshc6scgAY8qvJ2On/p+CXY0GA==", + "license": "MIT", + "dependencies": { + "pg-int8": "1.0.1", + "postgres-array": "~2.0.0", + "postgres-bytea": "~1.0.0", + "postgres-date": "~1.0.4", + "postgres-interval": "^1.1.0" + }, + "engines": { + "node": ">=4" + } + }, + "node_modules/pgpass": { + "version": "1.0.5", + "resolved": "https://registry.npmjs.org/pgpass/-/pgpass-1.0.5.tgz", + "integrity": "sha512-FdW9r/jQZhSeohs1Z3sI1yxFQNFvMcnmfuj4WBMUTxOrAyLMaTcE1aAMBiTlbMNaXvBCQuVi0R7hd8udDSP7ug==", + "license": "MIT", + "dependencies": { + "split2": "^4.1.0" + } + }, "node_modules/picocolors": { "version": "1.1.1", "license": "ISC" @@ -6428,9 +6529,49 @@ "version": "4.2.0", "license": "MIT" }, + "node_modules/postgres-array": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/postgres-array/-/postgres-array-2.0.0.tgz", + "integrity": "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA==", + "license": "MIT", + "engines": { + "node": ">=4" + } + }, + "node_modules/postgres-bytea": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/postgres-bytea/-/postgres-bytea-1.0.1.tgz", + "integrity": "sha512-5+5HqXnsZPE65IJZSMkZtURARZelel2oXUEO8rH83VS/hxH5vv1uHquPg5wZs8yMAfdv971IU+kcPUczi7NVBQ==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-date": { + "version": "1.0.7", + "resolved": "https://registry.npmjs.org/postgres-date/-/postgres-date-1.0.7.tgz", + "integrity": "sha512-suDmjLVQg78nMK2UZ454hAG+OAW+HQPZ6n++TNDUX+L0+uUlLywnoxJKDou51Zm+zTCjrCl0Nq6J9C5hP9vK/Q==", + "license": "MIT", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/postgres-interval": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/postgres-interval/-/postgres-interval-1.2.0.tgz", + "integrity": "sha512-9ZhXKM/rw350N1ovuWHbGxnGh/SNJ4cnxHiM0rxE4VN41wsg8P8zWn9hv/buK00RP4WvlOyr/RBDiptyxVbkZQ==", + "license": "MIT", + "dependencies": { + "xtend": "^4.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/prebuild-install": { "version": "7.1.3", "license": "MIT", + "optional": true, "dependencies": { "detect-libc": "^2.0.0", "expand-template": "^2.0.3", @@ -6471,6 +6612,7 @@ "node_modules/pump": { "version": "3.0.3", "license": "MIT", + "optional": true, "dependencies": { "end-of-stream": "^1.1.0", "once": "^1.3.1" @@ -6504,6 +6646,7 @@ "node_modules/rc": { "version": "1.2.8", "license": "(BSD-2-Clause OR MIT OR Apache-2.0)", + "optional": true, "dependencies": { "deep-extend": "^0.6.0", "ini": "~1.3.0", @@ -6517,6 +6660,7 @@ "node_modules/rc/node_modules/strip-json-comments": { "version": "2.0.1", "license": "MIT", + "optional": true, "engines": { "node": ">=0.10.0" } @@ -6626,6 +6770,7 @@ "node_modules/readable-stream": { "version": "3.6.2", "license": "MIT", + "optional": true, "dependencies": { "inherits": "^2.0.3", "string_decoder": "^1.1.1", @@ -6780,7 +6925,8 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/safe-push-apply": { "version": "1.0.0", @@ -7013,7 +7159,8 @@ "url": "https://feross.org/support" } ], - "license": "MIT" + "license": "MIT", + "optional": true }, "node_modules/simple-get": { "version": "4.0.1", @@ -7032,6 +7179,7 @@ } ], "license": "MIT", + "optional": true, "dependencies": { "decompress-response": "^6.0.0", "once": "^1.3.1", @@ -7062,6 +7210,15 @@ "source-map": "^0.6.0" } }, + "node_modules/split2": { + "version": "4.2.0", + "resolved": "https://registry.npmjs.org/split2/-/split2-4.2.0.tgz", + "integrity": "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg==", + "license": "ISC", + "engines": { + "node": ">= 10.x" + } + }, "node_modules/stable-hash": { "version": "0.0.5", "license": "MIT" @@ -7080,6 +7237,7 @@ "node_modules/string_decoder": { "version": "1.3.0", "license": "MIT", + "optional": true, "dependencies": { "safe-buffer": "~5.2.0" } @@ -7339,6 +7497,7 @@ "node_modules/tar-fs": { "version": "2.1.4", "license": "MIT", + "optional": true, "dependencies": { "chownr": "^1.1.1", "mkdirp-classic": "^0.5.2", @@ -7349,6 +7508,7 @@ "node_modules/tar-stream": { "version": "2.2.0", "license": "MIT", + "optional": true, "dependencies": { "bl": "^4.0.3", "end-of-stream": "^1.4.1", @@ -7982,6 +8142,7 @@ "node_modules/tunnel-agent": { "version": "0.6.0", "license": "Apache-2.0", + "optional": true, "dependencies": { "safe-buffer": "^5.0.1" }, @@ -8230,6 +8391,7 @@ }, "node_modules/util-deprecate": { "version": "1.0.2", + "devOptional": true, "license": "MIT" }, "node_modules/which": { @@ -8331,7 +8493,17 @@ }, "node_modules/wrappy": { "version": "1.0.2", - "license": "ISC" + "license": "ISC", + "optional": true + }, + "node_modules/xtend": { + "version": "4.0.2", + "resolved": "https://registry.npmjs.org/xtend/-/xtend-4.0.2.tgz", + "integrity": "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ==", + "license": "MIT", + "engines": { + "node": ">=0.4" + } }, "node_modules/yallist": { "version": "3.1.1", diff --git a/package.json b/package.json index 3e6e18d..e527b97 100644 --- a/package.json +++ b/package.json @@ -19,13 +19,12 @@ "@radix-ui/react-label": "^2.1.8", "@radix-ui/react-select": "^2.2.6", "@radix-ui/react-slider": "^1.3.6", - "@types/better-sqlite3": "^7.6.13", "@types/node": "^25.2.3", "@types/papaparse": "^5.5.2", + "@types/pg": "^8.11.10", "@types/react": "^19.2.14", "@types/react-dom": "^19.2.3", "autoprefixer": "^10.4.24", - "better-sqlite3": "^12.6.2", "class-variance-authority": "^0.7.1", "clsx": "^2.1.1", "drizzle-orm": "^0.45.1", @@ -36,6 +35,7 @@ "next": "^16.1.6", "next-themes": "^0.4.6", "papaparse": "^5.5.3", + "pg": "^8.13.1", "postcss": "^8.5.6", "react": "^19.2.4", "react-dom": "^19.2.4", diff --git a/src/app/api/annotation-types/route.ts b/src/app/api/annotation-types/route.ts index 1a90966..7fa0c7d 100644 --- a/src/app/api/annotation-types/route.ts +++ b/src/app/api/annotation-types/route.ts @@ -30,7 +30,6 @@ export async function POST(request: NextRequest) { return NextResponse.json({ message: 'Types already seeded' }); } - const now = Math.floor(Date.now() / 1000); const defaultTypes = [ { name: 'break_up', @@ -38,8 +37,7 @@ export async function POST(request: NextRequest) { color: '#10b981', category: 'marker', icon: 'arrowUp', - is_active: 1, - created_at: now, + is_active: true, }, { name: 'break_down', @@ -47,8 +45,7 @@ export async function POST(request: NextRequest) { color: '#ef4444', category: 'marker', icon: 'arrowDown', - is_active: 1, - created_at: now, + is_active: true, }, { name: 'line', @@ -56,8 +53,7 @@ export async function POST(request: NextRequest) { color: '#3b82f6', category: 'line', icon: 'line', - is_active: 1, - created_at: now, + is_active: true, }, ]; @@ -83,8 +79,7 @@ export async function POST(request: NextRequest) { color, category, icon: icon || null, - is_active: 1, - created_at: Math.floor(Date.now() / 1000), + is_active: true, }) .returning(); diff --git a/src/app/api/annotations/[id]/route.ts b/src/app/api/annotations/[id]/route.ts index 971f071..afbc056 100644 --- a/src/app/api/annotations/[id]/route.ts +++ b/src/app/api/annotations/[id]/route.ts @@ -30,7 +30,7 @@ export async function PATCH( const result = await db .update(annotations) - .set({ geometry: JSON.stringify(geometry) }) + .set({ geometry }) .where(eq(annotations.id, id)) .returning(); @@ -41,11 +41,7 @@ export async function PATCH( ); } - const updated = result[0]; - return NextResponse.json({ - ...updated, - geometry: updated.geometry ? JSON.parse(updated.geometry as string) : null, - }); + return NextResponse.json(result[0]); } catch (error: any) { return NextResponse.json( { error: error.message || 'Failed to update annotation' }, diff --git a/src/app/api/annotations/route.ts b/src/app/api/annotations/route.ts index cf57b8c..2a49e65 100644 --- a/src/app/api/annotations/route.ts +++ b/src/app/api/annotations/route.ts @@ -23,13 +23,7 @@ export async function GET(request: NextRequest) { .from(annotations) .where(eq(annotations.chart_id, parseInt(chartId, 10))); - // Parse geometry from JSON string - const parsed = allAnnotations.map((annotation) => ({ - ...annotation, - geometry: annotation.geometry ? JSON.parse(annotation.geometry) : null, - })); - - return NextResponse.json(parsed); + return NextResponse.json(allAnnotations); } catch (error: any) { return NextResponse.json( { error: error.message || 'Failed to fetch annotations' }, @@ -61,30 +55,18 @@ export async function POST(request: NextRequest) { ); } - // Serialize geometry to JSON string if present - const geometryString = geometry ? JSON.stringify(geometry) : null; - const result = await db .insert(annotations) .values({ chart_id, timestamp, label_type, - geometry: geometryString, + geometry: geometry || null, color: color || '#3b82f6', - created_at: Math.floor(Date.now() / 1000), }) .returning(); - const created = result[0]; - - return NextResponse.json( - { - ...created, - geometry: created.geometry ? JSON.parse(created.geometry) : null, - }, - { status: 201 } - ); + return NextResponse.json(result[0], { status: 201 }); } catch (error: any) { return NextResponse.json( { error: error.message || 'Failed to create annotation' }, diff --git a/src/app/api/export/route.ts b/src/app/api/export/route.ts index 17d202f..5ce90a0 100644 --- a/src/app/api/export/route.ts +++ b/src/app/api/export/route.ts @@ -45,7 +45,7 @@ export async function GET(request: NextRequest) { price = candleResult[0].close; } } else if (annotation.label_type === 'line' && annotation.geometry) { - const geometry = JSON.parse(annotation.geometry); + const geometry = annotation.geometry as any; price = geometry.startPrice || null; } diff --git a/src/app/api/export/spans/route.ts b/src/app/api/export/spans/route.ts index 1681c93..9523020 100644 --- a/src/app/api/export/spans/route.ts +++ b/src/app/api/export/spans/route.ts @@ -44,7 +44,7 @@ export async function GET(request: NextRequest) { confidence: span.confidence, outcome: span.outcome, notes: span.notes, - sub_spans: span.sub_spans ? JSON.parse(span.sub_spans as string) : null, + sub_spans: span.sub_spans, color: span.color, created_at: span.created_at, })), diff --git a/src/app/api/span-annotations/[id]/route.ts b/src/app/api/span-annotations/[id]/route.ts index e67dae9..cbff156 100644 --- a/src/app/api/span-annotations/[id]/route.ts +++ b/src/app/api/span-annotations/[id]/route.ts @@ -34,7 +34,7 @@ export async function PATCH( if (confidence !== undefined) updates.confidence = confidence; if (outcome !== undefined) updates.outcome = outcome; if (notes !== undefined) updates.notes = notes; - if (sub_spans !== undefined) updates.sub_spans = sub_spans ? JSON.stringify(sub_spans) : null; + if (sub_spans !== undefined) updates.sub_spans = sub_spans || null; const result = await db .update(spanAnnotations) diff --git a/src/app/api/span-annotations/export/route.ts b/src/app/api/span-annotations/export/route.ts index d0f5ef4..7f8c6ba 100644 --- a/src/app/api/span-annotations/export/route.ts +++ b/src/app/api/span-annotations/export/route.ts @@ -69,17 +69,17 @@ export async function GET(request: NextRequest) { // Convert to ML pipeline format const annotations = spans.map(span => ({ id: span.id, - start_time: new Date(span.start_time * 1000).toISOString(), - end_time: new Date(span.end_time * 1000).toISOString(), + start_time: span.start_time, + end_time: span.end_time, label: span.label, confidence: span.confidence, outcome: span.outcome, notes: span.notes, - sub_spans: span.sub_spans ? JSON.parse(span.sub_spans) : null, + sub_spans: span.sub_spans, color: span.color, source: span.source, - model_prediction: span.model_prediction ? JSON.parse(span.model_prediction) : null, - created_at: new Date(span.created_at * 1000).toISOString(), + model_prediction: span.model_prediction, + created_at: span.created_at, })); return NextResponse.json({ @@ -93,12 +93,10 @@ export async function GET(request: NextRequest) { const csvRows = ['id,start_time,end_time,label,confidence,outcome,notes']; for (const span of spans) { - const startTime = new Date(span.start_time * 1000).toISOString(); - const endTime = new Date(span.end_time * 1000).toISOString(); const notes = span.notes ? `"${span.notes.replace(/"/g, '""')}"` : ''; csvRows.push( - `${span.id},${startTime},${endTime},${span.label},${span.confidence || ''},${span.outcome || ''},${notes}` + `${span.id},${span.start_time},${span.end_time},${span.label},${span.confidence || ''},${span.outcome || ''},${notes}` ); } diff --git a/src/app/api/span-annotations/route.ts b/src/app/api/span-annotations/route.ts index 9bd763e..c2aec67 100644 --- a/src/app/api/span-annotations/route.ts +++ b/src/app/api/span-annotations/route.ts @@ -71,11 +71,10 @@ export async function POST(request: NextRequest) { confidence: confidence || null, outcome: outcome || null, notes: notes || null, - sub_spans: sub_spans ? JSON.stringify(sub_spans) : null, + sub_spans: sub_spans || null, color: color || '#2196F3', source: source || 'human', // 'human', 'model', or 'human_correction' - model_prediction: model_prediction ? JSON.stringify(model_prediction) : null, - created_at: Math.floor(Date.now() / 1000), + model_prediction: model_prediction || null, }) .returning(); diff --git a/src/app/api/span-label-types/route.ts b/src/app/api/span-label-types/route.ts index b603be3..f6563b6 100644 --- a/src/app/api/span-label-types/route.ts +++ b/src/app/api/span-label-types/route.ts @@ -9,7 +9,7 @@ export async function GET() { const types = await db .select() .from(spanLabelTypes) - .where(eq(spanLabelTypes.is_active, 1)) + .where(eq(spanLabelTypes.is_active, true)) .orderBy(spanLabelTypes.sort_order); return NextResponse.json(types); @@ -42,9 +42,8 @@ export async function POST(request: NextRequest) { display_name, color, hotkey: hotkey || null, - is_active: 1, + is_active: true, sort_order: sort_order ?? 0, - created_at: Math.floor(Date.now() / 1000), }) .returning(); diff --git a/src/app/api/upload/route.ts b/src/app/api/upload/route.ts index 528147c..d209e49 100644 --- a/src/app/api/upload/route.ts +++ b/src/app/api/upload/route.ts @@ -85,7 +85,6 @@ export async function POST(request: NextRequest): Promise { // Create the chart const [newChart] = await db.insert(charts).values({ name: chartName, - created_at: Math.floor(Date.now() / 1000), }).returning(); // Parse and prepare candle data @@ -99,9 +98,10 @@ export async function POST(request: NextRequest): Promise { if (isNaN(date.getTime())) { throw new Error(`Invalid date format: ${row.time}`); } - timestamp = Math.floor(date.getTime() / 1000); + timestamp = date; // PostgreSQL timestamp type expects Date object or ISO string } else if (typeof row.time === 'number') { - timestamp = row.time; + // If Unix timestamp (seconds), convert to Date + timestamp = new Date(row.time * 1000); } else { throw new Error(`Invalid time value: ${row.time}`); } diff --git a/src/lib/db/index.ts b/src/lib/db/index.ts index 9723168..e9d2053 100644 --- a/src/lib/db/index.ts +++ b/src/lib/db/index.ts @@ -1,28 +1,30 @@ -import Database from 'better-sqlite3'; -import { drizzle } from 'drizzle-orm/better-sqlite3'; -import { migrate } from 'drizzle-orm/better-sqlite3/migrator'; +import { Pool } from 'pg'; +import { drizzle } from 'drizzle-orm/node-postgres'; +import { migrate } from 'drizzle-orm/node-postgres/migrator'; import * as schema from './schema'; import path from 'path'; -import fs from 'fs'; -// Ensure data directory exists -const dataDir = path.join(process.cwd(), 'data'); -if (!fs.existsSync(dataDir)) { - fs.mkdirSync(dataDir, { recursive: true }); +// Read DATABASE_URL from environment +const DATABASE_URL = process.env.DATABASE_URL; + +if (!DATABASE_URL) { + throw new Error('DATABASE_URL environment variable is not set'); } -const dbPath = path.join(dataDir, 'candles.db'); -const sqlite = new Database(dbPath); -export const db = drizzle(sqlite, { schema }); +// Create PostgreSQL connection pool +const pool = new Pool({ + connectionString: DATABASE_URL, + max: 10, +}); -// Run migrations at startup (for local dev). -// In Docker, migrations are run by scripts/run-migrations.js before the app starts, -// so this will be a no-op (all migrations already applied). +export const db = drizzle(pool, { schema }); + +// Run migrations at startup (skip during build phase) const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build'; if (!isBuildTime) { try { - migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') }); + await migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') }); console.log('✅ Database migrations completed'); } catch (error) { console.error('❌ Migration failed:', error); diff --git a/src/lib/db/schema.ts b/src/lib/db/schema.ts index 34f8f5a..c4b1a88 100644 --- a/src/lib/db/schema.ts +++ b/src/lib/db/schema.ts @@ -1,67 +1,67 @@ -import { sqliteTable, integer, real, text, uniqueIndex } from 'drizzle-orm/sqlite-core'; +import { pgTable, serial, text, timestamp, doublePrecision, integer, boolean, jsonb, uniqueIndex } from 'drizzle-orm/pg-core'; -export const charts = sqliteTable('charts', { - id: integer('id').primaryKey({ autoIncrement: true }), +export const charts = pgTable('charts', { + id: serial('id').primaryKey(), name: text('name').notNull().unique(), - created_at: integer('created_at').notNull(), + created_at: timestamp('created_at').notNull().defaultNow(), }); -export const candles = sqliteTable('candles', { - id: integer('id').primaryKey({ autoIncrement: true }), +export const candles = pgTable('candles', { + id: serial('id').primaryKey(), chart_id: integer('chart_id').notNull().references(() => charts.id), - time: integer('time').notNull(), - open: real('open').notNull(), - high: real('high').notNull(), - low: real('low').notNull(), - close: real('close').notNull(), + time: timestamp('time').notNull(), + open: doublePrecision('open').notNull(), + high: doublePrecision('high').notNull(), + low: doublePrecision('low').notNull(), + close: doublePrecision('close').notNull(), }, (table) => [ uniqueIndex('candles_chart_time_unique').on(table.chart_id, table.time), ]); -export const annotationTypes = sqliteTable('annotation_types', { - id: integer('id').primaryKey({ autoIncrement: true }), +export const annotationTypes = pgTable('annotation_types', { + id: serial('id').primaryKey(), name: text('name').notNull().unique(), // internal name (e.g., 'break_up') display_name: text('display_name').notNull(), // display name (e.g., 'Break Up') color: text('color').notNull(), // hex color code category: text('category').notNull(), // 'marker' or 'line' icon: text('icon'), // icon name or symbol - is_active: integer('is_active').notNull().default(1), // 1 = active, 0 = inactive - created_at: integer('created_at').notNull(), + is_active: boolean('is_active').notNull().default(true), // true = active, false = inactive + created_at: timestamp('created_at').notNull().defaultNow(), }); -export const annotations = sqliteTable('annotations', { - id: integer('id').primaryKey({ autoIncrement: true }), +export const annotations = pgTable('annotations', { + id: serial('id').primaryKey(), chart_id: integer('chart_id').notNull().references(() => charts.id), - timestamp: integer('timestamp').notNull(), + timestamp: timestamp('timestamp').notNull(), label_type: text('label_type').notNull(), - geometry: text('geometry'), // JSON string for line coordinates + geometry: jsonb('geometry'), // JSON for line coordinates color: text('color').default('#3b82f6'), // hex color code - created_at: integer('created_at').notNull(), + created_at: timestamp('created_at').notNull().defaultNow(), }); -export const spanLabelTypes = sqliteTable('span_label_types', { - id: integer('id').primaryKey({ autoIncrement: true }), +export const spanLabelTypes = pgTable('span_label_types', { + id: serial('id').primaryKey(), name: text('name').notNull().unique(), // internal name (e.g., 'bull_flag') display_name: text('display_name').notNull(), // UI label (e.g., 'Bull Flag') color: text('color').notNull(), // hex color for rectangle fill hotkey: text('hotkey'), // keyboard shortcut (e.g., '1') - is_active: integer('is_active').notNull().default(1), // 1 = active, 0 = inactive + is_active: boolean('is_active').notNull().default(true), // true = active, false = inactive sort_order: integer('sort_order').notNull().default(0), // display order - created_at: integer('created_at').notNull(), + created_at: timestamp('created_at').notNull().defaultNow(), }); -export const spanAnnotations = sqliteTable('span_annotations', { - id: integer('id').primaryKey({ autoIncrement: true }), +export const spanAnnotations = pgTable('span_annotations', { + id: serial('id').primaryKey(), chart_id: integer('chart_id').notNull().references(() => charts.id), - start_time: integer('start_time').notNull(), // Unix timestamp of first candle - end_time: integer('end_time').notNull(), // Unix timestamp of last candle + start_time: timestamp('start_time').notNull(), // timestamp of first candle + end_time: timestamp('end_time').notNull(), // timestamp of last candle label: text('label').notNull(), // pattern name referencing span_label_types.name confidence: integer('confidence'), // 1-5 scale, nullable outcome: text('outcome'), // 'win'|'loss'|'breakeven'|null notes: text('notes'), // free-text, nullable - sub_spans: text('sub_spans'), // JSON array of sub-span objects, nullable + sub_spans: jsonb('sub_spans'), // JSON array of sub-span objects, nullable color: text('color').notNull().default('#2196F3'), // hex color source: text('source').notNull().default('human'), // 'human'|'model'|'human_correction' - model_prediction: text('model_prediction'), // JSON metadata when confirming/correcting predictions - created_at: integer('created_at').notNull(), + model_prediction: jsonb('model_prediction'), // JSON metadata when confirming/correcting predictions + created_at: timestamp('created_at').notNull().defaultNow(), });