fix: replace all SQLite references with PostgreSQL in scripts and lazy-init db connection
- Rewrite scripts/run-migrations.js for PostgreSQL (was better-sqlite3) - Rewrite scripts/load-initial-data.js for PostgreSQL (was better-sqlite3) - Make db connection lazy in src/lib/db/index.ts to avoid build-time errors when DATABASE_URL is not available in Docker build stage
This commit is contained in:
Marko Djordjevic
parent
2481fda68c
commit
d5fc4662e9
3 changed files with 139 additions and 149 deletions
|
|
@ -1,33 +1,37 @@
|
|||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const Database = require('better-sqlite3');
|
||||
const { Pool } = require('pg');
|
||||
const Papa = require('papaparse');
|
||||
|
||||
const DB_PATH = process.env.DATABASE_PATH || path.join(__dirname, '..', 'data', 'candles.db');
|
||||
const CSV_PATH = process.env.CSV_PATH || path.join(__dirname, '..', 'EURUSD.csv');
|
||||
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error('❌ DATABASE_URL environment variable is not set');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
async function loadInitialData() {
|
||||
console.log('Checking if initial data needs to be loaded...');
|
||||
|
||||
const db = new Database(DB_PATH);
|
||||
const pool = new Pool({ connectionString: DATABASE_URL, max: 2 });
|
||||
|
||||
try {
|
||||
// Check if candles table exists and has any data
|
||||
const tableExists = db.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='candles'"
|
||||
).get();
|
||||
const tableCheck = await pool.query(
|
||||
"SELECT EXISTS (SELECT FROM information_schema.tables WHERE table_name = 'candles') AS exists"
|
||||
);
|
||||
|
||||
if (!tableExists) {
|
||||
if (!tableCheck.rows[0].exists) {
|
||||
console.log('Candles table does not exist yet (migrations will create it). Skipping initial data load.');
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
const count = db.prepare('SELECT COUNT(*) as count FROM candles').get();
|
||||
const countResult = await pool.query('SELECT COUNT(*) as count FROM candles');
|
||||
const count = parseInt(countResult.rows[0].count, 10);
|
||||
|
||||
if (count.count > 0) {
|
||||
console.log(`Database already has ${count.count} candles. Skipping initial data load.`);
|
||||
db.close();
|
||||
if (count > 0) {
|
||||
console.log(`Database already has ${count} candles. Skipping initial data load.`);
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -36,94 +40,90 @@ async function loadInitialData() {
|
|||
// Check if CSV file exists
|
||||
if (!fs.existsSync(CSV_PATH)) {
|
||||
console.log(`CSV file not found at ${CSV_PATH}. Skipping initial data load.`);
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
// Create a default chart for the initial data
|
||||
const chartName = 'EURUSD';
|
||||
const now = Math.floor(Date.now() / 1000);
|
||||
const chartResult = db.prepare(
|
||||
'INSERT INTO charts (name, created_at) VALUES (?, ?) RETURNING id'
|
||||
).get(chartName, now);
|
||||
const chartId = chartResult.id;
|
||||
const chartResult = await pool.query(
|
||||
'INSERT INTO charts (name, created_at) VALUES ($1, NOW()) RETURNING id',
|
||||
[chartName]
|
||||
);
|
||||
const chartId = chartResult.rows[0].id;
|
||||
console.log(`Created chart "${chartName}" with id ${chartId}`);
|
||||
|
||||
// Read and parse CSV
|
||||
const csvContent = fs.readFileSync(CSV_PATH, 'utf8');
|
||||
|
||||
Papa.parse(csvContent, {
|
||||
const results = Papa.parse(csvContent, {
|
||||
header: true,
|
||||
dynamicTyping: true,
|
||||
skipEmptyLines: true,
|
||||
complete: (results) => {
|
||||
try {
|
||||
const rows = results.data;
|
||||
|
||||
if (rows.length === 0) {
|
||||
console.log('CSV file is empty.');
|
||||
db.close();
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Parsed ${rows.length} rows from CSV`);
|
||||
|
||||
// Prepare insert statement
|
||||
const insert = db.prepare(
|
||||
'INSERT INTO candles (chart_id, time, open, high, low, close) VALUES (?, ?, ?, ?, ?, ?)'
|
||||
);
|
||||
|
||||
const insertMany = db.transaction((candles) => {
|
||||
for (const candle of candles) {
|
||||
insert.run(chartId, candle.time, candle.open, candle.high, candle.low, candle.close);
|
||||
}
|
||||
});
|
||||
|
||||
// Parse and prepare candle data
|
||||
const candleData = rows.map((row) => {
|
||||
let timestamp;
|
||||
|
||||
// Handle both date strings and Unix timestamps
|
||||
if (typeof row.time === 'string') {
|
||||
// Try parsing as date string
|
||||
const date = new Date(row.time);
|
||||
if (isNaN(date.getTime())) {
|
||||
throw new Error(`Invalid date format: ${row.time}`);
|
||||
}
|
||||
timestamp = Math.floor(date.getTime() / 1000);
|
||||
} else if (typeof row.time === 'number') {
|
||||
timestamp = row.time;
|
||||
} else {
|
||||
throw new Error(`Invalid time value: ${row.time}`);
|
||||
}
|
||||
|
||||
return {
|
||||
time: timestamp,
|
||||
open: Number(row.open),
|
||||
high: Number(row.high),
|
||||
low: Number(row.low),
|
||||
close: Number(row.close),
|
||||
};
|
||||
});
|
||||
|
||||
// Insert all candles in a transaction
|
||||
insertMany(candleData);
|
||||
|
||||
console.log(`Successfully loaded ${candleData.length} candles into the database.`);
|
||||
} catch (error) {
|
||||
console.error('Error loading initial data:', error);
|
||||
} finally {
|
||||
db.close();
|
||||
}
|
||||
},
|
||||
error: (error) => {
|
||||
console.error('CSV parsing error:', error);
|
||||
db.close();
|
||||
},
|
||||
});
|
||||
|
||||
const rows = results.data;
|
||||
|
||||
if (rows.length === 0) {
|
||||
console.log('CSV file is empty.');
|
||||
return;
|
||||
}
|
||||
|
||||
console.log(`Parsed ${rows.length} rows from CSV`);
|
||||
|
||||
// Parse candle data
|
||||
const candleData = rows.map((row) => {
|
||||
let timestamp;
|
||||
|
||||
if (typeof row.time === 'string') {
|
||||
const date = new Date(row.time);
|
||||
if (isNaN(date.getTime())) {
|
||||
throw new Error(`Invalid date format: ${row.time}`);
|
||||
}
|
||||
timestamp = date.toISOString();
|
||||
} else if (typeof row.time === 'number') {
|
||||
timestamp = new Date(row.time * 1000).toISOString();
|
||||
} else {
|
||||
throw new Error(`Invalid time value: ${row.time}`);
|
||||
}
|
||||
|
||||
return {
|
||||
time: timestamp,
|
||||
open: Number(row.open),
|
||||
high: Number(row.high),
|
||||
low: Number(row.low),
|
||||
close: Number(row.close),
|
||||
};
|
||||
});
|
||||
|
||||
// Batch insert using parameterized queries
|
||||
const BATCH_SIZE = 500;
|
||||
let inserted = 0;
|
||||
|
||||
for (let i = 0; i < candleData.length; i += BATCH_SIZE) {
|
||||
const batch = candleData.slice(i, i + BATCH_SIZE);
|
||||
const values = [];
|
||||
const params = [];
|
||||
let paramIdx = 1;
|
||||
|
||||
for (const candle of batch) {
|
||||
values.push(`($${paramIdx}, $${paramIdx + 1}, $${paramIdx + 2}, $${paramIdx + 3}, $${paramIdx + 4}, $${paramIdx + 5})`);
|
||||
params.push(chartId, candle.time, candle.open, candle.high, candle.low, candle.close);
|
||||
paramIdx += 6;
|
||||
}
|
||||
|
||||
await pool.query(
|
||||
`INSERT INTO candles (chart_id, time, open, high, low, close) VALUES ${values.join(', ')}`,
|
||||
params
|
||||
);
|
||||
inserted += batch.length;
|
||||
}
|
||||
|
||||
console.log(`Successfully loaded ${inserted} candles into the database.`);
|
||||
} catch (error) {
|
||||
console.error('Error checking database:', error);
|
||||
db.close();
|
||||
console.error('Error loading initial data:', error);
|
||||
process.exit(1);
|
||||
} finally {
|
||||
await pool.end();
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -1,51 +1,25 @@
|
|||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
const Database = require('better-sqlite3');
|
||||
const { Pool } = require('pg');
|
||||
const { drizzle } = require('drizzle-orm/node-postgres');
|
||||
const { migrate } = require('drizzle-orm/node-postgres/migrator');
|
||||
|
||||
const dataDir = path.join(__dirname, '..', 'data');
|
||||
const dbPath = path.join(dataDir, 'candles.db');
|
||||
const migrationsFolder = path.join(__dirname, '..', 'drizzle');
|
||||
|
||||
// Ensure data directory exists
|
||||
if (!fs.existsSync(dataDir)) {
|
||||
fs.mkdirSync(dataDir, { recursive: true });
|
||||
}
|
||||
|
||||
// Check for inconsistent DB state before migrating
|
||||
if (fs.existsSync(dbPath)) {
|
||||
try {
|
||||
const checkDb = new Database(dbPath);
|
||||
const hasDrizzleMigrations = checkDb.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name='__drizzle_migrations'"
|
||||
).get();
|
||||
const hasAnyTables = checkDb.prepare(
|
||||
"SELECT name FROM sqlite_master WHERE type='table' AND name != '__drizzle_migrations'"
|
||||
).get();
|
||||
checkDb.close();
|
||||
|
||||
if (hasAnyTables && !hasDrizzleMigrations) {
|
||||
console.log('⚠️ Database has tables but no migration tracking. Recreating...');
|
||||
fs.unlinkSync(dbPath);
|
||||
}
|
||||
} catch {
|
||||
console.log('⚠️ Database file is corrupted. Recreating...');
|
||||
try { fs.unlinkSync(dbPath); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
// Run migrations using better-sqlite3 directly
|
||||
const { drizzle } = require('drizzle-orm/better-sqlite3');
|
||||
const { migrate } = require('drizzle-orm/better-sqlite3/migrator');
|
||||
|
||||
const sqlite = new Database(dbPath);
|
||||
const db = drizzle(sqlite);
|
||||
|
||||
try {
|
||||
migrate(db, { migrationsFolder });
|
||||
console.log('✅ Database migrations completed');
|
||||
} catch (error) {
|
||||
console.error('❌ Migration failed:', error);
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
console.error('❌ DATABASE_URL environment variable is not set');
|
||||
process.exit(1);
|
||||
} finally {
|
||||
sqlite.close();
|
||||
}
|
||||
|
||||
const pool = new Pool({ connectionString: DATABASE_URL, max: 2 });
|
||||
const db = drizzle(pool);
|
||||
|
||||
migrate(db, { migrationsFolder })
|
||||
.then(() => {
|
||||
console.log('✅ Database migrations completed');
|
||||
return pool.end();
|
||||
})
|
||||
.catch((error) => {
|
||||
console.error('❌ Migration failed:', error);
|
||||
pool.end().finally(() => process.exit(1));
|
||||
});
|
||||
|
|
|
|||
|
|
@ -1,35 +1,51 @@
|
|||
import { Pool } from 'pg';
|
||||
import { drizzle } from 'drizzle-orm/node-postgres';
|
||||
import { drizzle, type NodePgDatabase } from 'drizzle-orm/node-postgres';
|
||||
import { migrate } from 'drizzle-orm/node-postgres/migrator';
|
||||
import * as schema from './schema';
|
||||
import path from 'path';
|
||||
|
||||
// Read DATABASE_URL from environment
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
let _pool: Pool | null = null;
|
||||
let _db: NodePgDatabase<typeof schema> | null = null;
|
||||
let _migrated = false;
|
||||
|
||||
if (!DATABASE_URL) {
|
||||
throw new Error('DATABASE_URL environment variable is not set');
|
||||
function getPool(): Pool {
|
||||
if (!_pool) {
|
||||
const DATABASE_URL = process.env.DATABASE_URL;
|
||||
if (!DATABASE_URL) {
|
||||
throw new Error('DATABASE_URL environment variable is not set');
|
||||
}
|
||||
_pool = new Pool({ connectionString: DATABASE_URL, max: 10 });
|
||||
}
|
||||
return _pool;
|
||||
}
|
||||
|
||||
// Create PostgreSQL connection pool
|
||||
const pool = new Pool({
|
||||
connectionString: DATABASE_URL,
|
||||
max: 10,
|
||||
});
|
||||
function getDb(): NodePgDatabase<typeof schema> {
|
||||
if (!_db) {
|
||||
_db = drizzle(getPool(), { schema });
|
||||
}
|
||||
return _db;
|
||||
}
|
||||
|
||||
export const db = drizzle(pool, { schema });
|
||||
|
||||
// Run migrations at startup (skip during build phase)
|
||||
const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build';
|
||||
|
||||
if (!isBuildTime) {
|
||||
export async function runMigrations() {
|
||||
if (_migrated) return;
|
||||
const isBuildTime = process.env.NEXT_PHASE === 'phase-production-build' || process.env.NEXT_PHASE === 'phase-development-build';
|
||||
if (isBuildTime) {
|
||||
console.log('ℹ️ Skipping migrations during build phase');
|
||||
return;
|
||||
}
|
||||
try {
|
||||
await migrate(db, { migrationsFolder: path.join(process.cwd(), 'drizzle') });
|
||||
await migrate(getDb(), { migrationsFolder: path.join(process.cwd(), 'drizzle') });
|
||||
console.log('✅ Database migrations completed');
|
||||
_migrated = true;
|
||||
} catch (error) {
|
||||
console.error('❌ Migration failed:', error);
|
||||
throw error;
|
||||
}
|
||||
} else {
|
||||
console.log('ℹ️ Skipping migrations during build phase');
|
||||
}
|
||||
|
||||
// Lazy proxy: db is accessed as a module export but only connects on first use
|
||||
export const db = new Proxy({} as NodePgDatabase<typeof schema>, {
|
||||
get(_target, prop, receiver) {
|
||||
return Reflect.get(getDb(), prop, receiver);
|
||||
},
|
||||
});
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue