diff --git a/packages/db/src/client.ts b/packages/db/src/client.ts index 82eb764..96d5b3d 100644 --- a/packages/db/src/client.ts +++ b/packages/db/src/client.ts @@ -5,58 +5,15 @@ import * as schema from "@/schema"; const isDevelopment = process.env.NODE_ENV === "development"; -const connectionConfig = { +const pool = new Pool({ allowExitOnIdle: true, + connectionString: process.env.BASANGO_DATABASE_URL!, connectionTimeoutMillis: 15_000, idleTimeoutMillis: isDevelopment ? 5_000 : 60_000, max: isDevelopment ? 8 : 12, maxUses: isDevelopment ? 100 : 0, -}; - -const pool = new Pool({ - connectionString: process.env.BASANGO_DATABASE_URL!, - ...connectionConfig, }); -/** - * Retrieves runtime statistics for the database connection pool. - * - * This function reads internal pool and connection configuration values and returns - * a snapshot describing pool usage, capacity and utilization. Values that are not - * available on the underlying pool or configuration are normalized to safe defaults - * (zeros or false) so the result is stable. - * - * @returns An object describing the current connection pool statistics and a small summary. - */ -export const getConnectionPoolStats = () => { - const stats = { - active: Math.max(0, (pool.totalCount ?? 0) - (pool.idleCount ?? 0)), - ended: pool.ended ?? false, - idle: pool.idleCount ?? 0, - name: "primary", - total: pool.options.max ?? 0, - waiting: pool.waitingCount ?? 0, - }; - - const totalConnections = connectionConfig.max; - const utilization = - totalConnections > 0 ? Math.round((stats.active / totalConnections) * 100) : 0; - - return { - instance: "local", - pools: { primary: stats }, - region: "unknown", - summary: { - hasExhaustedPools: stats.active >= totalConnections || (stats.waiting ?? 0) > 0, - totalActive: stats.active, - totalConnections, - totalWaiting: stats.waiting, - utilizationPercent: utilization, - }, - timestamp: new Date().toISOString(), - }; -}; - export const db = drizzle(pool, { casing: "snake_case", schema, diff --git a/packages/db/src/importer/engine.ts b/packages/db/src/importer/engine.ts index 33e448f..0306f31 100644 --- a/packages/db/src/importer/engine.ts +++ b/packages/db/src/importer/engine.ts @@ -59,11 +59,11 @@ export class Engine { this.target = new Pool({ allowExitOnIdle: true, connectionString: this.targetOptions.database, - max: 8, + max: 16, }); this.ignore = { ...DEFAULT_IGNORE, ...(this.targetOptions.ignoreColumns ?? {}) }; - this.pageSize = this.targetOptions.pageSize ?? 10_000; - this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 1000); + this.pageSize = this.targetOptions.pageSize ?? 1000; + this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 50); console.log( `Engine initialized with pageSize=${this.pageSize} and batchSize=${this.batchSize}`, ); @@ -162,9 +162,10 @@ export class Engine { try { await target.query(insertSql, params); } catch (err: unknown) { + // Fallback: coerce all *_at params to now() and retry once + // This will never happen in production but anyway let's keep it safe const msg = String((err as Error)?.message ?? ""); if (msg.includes("invalid input syntax for type timestamp")) { - // Fallback: coerce all *_at params to now() and retry once const fixed = columns!.map((c, i) => (c.endsWith("_at") ? new Date() : params[i])); await target.query(insertSql, fixed); } else { @@ -224,7 +225,6 @@ export class Engine { const t = this.normalizedName(table); const clone: Record = { ...row }; - // Normalize UUIDs and timestamps and categories for (const [key, val] of Object.entries(clone)) { if (val == null) continue; @@ -233,7 +233,6 @@ export class Engine { continue; } - // Robust timestamp normalization for *_at columns if (key.endsWith("_at")) { clone[key] = this.normalizeTimestampValue(val); continue; @@ -263,14 +262,6 @@ export class Engine { } } - if (t === "article" && key === "token_statistics") { - clone[key] = computeTokenStatistics({ - body: String(clone.body ?? ""), - categories: Array.isArray(clone.categories) ? clone.categories : [], - title: String(clone.title ?? ""), - }); - } - if (t === "article" && key === "reading_time") { clone[key] = Math.max(1, computeReadingTime(String(clone.body ?? ""))); } @@ -279,24 +270,9 @@ export class Engine { if (Array.isArray(val)) { clone[key] = val; } else if (typeof val === "string") { - const raw = val.trim(); - - // If the value is a JSON array string like '["ROLE_USER","ROLE_ADMIN"]', parse it. - try { - const parsed = JSON.parse(raw); - if (Array.isArray(parsed)) { - clone[key] = parsed; - continue; - } - } catch { - // not JSON, fall back to CSV-like parsing below - } - - // Remove surrounding brackets/quotes then split by comma and strip quotes/space - const parts = raw - .replace(/^\[|\]$/g, "") + const parts = val .split(",") - .map((s) => s.replace(/^["']|["']$/g, "").trim()) + .map((s) => s.trim()) .filter(Boolean); clone[key] = parts.length ? parts : ["ROLE_USER"]; @@ -304,7 +280,6 @@ export class Engine { } } - // compute credibility JSON if bias/reliability/transparency present if (t === "article" || t === "source") { const bias = clone.bias ?? null; const reliability = clone.reliability ?? null; @@ -318,14 +293,10 @@ export class Engine { } } - // Ensure article token_statistics exists (computed on the fly) - if ( - t === "article" && - (clone.token_statistics == null || typeof clone.token_statistics !== "object") - ) { + if (t === "article") { clone.token_statistics = computeTokenStatistics({ body: String(clone.body ?? ""), - categories: Array.isArray(clone.categories) ? (clone.categories as string[]) : [], + categories: Array.isArray(clone.categories) ? clone.categories : [], title: String(clone.title ?? ""), }); } @@ -352,7 +323,7 @@ export class Engine { return JSON.stringify(v); } if (col === "roles" && v) { - return JSON.stringify(v); + return v; } if (col === "metadata" && v && typeof v === "object") { return JSON.stringify(v); diff --git a/packages/db/src/importer/import.ts b/packages/db/src/importer/import.ts index 0d7d5d9..34be5ef 100644 --- a/packages/db/src/importer/import.ts +++ b/packages/db/src/importer/import.ts @@ -14,7 +14,7 @@ const env = createEnvAccessor([ "BASANGO_DATABASE_URL", ]); -async function promptConfirm(question: string, def = false) { +async function askConfirmation(question: string, def = false) { const rl = createInterface({ input, output }); const suffix = def ? "[Y/n]" : "[y/N]"; const answer = await rl.question(`${question} ${suffix} `); @@ -28,7 +28,7 @@ async function promptConfirm(question: string, def = false) { } async function main() { - const ok = await promptConfirm("Do you want to continue?", false); + const ok = await askConfirmation("Do you want to continue?", false); if (!ok) { console.warn("Process aborted"); process.exit(1); diff --git a/packages/db/src/schema.ts b/packages/db/src/schema.ts index 15f44ad..eaad54c 100644 --- a/packages/db/src/schema.ts +++ b/packages/db/src/schema.ts @@ -101,12 +101,6 @@ export type GeoLocation = { accuracyRadius?: number; }; -export type ClientProfile = { - userIp?: string; - userAgent?: string; - hints: unknown[]; -}; - export type ArticleMetadata = { title?: string; description?: string; diff --git a/packages/db/src/utils/api-keys.ts b/packages/db/src/utils/api-keys.ts deleted file mode 100644 index d20a6b1..0000000 --- a/packages/db/src/utils/api-keys.ts +++ /dev/null @@ -1,20 +0,0 @@ -import { randomBytes } from "node:crypto"; - -/** - * Generates a new API key with the format mid_{random_string} - * @returns A new API key string - */ -export function generateApiKey(): string { - // Generate 32 random bytes and convert to hex - const randomString = randomBytes(32).toString("hex"); - return `basango_${randomString}`; -} - -/** - * Validates if a string is a valid API key format - * @param key The key to validate - * @returns True if the key starts with 'basango_' and has the correct length - */ -export function isValidApiKeyFormat(key: string): boolean { - return key.startsWith("basango_") && key.length === 68; // basango_ (8) + 64 hex chars -} diff --git a/packages/db/src/utils/computed.ts b/packages/db/src/utils/computed.ts index fb83d13..b020956 100644 --- a/packages/db/src/utils/computed.ts +++ b/packages/db/src/utils/computed.ts @@ -31,10 +31,12 @@ export const computeTokenStatistics = (data: { body: string; categories: string[]; }): TokenStatistics => { - const title = computeTokenCount(data.title); - const body = computeTokenCount(data.body); - const categories = computeTokenCount(data.categories.join(",")); - const excerpt = computeTokenCount(data.body.substring(0, 200)); + const [title, body, categories, excerpt] = [ + computeTokenCount(data.title), + computeTokenCount(data.body), + computeTokenCount(data.categories.join(",")), + computeTokenCount(data.body.substring(0, 200)), + ]; return { body, diff --git a/packages/db/src/utils/health.ts b/packages/db/src/utils/health.ts deleted file mode 100644 index 19fc0bc..0000000 --- a/packages/db/src/utils/health.ts +++ /dev/null @@ -1,7 +0,0 @@ -import { sql } from "drizzle-orm"; - -import { db } from "@/client"; - -export async function checkHealth() { - await db.execute(sql`SELECT 1`); -} diff --git a/packages/db/src/utils/index.ts b/packages/db/src/utils/index.ts index f6feccc..92179a1 100644 --- a/packages/db/src/utils/index.ts +++ b/packages/db/src/utils/index.ts @@ -1,4 +1,3 @@ -export * from "./api-keys"; -export * from "./health"; +export * from "./computed"; export * from "./pagination"; export * from "./search-query";