fix(db): compute token statistics

packages/db/src/client.ts

@@ -5,58 +5,15 @@ import * as schema from "@/schema";
 
 const isDevelopment = process.env.NODE_ENV === "development";
 
-const connectionConfig = {
+const pool = new Pool({
   allowExitOnIdle: true,
+  connectionString: process.env.BASANGO_DATABASE_URL!,
   connectionTimeoutMillis: 15_000,
   idleTimeoutMillis: isDevelopment ? 5_000 : 60_000,
   max: isDevelopment ? 8 : 12,
   maxUses: isDevelopment ? 100 : 0,
-};
-
-const pool = new Pool({
-  connectionString: process.env.BASANGO_DATABASE_URL!,
-  ...connectionConfig,
 });
 
-/**
- * Retrieves runtime statistics for the database connection pool.
- *
- * This function reads internal pool and connection configuration values and returns
- * a snapshot describing pool usage, capacity and utilization. Values that are not
- * available on the underlying pool or configuration are normalized to safe defaults
- * (zeros or false) so the result is stable.
- *
- * @returns An object describing the current connection pool statistics and a small summary.
- */
-export const getConnectionPoolStats = () => {
-  const stats = {
-    active: Math.max(0, (pool.totalCount ?? 0) - (pool.idleCount ?? 0)),
-    ended: pool.ended ?? false,
-    idle: pool.idleCount ?? 0,
-    name: "primary",
-    total: pool.options.max ?? 0,
-    waiting: pool.waitingCount ?? 0,
-  };
-
-  const totalConnections = connectionConfig.max;
-  const utilization =
-    totalConnections > 0 ? Math.round((stats.active / totalConnections) * 100) : 0;
-
-  return {
-    instance: "local",
-    pools: { primary: stats },
-    region: "unknown",
-    summary: {
-      hasExhaustedPools: stats.active >= totalConnections || (stats.waiting ?? 0) > 0,
-      totalActive: stats.active,
-      totalConnections,
-      totalWaiting: stats.waiting,
-      utilizationPercent: utilization,
-    },
-    timestamp: new Date().toISOString(),
-  };
-};
-
 export const db = drizzle(pool, {
   casing: "snake_case",
   schema,

packages/db/src/importer/engine.ts

@@ -59,11 +59,11 @@ export class Engine {
     this.target = new Pool({
       allowExitOnIdle: true,
       connectionString: this.targetOptions.database,
-      max: 8,
+      max: 16,
     });
     this.ignore = { ...DEFAULT_IGNORE, ...(this.targetOptions.ignoreColumns ?? {}) };
-    this.pageSize = this.targetOptions.pageSize ?? 10_000;
-    this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 1000);
+    this.pageSize = this.targetOptions.pageSize ?? 1000;
+    this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 50);
     console.log(
       `Engine initialized with pageSize=${this.pageSize} and batchSize=${this.batchSize}`,
     );
@@ -162,9 +162,10 @@ export class Engine {
         try {
           await target.query(insertSql, params);
         } catch (err: unknown) {
+          // Fallback: coerce all *_at params to now() and retry once
+          // This will never happen in production but anyway let's keep it safe
           const msg = String((err as Error)?.message ?? "");
           if (msg.includes("invalid input syntax for type timestamp")) {
-            // Fallback: coerce all *_at params to now() and retry once
             const fixed = columns!.map((c, i) => (c.endsWith("_at") ? new Date() : params[i]));
             await target.query(insertSql, fixed);
           } else {
@@ -224,7 +225,6 @@ export class Engine {
     const t = this.normalizedName(table);
     const clone: Record<string, unknown> = { ...row };
 
-    // Normalize UUIDs and timestamps and categories
     for (const [key, val] of Object.entries(clone)) {
       if (val == null) continue;
 
@@ -233,7 +233,6 @@ export class Engine {
         continue;
       }
 
-      // Robust timestamp normalization for *_at columns
       if (key.endsWith("_at")) {
         clone[key] = this.normalizeTimestampValue(val);
         continue;
@@ -263,14 +262,6 @@ export class Engine {
         }
       }
 
-      if (t === "article" && key === "token_statistics") {
-        clone[key] = computeTokenStatistics({
-          body: String(clone.body ?? ""),
-          categories: Array.isArray(clone.categories) ? clone.categories : [],
-          title: String(clone.title ?? ""),
-        });
-      }
-
       if (t === "article" && key === "reading_time") {
         clone[key] = Math.max(1, computeReadingTime(String(clone.body ?? "")));
       }
@@ -279,24 +270,9 @@ export class Engine {
         if (Array.isArray(val)) {
           clone[key] = val;
         } else if (typeof val === "string") {
-          const raw = val.trim();
-
-          // If the value is a JSON array string like '["ROLE_USER","ROLE_ADMIN"]', parse it.
-          try {
-            const parsed = JSON.parse(raw);
-            if (Array.isArray(parsed)) {
-              clone[key] = parsed;
-              continue;
-            }
-          } catch {
-            // not JSON, fall back to CSV-like parsing below
-          }
-
-          // Remove surrounding brackets/quotes then split by comma and strip quotes/space
-          const parts = raw
-            .replace(/^\[|\]$/g, "")
+          const parts = val
             .split(",")
-            .map((s) => s.replace(/^["']|["']$/g, "").trim())
+            .map((s) => s.trim())
             .filter(Boolean);
 
           clone[key] = parts.length ? parts : ["ROLE_USER"];
@@ -304,7 +280,6 @@ export class Engine {
       }
     }
 
-    // compute credibility JSON if bias/reliability/transparency present
     if (t === "article" || t === "source") {
       const bias = clone.bias ?? null;
       const reliability = clone.reliability ?? null;
@@ -318,14 +293,10 @@ export class Engine {
       }
     }
 
-    // Ensure article token_statistics exists (computed on the fly)
-    if (
-      t === "article" &&
-      (clone.token_statistics == null || typeof clone.token_statistics !== "object")
-    ) {
+    if (t === "article") {
       clone.token_statistics = computeTokenStatistics({
         body: String(clone.body ?? ""),
-        categories: Array.isArray(clone.categories) ? (clone.categories as string[]) : [],
+        categories: Array.isArray(clone.categories) ? clone.categories : [],
         title: String(clone.title ?? ""),
       });
     }
@@ -352,7 +323,7 @@ export class Engine {
       return JSON.stringify(v);
     }
     if (col === "roles" && v) {
-      return JSON.stringify(v);
+      return v;
     }
     if (col === "metadata" && v && typeof v === "object") {
       return JSON.stringify(v);

packages/db/src/importer/import.ts

@@ -14,7 +14,7 @@ const env = createEnvAccessor([
   "BASANGO_DATABASE_URL",
 ]);
 
-async function promptConfirm(question: string, def = false) {
+async function askConfirmation(question: string, def = false) {
   const rl = createInterface({ input, output });
   const suffix = def ? "[Y/n]" : "[y/N]";
   const answer = await rl.question(`${question} ${suffix} `);
@@ -28,7 +28,7 @@ async function promptConfirm(question: string, def = false) {
 }
 
 async function main() {
-  const ok = await promptConfirm("Do you want to continue?", false);
+  const ok = await askConfirmation("Do you want to continue?", false);
   if (!ok) {
     console.warn("Process aborted");
     process.exit(1);

packages/db/src/schema.ts

@@ -101,12 +101,6 @@ export type GeoLocation = {
   accuracyRadius?: number;
 };
 
-export type ClientProfile = {
-  userIp?: string;
-  userAgent?: string;
-  hints: unknown[];
-};
-
 export type ArticleMetadata = {
   title?: string;
   description?: string;

packages/db/src/utils/api-keys.ts

@@ -1,20 +0,0 @@
-import { randomBytes } from "node:crypto";
-
-/**
- * Generates a new API key with the format mid_{random_string}
- * @returns A new API key string
- */
-export function generateApiKey(): string {
-  // Generate 32 random bytes and convert to hex
-  const randomString = randomBytes(32).toString("hex");
-  return `basango_${randomString}`;
-}
-
-/**
- * Validates if a string is a valid API key format
- * @param key The key to validate
- * @returns True if the key starts with 'basango_' and has the correct length
- */
-export function isValidApiKeyFormat(key: string): boolean {
-  return key.startsWith("basango_") && key.length === 68; // basango_ (8) + 64 hex chars
-}

packages/db/src/utils/computed.ts

@@ -31,10 +31,12 @@ export const computeTokenStatistics = (data: {
   body: string;
   categories: string[];
 }): TokenStatistics => {
-  const title = computeTokenCount(data.title);
-  const body = computeTokenCount(data.body);
-  const categories = computeTokenCount(data.categories.join(","));
-  const excerpt = computeTokenCount(data.body.substring(0, 200));
+  const [title, body, categories, excerpt] = [
+    computeTokenCount(data.title),
+    computeTokenCount(data.body),
+    computeTokenCount(data.categories.join(",")),
+    computeTokenCount(data.body.substring(0, 200)),
+  ];
 
   return {
     body,

packages/db/src/utils/health.ts

@@ -1,7 +0,0 @@
-import { sql } from "drizzle-orm";
-
-import { db } from "@/client";
-
-export async function checkHealth() {
-  await db.execute(sql`SELECT 1`);
-}

packages/db/src/utils/index.ts

@@ -1,4 +1,3 @@
-export * from "./api-keys";
-export * from "./health";
+export * from "./computed";
 export * from "./pagination";
 export * from "./search-query";