bernard-ng/basango
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix(db): compute token statistics

Browse Source
This commit is contained in:
bernard-ng
2025-11-10 20:39:58 +02:00
parent fbca02bec6
commit 801bc27a88
8 changed files with 21 additions and 125 deletions
Download Patch File Download Diff File Expand all files Collapse all files
packages/db/src/client.ts
+2 -45
View File
@@ -5,58 +5,15 @@ import * as schema from "@/schema";
const isDevelopment = process.env.NODE_ENV === "development";
const connectionConfig = {
const pool = new Pool({
allowExitOnIdle: true,
connectionString: process.env.BASANGO_DATABASE_URL!,
connectionTimeoutMillis: 15_000,
idleTimeoutMillis: isDevelopment ? 5_000 : 60_000,
max: isDevelopment ? 8 : 12,
maxUses: isDevelopment ? 100 : 0,
};
const pool = new Pool({
connectionString: process.env.BASANGO_DATABASE_URL!,
...connectionConfig,
});
/**
* Retrieves runtime statistics for the database connection pool.
*
* This function reads internal pool and connection configuration values and returns
* a snapshot describing pool usage, capacity and utilization. Values that are not
* available on the underlying pool or configuration are normalized to safe defaults
* (zeros or false) so the result is stable.
*
* @returns An object describing the current connection pool statistics and a small summary.
*/
export const getConnectionPoolStats = () => {
const stats = {
active: Math.max(0, (pool.totalCount ?? 0) - (pool.idleCount ?? 0)),
ended: pool.ended ?? false,
idle: pool.idleCount ?? 0,
name: "primary",
total: pool.options.max ?? 0,
waiting: pool.waitingCount ?? 0,
};
const totalConnections = connectionConfig.max;
const utilization =
totalConnections > 0 ? Math.round((stats.active / totalConnections) * 100) : 0;
return {
instance: "local",
pools: { primary: stats },
region: "unknown",
summary: {
hasExhaustedPools: stats.active >= totalConnections || (stats.waiting ?? 0) > 0,
totalActive: stats.active,
totalConnections,
totalWaiting: stats.waiting,
utilizationPercent: utilization,
},
timestamp: new Date().toISOString(),
};
};
export const db = drizzle(pool, {
casing: "snake_case",
schema,
packages/db/src/importer/engine.ts
+10 -39
View File
@@ -59,11 +59,11 @@ export class Engine {
this.target = new Pool({
allowExitOnIdle: true,
connectionString: this.targetOptions.database,
max: 8,
max: 16,
});
this.ignore = { ...DEFAULT_IGNORE, ...(this.targetOptions.ignoreColumns ?? {}) };
this.pageSize = this.targetOptions.pageSize ?? 10_000;
this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 1000);
this.pageSize = this.targetOptions.pageSize ?? 1000;
this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 50);
console.log(
`Engine initialized with pageSize=${this.pageSize} and batchSize=${this.batchSize}`,
);
@@ -162,9 +162,10 @@ export class Engine {
try {
await target.query(insertSql, params);
} catch (err: unknown) {
// Fallback: coerce all *_at params to now() and retry once
// This will never happen in production but anyway let's keep it safe
const msg = String((err as Error)?.message ?? "");
if (msg.includes("invalid input syntax for type timestamp")) {
// Fallback: coerce all *_at params to now() and retry once
const fixed = columns!.map((c, i) => (c.endsWith("_at") ? new Date() : params[i]));
await target.query(insertSql, fixed);
} else {
@@ -224,7 +225,6 @@ export class Engine {
const t = this.normalizedName(table);
const clone: Record<string, unknown> = { ...row };
// Normalize UUIDs and timestamps and categories
for (const [key, val] of Object.entries(clone)) {
if (val == null) continue;
@@ -233,7 +233,6 @@ export class Engine {
continue;
}
// Robust timestamp normalization for *_at columns
if (key.endsWith("_at")) {
clone[key] = this.normalizeTimestampValue(val);
continue;
@@ -263,14 +262,6 @@ export class Engine {
}
}
if (t === "article" && key === "token_statistics") {
clone[key] = computeTokenStatistics({
body: String(clone.body ?? ""),
categories: Array.isArray(clone.categories) ? clone.categories : [],
title: String(clone.title ?? ""),
});
}
if (t === "article" && key === "reading_time") {
clone[key] = Math.max(1, computeReadingTime(String(clone.body ?? "")));
}
@@ -279,24 +270,9 @@ export class Engine {
if (Array.isArray(val)) {
clone[key] = val;
} else if (typeof val === "string") {
const raw = val.trim();
// If the value is a JSON array string like '["ROLE_USER","ROLE_ADMIN"]', parse it.
try {
const parsed = JSON.parse(raw);
if (Array.isArray(parsed)) {
clone[key] = parsed;
continue;
}
} catch {
// not JSON, fall back to CSV-like parsing below
}
// Remove surrounding brackets/quotes then split by comma and strip quotes/space
const parts = raw
.replace(/^\[|\]$/g, "")
const parts = val
.split(",")
.map((s) => s.replace(/^["']|["']$/g, "").trim())
.map((s) => s.trim())
.filter(Boolean);
clone[key] = parts.length ? parts : ["ROLE_USER"];
@@ -304,7 +280,6 @@ export class Engine {
}
}
// compute credibility JSON if bias/reliability/transparency present
if (t === "article" || t === "source") {
const bias = clone.bias ?? null;
const reliability = clone.reliability ?? null;
@@ -318,14 +293,10 @@ export class Engine {
}
}
// Ensure article token_statistics exists (computed on the fly)
if (
t === "article" &&
(clone.token_statistics == null || typeof clone.token_statistics !== "object")
) {
if (t === "article") {
clone.token_statistics = computeTokenStatistics({
body: String(clone.body ?? ""),
categories: Array.isArray(clone.categories) ? (clone.categories as string[]) : [],
categories: Array.isArray(clone.categories) ? clone.categories : [],
title: String(clone.title ?? ""),
});
}
@@ -352,7 +323,7 @@ export class Engine {
return JSON.stringify(v);
}
if (col === "roles" && v) {
return JSON.stringify(v);
return v;
}
if (col === "metadata" && v && typeof v === "object") {
return JSON.stringify(v);
packages/db/src/importer/import.ts
+2 -2
View File
@@ -14,7 +14,7 @@ const env = createEnvAccessor([
"BASANGO_DATABASE_URL",
]);
async function promptConfirm(question: string, def = false) {
async function askConfirmation(question: string, def = false) {
const rl = createInterface({ input, output });
const suffix = def ? "[Y/n]" : "[y/N]";
const answer = await rl.question(`${question} ${suffix} `);
@@ -28,7 +28,7 @@ async function promptConfirm(question: string, def = false) {
}
async function main() {
const ok = await promptConfirm("Do you want to continue?", false);
const ok = await askConfirmation("Do you want to continue?", false);
if (!ok) {
console.warn("Process aborted");
process.exit(1);
packages/db/src/schema.ts
-6
View File
@@ -101,12 +101,6 @@ export type GeoLocation = {
accuracyRadius?: number;
};
export type ClientProfile = {
userIp?: string;
userAgent?: string;
hints: unknown[];
};
export type ArticleMetadata = {
title?: string;
description?: string;
packages/db/src/utils/api-keys.ts
-20
View File
@@ -1,20 +0,0 @@
import { randomBytes } from "node:crypto";
/**
* Generates a new API key with the format mid_{random_string}
* @returns A new API key string
*/
export function generateApiKey(): string {
// Generate 32 random bytes and convert to hex
const randomString = randomBytes(32).toString("hex");
return `basango_${randomString}`;
}
/**
* Validates if a string is a valid API key format
* @param key The key to validate
* @returns True if the key starts with 'basango_' and has the correct length
*/
export function isValidApiKeyFormat(key: string): boolean {
return key.startsWith("basango_") && key.length === 68; // basango_ (8) + 64 hex chars
}
packages/db/src/utils/computed.ts
+6 -4
View File
@@ -31,10 +31,12 @@ export const computeTokenStatistics = (data: {
body: string;
categories: string[];
}): TokenStatistics => {
const title = computeTokenCount(data.title);
const body = computeTokenCount(data.body);
const categories = computeTokenCount(data.categories.join(","));
const excerpt = computeTokenCount(data.body.substring(0, 200));
const [title, body, categories, excerpt] = [
computeTokenCount(data.title),
computeTokenCount(data.body),
computeTokenCount(data.categories.join(",")),
computeTokenCount(data.body.substring(0, 200)),
];
return {
body,
packages/db/src/utils/health.ts
-7
View File
@@ -1,7 +0,0 @@
import { sql } from "drizzle-orm";
import { db } from "@/client";
export async function checkHealth() {
await db.execute(sql`SELECT 1`);
}
packages/db/src/utils/index.ts
+1 -2
View File
@@ -1,4 +1,3 @@
export * from "./api-keys";
export * from "./health";
export * from "./computed";
export * from "./pagination";
export * from "./search-query";