feat(db): migration and database setup
This commit is contained in:
@@ -1,4 +1,6 @@
|
||||
NODE_ENV=development
|
||||
BASANGO_API_HOST=localhost
|
||||
BASANGO_API_PORT=3000
|
||||
BASANGO_API_ALLOWED_ORIGINS=http://localhost:3000,http://127.0.0.1:3000
|
||||
BASANGO_API_KEY=your_api_key_here
|
||||
BASANGO_CRAWLER_KEY=dev
|
||||
|
||||
@@ -27,6 +27,7 @@ export const { env, config } = defineConfig({
|
||||
"BASANGO_API_PORT",
|
||||
"BASANGO_API_ALLOWED_ORIGINS",
|
||||
"BASANGO_API_KEY",
|
||||
"BASANGO_CRAWLER_KEY",
|
||||
],
|
||||
path: path.join(PROJECT_DIR, ".env"),
|
||||
},
|
||||
|
||||
+3
-18
@@ -4,7 +4,7 @@ import { cors } from "hono/cors";
|
||||
import { secureHeaders } from "hono/secure-headers";
|
||||
|
||||
import { config, env } from "@/config";
|
||||
import { checkHealth } from "@/utils/health";
|
||||
import { routers } from "@/rest/routers";
|
||||
|
||||
const app = new OpenAPIHono();
|
||||
|
||||
@@ -21,26 +21,10 @@ app.use(
|
||||
}),
|
||||
);
|
||||
|
||||
app.get("/health", async (c) => {
|
||||
try {
|
||||
await checkHealth();
|
||||
|
||||
return c.json({ status: "ok" }, 200);
|
||||
} catch (error) {
|
||||
return c.json(
|
||||
{
|
||||
message: error instanceof Error ? error.message : "Unknown error",
|
||||
status: "error",
|
||||
},
|
||||
500,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
app.doc("/openapi", {
|
||||
info: {
|
||||
contact: {
|
||||
email: "engineer@basango.io",
|
||||
email: "engineering@basango.io",
|
||||
name: "Basango",
|
||||
url: "https://basango.io",
|
||||
},
|
||||
@@ -76,6 +60,7 @@ app.openAPIRegistry.registerComponent("securitySchemes", "token", {
|
||||
});
|
||||
|
||||
app.get("/", Scalar({ pageTitle: "Basango API", theme: "saturn", url: "/openapi" }));
|
||||
app.route("/", routers);
|
||||
|
||||
export default {
|
||||
fetch: app.fetch,
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
import type { MiddlewareHandler } from "hono";
|
||||
import { HTTPException } from "hono/http-exception";
|
||||
|
||||
import { env } from "@/config";
|
||||
|
||||
export const withCrawlerAuth: MiddlewareHandler = async (c, next) => {
|
||||
const token = c.req.header("Authorization");
|
||||
|
||||
if (!token) {
|
||||
throw new HTTPException(401, { message: "Authorization header required" });
|
||||
}
|
||||
|
||||
if (token !== env("BASANGO_CRAWLER_KEY")) {
|
||||
throw new HTTPException(403, { message: "Invalid token" });
|
||||
}
|
||||
|
||||
await next();
|
||||
};
|
||||
@@ -0,0 +1,8 @@
|
||||
import { db } from "@basango/db/client";
|
||||
import type { MiddlewareHandler } from "hono";
|
||||
|
||||
export const withDatabase: MiddlewareHandler = async (c, next) => {
|
||||
c.set("db", db);
|
||||
|
||||
await next();
|
||||
};
|
||||
@@ -0,0 +1,36 @@
|
||||
import type { MiddlewareHandler } from "hono";
|
||||
|
||||
import type { Scope } from "@/utils/scopes";
|
||||
|
||||
export const withRequiredScope = (...requiredScopes: Scope[]): MiddlewareHandler => {
|
||||
return async (c, next) => {
|
||||
const scopes = c.get("scopes") as Scope[] | undefined;
|
||||
|
||||
if (!scopes) {
|
||||
return c.json(
|
||||
{
|
||||
description: "No scopes found for the current user. Authentication is required.",
|
||||
error: "Unauthorized",
|
||||
},
|
||||
401,
|
||||
);
|
||||
}
|
||||
|
||||
// Check if user has at least one of the required scopes
|
||||
const hasRequiredScope = requiredScopes.some((requiredScope) => scopes.includes(requiredScope));
|
||||
|
||||
if (!hasRequiredScope) {
|
||||
return c.json(
|
||||
{
|
||||
description: `Insufficient permissions. Required scopes: ${requiredScopes.join(
|
||||
", ",
|
||||
)}. Your scopes: ${scopes.join(", ")}`,
|
||||
error: "Forbidden",
|
||||
},
|
||||
403,
|
||||
);
|
||||
}
|
||||
|
||||
await next();
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,53 @@
|
||||
import { createArticle } from "@basango/db/queries";
|
||||
import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
|
||||
|
||||
import { withCrawlerAuth } from "@/rest/middlewares/crawler";
|
||||
import type { Context } from "@/rest/types";
|
||||
import { createArticleResponseSchema, createArticleSchema } from "@/schemas/articles";
|
||||
import { validateResponse } from "@/utils/response";
|
||||
|
||||
const app = new OpenAPIHono<Context>();
|
||||
|
||||
app.openapi(
|
||||
createRoute({
|
||||
description: "Store a new crawled article in the database.",
|
||||
method: "post",
|
||||
middleware: [withCrawlerAuth],
|
||||
operationId: "CreateArticle",
|
||||
path: "/",
|
||||
request: {
|
||||
body: {
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: createArticleSchema,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
responses: {
|
||||
201: {
|
||||
content: {
|
||||
"application/json": {
|
||||
schema: createArticleResponseSchema,
|
||||
},
|
||||
},
|
||||
description: "Article created",
|
||||
},
|
||||
},
|
||||
summary: "Create Article",
|
||||
tags: ["Articles"],
|
||||
"x-speakeasy-name-override": "create",
|
||||
}),
|
||||
async (c) => {
|
||||
const db = c.get("db");
|
||||
const body = c.req.valid("json");
|
||||
const result = await createArticle(db, { ...body });
|
||||
|
||||
return c.json(
|
||||
validateResponse(result, createArticleResponseSchema) as { id: string; sourceId: string },
|
||||
201,
|
||||
);
|
||||
},
|
||||
);
|
||||
|
||||
export const articlesRouter = app;
|
||||
@@ -0,0 +1,9 @@
|
||||
import { OpenAPIHono } from "@hono/zod-openapi";
|
||||
|
||||
import { articlesRouter } from "@/rest/routers/articles";
|
||||
|
||||
const routers = new OpenAPIHono();
|
||||
|
||||
routers.route("/articles", articlesRouter);
|
||||
|
||||
export { routers };
|
||||
@@ -0,0 +1,7 @@
|
||||
import type { Database } from "@basango/db/client";
|
||||
|
||||
export type Context = {
|
||||
Variables: {
|
||||
db: Database;
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,99 @@
|
||||
import { z } from "@hono/zod-openapi";
|
||||
|
||||
const sentimentSchema = z.enum(["positive", "neutral", "negative"]).openapi({
|
||||
default: "neutral",
|
||||
description: "The sentiment of the article content.",
|
||||
});
|
||||
|
||||
const readingTimeSchema = z.number().min(1).openapi({
|
||||
description: "The estimated reading time of the article in minutes.",
|
||||
example: 5,
|
||||
});
|
||||
|
||||
const tokenStatisticsSchema = z.object({
|
||||
body: z.number().min(0).openapi({
|
||||
description: "The number of tokens in the article body.",
|
||||
example: 350,
|
||||
}),
|
||||
categories: z.number().min(0).openapi({
|
||||
description: "The number of tokens in the article categories.",
|
||||
example: 25,
|
||||
}),
|
||||
excerpt: z.number().min(0).openapi({
|
||||
description: "The number of tokens in the article excerpt.",
|
||||
example: 50,
|
||||
}),
|
||||
title: z.number().min(0).openapi({
|
||||
description: "The number of tokens in the article title.",
|
||||
example: 15,
|
||||
}),
|
||||
total: z.number().min(0).openapi({
|
||||
description: "The total number of tokens in the article.",
|
||||
example: 440,
|
||||
}),
|
||||
});
|
||||
|
||||
const metadataSchema = z.object({
|
||||
description: z.string().optional().openapi({
|
||||
description: "A brief description or summary of the article.",
|
||||
example: "This article discusses the latest advancements in AI technology.",
|
||||
}),
|
||||
image: z.url().optional().openapi({
|
||||
description: "The URL of the main image associated with the article.",
|
||||
example: "https://example.com/image.jpg",
|
||||
}),
|
||||
title: z.string().optional().openapi({
|
||||
description: "The title of the article for metadata purposes.",
|
||||
example: "The Rise of AI",
|
||||
}),
|
||||
});
|
||||
|
||||
export const createArticleSchema = z
|
||||
.object({
|
||||
body: z.string().min(1).openapi({
|
||||
description: "The main content of the article.",
|
||||
example: "This is the body of the article...",
|
||||
}),
|
||||
categories: z.array(z.string()).openapi({
|
||||
description: "The categories or tags associated with the article.",
|
||||
example: ["Technology", "AI"],
|
||||
}),
|
||||
hash: z.string().min(1).openapi({
|
||||
description: "The unique hash of the article link.",
|
||||
example: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
}),
|
||||
link: z.url().openapi({
|
||||
description: "The URL of the article.",
|
||||
example: "https://example.com/article",
|
||||
}),
|
||||
metadata: metadataSchema.optional(),
|
||||
publishedAt: z.date().openapi({
|
||||
description: "The publication date of the article.",
|
||||
example: "2023-01-01T00:00:00Z",
|
||||
}),
|
||||
readingTime: readingTimeSchema.optional(),
|
||||
sentiment: sentimentSchema.optional().optional().default("neutral"),
|
||||
sourceId: z.string().openapi({
|
||||
description: "The unique identifier of the source from which the article was crawled.",
|
||||
example: "source-123",
|
||||
}),
|
||||
title: z.string().min(1).openapi({
|
||||
description: "The title of the article.",
|
||||
example: "The Rise of AI",
|
||||
}),
|
||||
tokenStatistics: tokenStatisticsSchema.optional(),
|
||||
})
|
||||
.openapi("CreateArticle");
|
||||
|
||||
export const createArticleResponseSchema = z
|
||||
.object({
|
||||
id: z.uuid().openapi({
|
||||
description: "The unique identifier of the article.",
|
||||
example: "b3b7c8e2-1f2a-4c3d-9e4f-5a6b7c8d9e0f",
|
||||
}),
|
||||
sourceId: z.uuid().openapi({
|
||||
description: "The unique identifier of the source associated with the article.",
|
||||
example: "a1a2b3c4-d5e6-7f8g-9h0i-j1k2l3m4n5o6",
|
||||
}),
|
||||
})
|
||||
.openapi("CreateArticleResponse");
|
||||
@@ -1,5 +0,0 @@
|
||||
import { checkHealth as checkDbHealth } from "@basango/db";
|
||||
|
||||
export async function checkHealth(): Promise<void> {
|
||||
await checkDbHealth();
|
||||
}
|
||||
@@ -0,0 +1,66 @@
|
||||
export const SCOPES = [
|
||||
"articles.read",
|
||||
"articles.write",
|
||||
"apis.all", // All API scopes
|
||||
"apis.read", // All read scopes
|
||||
] as const;
|
||||
|
||||
export type Scope = (typeof SCOPES)[number];
|
||||
export type ScopePreset = "all_access" | "read_only" | "restricted";
|
||||
|
||||
export const scopePresets = [
|
||||
{
|
||||
description: "full access to all resources",
|
||||
label: "All",
|
||||
value: "all_access",
|
||||
},
|
||||
{
|
||||
description: "read-only access to all resources",
|
||||
label: "Read Only",
|
||||
value: "read_only",
|
||||
},
|
||||
{
|
||||
description: "restricted access to some resources",
|
||||
label: "Restricted",
|
||||
value: "restricted",
|
||||
},
|
||||
];
|
||||
|
||||
export const scopesToName = (scopes: string[]) => {
|
||||
if (scopes.includes("apis.all")) {
|
||||
return {
|
||||
description: "full access to all resources",
|
||||
name: "All access",
|
||||
preset: "all_access",
|
||||
};
|
||||
}
|
||||
|
||||
if (scopes.includes("apis.read")) {
|
||||
return {
|
||||
description: "read-only access to all resources",
|
||||
name: "Read-only",
|
||||
preset: "read_only",
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
description: "restricted access to some resources",
|
||||
name: "Restricted",
|
||||
preset: "restricted",
|
||||
};
|
||||
};
|
||||
|
||||
export const expandScopes = (scopes: string[]): string[] => {
|
||||
if (scopes.includes("apis.all")) {
|
||||
// Return all scopes except any that start with "apis."
|
||||
return SCOPES.filter((scope) => !scope.startsWith("apis."));
|
||||
}
|
||||
|
||||
if (scopes.includes("apis.read")) {
|
||||
// Return all read scopes except any that start with "apis."
|
||||
return SCOPES.filter((scope) => scope.endsWith(".read") && !scope.startsWith("apis."));
|
||||
}
|
||||
|
||||
// For custom scopes, filter out any "apis." scopes
|
||||
return scopes.filter((scope) => !scope.startsWith("apis."));
|
||||
};
|
||||
@@ -111,11 +111,15 @@
|
||||
"packages/db": {
|
||||
"name": "@basango/db",
|
||||
"dependencies": {
|
||||
"@basango/encryption": "workspace:*",
|
||||
"@basango/logger": "workspace:*",
|
||||
"@date-fns/utc": "^2.1.1",
|
||||
"drizzle-orm": "^0.44.7",
|
||||
"mysql2": "^3.15.3",
|
||||
"pg": "^8.16.3",
|
||||
"snakecase-keys": "^9.0.2",
|
||||
"tiktoken": "^1.0.22",
|
||||
"uuid": "^13.0.0",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/pg": "^8.15.6",
|
||||
@@ -1028,6 +1032,8 @@
|
||||
|
||||
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
|
||||
|
||||
"aws-ssl-profiles": ["aws-ssl-profiles@1.1.2", "", {}, "sha512-NZKeq9AfyQvEeNlN0zSYAaWrmBffJh3IELMZfRpJVWgrpEbtEpnjvzqBPf+mxoI287JohRDoa+/nsfqqiZmF6g=="],
|
||||
|
||||
"babel-jest": ["babel-jest@29.7.0", "", { "dependencies": { "@jest/transform": "^29.7.0", "@types/babel__core": "^7.1.14", "babel-plugin-istanbul": "^6.1.1", "babel-preset-jest": "^29.6.3", "chalk": "^4.0.0", "graceful-fs": "^4.2.9", "slash": "^3.0.0" }, "peerDependencies": { "@babel/core": "^7.8.0" } }, "sha512-BrvGY3xZSwEcCzKvKsCi2GgHqDqsYkOP4/by5xCgIwGXQxIEh+8ew3gmrE1y7XRR6LHZIj6yLYnUi/mm2KXKBg=="],
|
||||
|
||||
"babel-plugin-istanbul": ["babel-plugin-istanbul@6.1.1", "", { "dependencies": { "@babel/helper-plugin-utils": "^7.0.0", "@istanbuljs/load-nyc-config": "^1.0.0", "@istanbuljs/schema": "^0.1.2", "istanbul-lib-instrument": "^5.0.4", "test-exclude": "^6.0.0" } }, "sha512-Y1IQok9821cC9onCx5otgFfRm7Lm+I+wwxOx738M/WLPZ9Q42m4IG5W0FNX8WLL2gYMZo3JkuXIH2DOpWM+qwA=="],
|
||||
@@ -1426,6 +1432,8 @@
|
||||
|
||||
"function-bind": ["function-bind@1.1.2", "", {}, "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA=="],
|
||||
|
||||
"generate-function": ["generate-function@2.3.1", "", { "dependencies": { "is-property": "^1.0.2" } }, "sha512-eeB5GfMNeevm/GRYq20ShmsaGcmI81kIX2K9XQx5miC8KdHaC6Jm0qQ8ZNeGOi7wYB8OsdxKs+Y2oVuTFuVwKQ=="],
|
||||
|
||||
"gensync": ["gensync@1.0.0-beta.2", "", {}, "sha512-3hN7NaskYvMDLQY55gnW3NQ+mesEAepTqlg+VEbj7zzqEMBVNhzcGYYeqFo/TlYz6eQiFcp1HcsCZO+nGgS8zg=="],
|
||||
|
||||
"get-caller-file": ["get-caller-file@2.0.5", "", {}, "sha512-DyFP3BM/3YHTQOCUL/w0OZHR0lpKeGrxotcHWcqNEdnltqFwXVfhEBQ94eIo34AfQpo0rGki4cyIiftY06h2Fg=="],
|
||||
@@ -1500,7 +1508,7 @@
|
||||
|
||||
"hyphenate-style-name": ["hyphenate-style-name@1.1.0", "", {}, "sha512-WDC/ui2VVRrz3jOVi+XtjqkDjiVjTtFaAGiW37k6b+ohyQ5wYDOGkvCZa8+H0nx3gyvv0+BST9xuOgIyGQ00gw=="],
|
||||
|
||||
"iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="],
|
||||
"iconv-lite": ["iconv-lite@0.7.0", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3.0.0" } }, "sha512-cf6L2Ds3h57VVmkZe+Pn+5APsT7FpqJtEhhieDCvrE2MK5Qk9MyffgQyuxQTm6BChfeZNtcOLHp9IcWRVcIcBQ=="],
|
||||
|
||||
"ieee754": ["ieee754@1.2.1", "", {}, "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA=="],
|
||||
|
||||
@@ -1556,6 +1564,8 @@
|
||||
|
||||
"is-path-inside": ["is-path-inside@3.0.3", "", {}, "sha512-Fd4gABb+ycGAmKou8eMftCupSir5lRxqf4aD/vd0cD2qc4HL07OjCeuHMr8Ro4CoMaeCKDB0/ECBOVWjTwUvPQ=="],
|
||||
|
||||
"is-property": ["is-property@1.0.2", "", {}, "sha512-Ks/IoX00TtClbGQr4TWXemAnktAQvYB7HzcCxDGqEZU6oCmb2INHuOoKxbtR+HFkmYWBKv/dOZtGRiAjDhj92g=="],
|
||||
|
||||
"is-stream": ["is-stream@2.0.1", "", {}, "sha512-hFoiJiTl63nn+kstHGBtewWSKnQLpyb155KHheA1l39uvtO9nWIop1p3udqPcUd/xbF1VLMO4n7OI6p7RbngDg=="],
|
||||
|
||||
"is-text-path": ["is-text-path@2.0.0", "", { "dependencies": { "text-extensions": "^2.0.0" } }, "sha512-+oDTluR6WEjdXEJMnC2z6A4FRwFoYuvShVVEGsS7ewc0UTi2QtAKMDJuL4BDEVt+5T7MjFo12RP8ghOM75oKJw=="],
|
||||
@@ -1700,6 +1710,8 @@
|
||||
|
||||
"log-symbols": ["log-symbols@4.1.0", "", { "dependencies": { "chalk": "^4.1.0", "is-unicode-supported": "^0.1.0" } }, "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg=="],
|
||||
|
||||
"long": ["long@5.3.2", "", {}, "sha512-mNAgZ1GmyNhD7AuqnTG3/VQ26o760+ZYBPKjPvugO8+nLbYfX6TVpJPseBvopbdY+qpZ/lKUnmEc1LeZYS3QAA=="],
|
||||
|
||||
"longest": ["longest@2.0.1", "", {}, "sha512-Ajzxb8CM6WAnFjgiloPsI3bF+WCxcvhdIG3KNA2KN962+tdBsHcuQ4k4qX/EcS/2CRkcc0iAkR956Nib6aXU/Q=="],
|
||||
|
||||
"loose-envify": ["loose-envify@1.4.0", "", { "dependencies": { "js-tokens": "^3.0.0 || ^4.0.0" }, "bin": { "loose-envify": "cli.js" } }, "sha512-lyuxPGr/Wfhrlem2CL/UcnUc1zcqKAImBDzukY7Y5F/yQiNdko6+fRLevlw1HgMySw7f611UIY408EtxRSoK3Q=="],
|
||||
@@ -1710,6 +1722,8 @@
|
||||
|
||||
"lru-cache": ["lru-cache@7.18.3", "", {}, "sha512-jumlc0BIUrS3qJGgIkWZsyfAM7NCWiBcCDhnd+3NNM5KbBmLTgHVfWBcg6W+rLUsIpzpERPsvwUP7CckAQSOoA=="],
|
||||
|
||||
"lru.min": ["lru.min@1.1.2", "", {}, "sha512-Nv9KddBcQSlQopmBHXSsZVY5xsdlZkdH/Iey0BlcBYggMd4two7cZnKOK9vmy3nY0O5RGH99z1PCeTpPqszUYg=="],
|
||||
|
||||
"lucide-react": ["lucide-react@0.475.0", "", { "peerDependencies": { "react": "^16.5.1 || ^17.0.0 || ^18.0.0 || ^19.0.0" } }, "sha512-NJzvVu1HwFVeZ+Gwq2q00KygM1aBhy/ZrhY9FsAgJtpB+E4R7uxRk9M2iKvHa6/vNxZydIB59htha4c2vvwvVg=="],
|
||||
|
||||
"luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="],
|
||||
@@ -1790,8 +1804,12 @@
|
||||
|
||||
"mute-stream": ["mute-stream@0.0.8", "", {}, "sha512-nnbWWOkoWyUsTjKrhgD0dcz22mdkSnpYqbEjIm2nhwhuxlSkpywJmBo8h0ZqJdkp73mb90SssHkN4rsRaBAfAA=="],
|
||||
|
||||
"mysql2": ["mysql2@3.15.3", "", { "dependencies": { "aws-ssl-profiles": "^1.1.1", "denque": "^2.1.0", "generate-function": "^2.3.1", "iconv-lite": "^0.7.0", "long": "^5.2.1", "lru.min": "^1.0.0", "named-placeholders": "^1.1.3", "seq-queue": "^0.0.5", "sqlstring": "^2.3.2" } }, "sha512-FBrGau0IXmuqg4haEZRBfHNWB5mUARw6hNwPDXXGg0XzVJ50mr/9hb267lvpVMnhZ1FON3qNd4Xfcez1rbFwSg=="],
|
||||
|
||||
"mz": ["mz@2.7.0", "", { "dependencies": { "any-promise": "^1.0.0", "object-assign": "^4.0.1", "thenify-all": "^1.0.0" } }, "sha512-z81GNO7nnYMEhrGh9LeymoE4+Yr0Wn5McHIZMK5cfQCl+NDX08sCZgUc9/6MHni9IWuFLm1Z3HTCXu2z9fN62Q=="],
|
||||
|
||||
"named-placeholders": ["named-placeholders@1.1.3", "", { "dependencies": { "lru-cache": "^7.14.1" } }, "sha512-eLoBxg6wE/rZkJPhU/xRX1WTpkFEwDJEN96oxFrTsqBdbT5ec295Q+CoHrL9IT0DipqKhmGcaZmwOt8OON5x1w=="],
|
||||
|
||||
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||
|
||||
"negotiator": ["negotiator@0.6.3", "", {}, "sha512-+EUsqGPLsM+j/zdChZjsnX51g4XrHFOIXwfnCVPGlQk/k5giakcKsuxCObBRu6DSm9opw/O6slWbJdghQM4bBg=="],
|
||||
@@ -2104,6 +2122,8 @@
|
||||
|
||||
"sentence-case": ["sentence-case@2.1.1", "", { "dependencies": { "no-case": "^2.2.0", "upper-case-first": "^1.1.2" } }, "sha512-ENl7cYHaK/Ktwk5OTD+aDbQ3uC8IByu/6Bkg+HDv8Mm+XnBnppVNalcfJTNsp1ibstKh030/JKQQWglDvtKwEQ=="],
|
||||
|
||||
"seq-queue": ["seq-queue@0.0.5", "", {}, "sha512-hr3Wtp/GZIc/6DAGPDcV4/9WoZhjrkXsi5B/07QgX8tsdc6ilr7BFM6PM6rbdAX1kFSDYeZGLipIZZKyQP0O5Q=="],
|
||||
|
||||
"serialize-error": ["serialize-error@2.1.0", "", {}, "sha512-ghgmKt5o4Tly5yEG/UJp8qTd0AN7Xalw4XBtDEKP655B699qMEtra1WlXeE6WIvdEG481JvRxULKsInq/iNysw=="],
|
||||
|
||||
"serve-static": ["serve-static@1.16.2", "", { "dependencies": { "encodeurl": "~2.0.0", "escape-html": "~1.0.3", "parseurl": "~1.3.3", "send": "0.19.0" } }, "sha512-VqpjJZKadQB/PEbEwvFdO43Ax5dFBZ2UECszz8bQ7pi7wt//PWe1P6MN7eCnjsatYtBT6EuiClbjSWP2WrIoTw=="],
|
||||
@@ -2164,6 +2184,8 @@
|
||||
|
||||
"sprintf-js": ["sprintf-js@1.0.3", "", {}, "sha512-D9cPgkvLlV3t3IzL0D0YLvGA9Ahk4PcvVwUbN0dSGr1aP0Nrt4AEnTUbuGvquEC0mA64Gqt1fzirlRs5ibXx8g=="],
|
||||
|
||||
"sqlstring": ["sqlstring@2.3.3", "", {}, "sha512-qC9iz2FlN7DQl3+wjwn3802RTyjCx7sDvfQEXchwa6CWOx07/WVfh91gBmQ9fahw8snwGEWU3xGzOt4tFyHLxg=="],
|
||||
|
||||
"stack-utils": ["stack-utils@2.0.6", "", { "dependencies": { "escape-string-regexp": "^2.0.0" } }, "sha512-XlkWvfIm6RmsWtNJx+uqtKLS8eqFbxUg0ZzLXqY0caEy9l7hruX8IpiDnjsLavoBgqCCR71TqWO8MaXYheJ3RQ=="],
|
||||
|
||||
"stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="],
|
||||
@@ -2346,7 +2368,7 @@
|
||||
|
||||
"utils-merge": ["utils-merge@1.0.1", "", {}, "sha512-pMZTvIkT1d+TFGvDOqodOclx0QWkkgi6Tdoa8gC8ffGAAqz9pzPTZWAybbsHHoED/ztMtkv/VoYTYyShUn81hA=="],
|
||||
|
||||
"uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
|
||||
"uuid": ["uuid@13.0.0", "", { "bin": { "uuid": "dist-node/bin/uuid" } }, "sha512-XQegIaBTVUjSHliKqcnFqYypAd4S+WCYt5NIeRs6w/UAry7z8Y9j5ZwRRL4kzq9U3sD6v+85er9FvkEaBpji2w=="],
|
||||
|
||||
"v8-compile-cache-lib": ["v8-compile-cache-lib@3.0.1", "", {}, "sha512-wa7YjyUGfNZngI/vtK0UHAN+lgDCxBPCylVXGp0zu59Fz5aiGtNXaq3DhIov063MorB+VfufLh3JlF2KdTK3xg=="],
|
||||
|
||||
@@ -2590,6 +2612,8 @@
|
||||
|
||||
"bullmq/glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="],
|
||||
|
||||
"bullmq/uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
|
||||
|
||||
"chalk/escape-string-regexp": ["escape-string-regexp@1.0.5", "", {}, "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg=="],
|
||||
|
||||
"chromium-edge-launcher/mkdirp": ["mkdirp@1.0.4", "", { "bin": { "mkdirp": "bin/cmd.js" } }, "sha512-vVqVZQyf3WLx2Shd0qJ9xuvqgAyKPLAiqITEtqW0oIUjzo3PePDd6fW9iFz30ef7Ysp/oiWqbhszeGWW2T6Gzw=="],
|
||||
@@ -2616,6 +2640,8 @@
|
||||
|
||||
"expo-router/semver": ["semver@7.6.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A=="],
|
||||
|
||||
"external-editor/iconv-lite": ["iconv-lite@0.4.24", "", { "dependencies": { "safer-buffer": ">= 2.1.2 < 3" } }, "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA=="],
|
||||
|
||||
"fbjs/promise": ["promise@7.3.1", "", { "dependencies": { "asap": "~2.0.3" } }, "sha512-nolQXZ/4L+bP/UGlkfaIujX9BKxGwmQ9OT4mOt5yvy8iK1h3wqTEJCijzGANTCCl9nWjY41juyAn2K3Q1hLLTg=="],
|
||||
|
||||
"figures/escape-string-regexp": ["escape-string-regexp@1.0.5", "", {}, "sha512-vbRorB5FUQWvla16U8R/qgaFIya2qGzwDrNmCZuYKrbdSUMG6I1ZCGQRefkRVhuOkIGVne7BQ35DSfo1qvJqFg=="],
|
||||
|
||||
+6
-1
@@ -1 +1,6 @@
|
||||
BASANGO_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/app?serverVersion=16&charset=utf8"
|
||||
BASANGO_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/app?serverVersion=16&charset=utf8"
|
||||
|
||||
BASANGO_SOURCE_DATABASE_HOST="localhost"
|
||||
BASANGO_SOURCE_DATABASE_PASS="root"
|
||||
BASANGO_SOURCE_DATABASE_NAME="app"
|
||||
BASANGO_SOURCE_DATABASE_USER="root"
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
-- Current sql file was generated after introspecting the database
|
||||
-- If you want to run this migration please uncomment this code before executing migrations
|
||||
/*
|
||||
CREATE SEQUENCE "public"."refresh_tokens_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1;--> statement-breakpoint
|
||||
CREATE TABLE "doctrine_migration_versions" (
|
||||
"version" varchar(191) PRIMARY KEY NOT NULL,
|
||||
"executed_at" timestamp(0) DEFAULT NULL,
|
||||
"execution_time" integer
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"description" varchar(512) DEFAULT NULL,
|
||||
"is_public" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_attempt" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_history" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"ip_address" "inet",
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"device_operating_system" varchar(255) DEFAULT NULL,
|
||||
"device_client" varchar(255) DEFAULT NULL,
|
||||
"device_device" varchar(255) DEFAULT NULL,
|
||||
"device_is_bot" boolean DEFAULT false NOT NULL,
|
||||
"location_time_zone" varchar(255) DEFAULT NULL,
|
||||
"location_longitude" double precision,
|
||||
"location_latitude" double precision,
|
||||
"location_accuracy_radius" integer
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "verification_token" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"purpose" varchar(255) NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"token" varchar(60) DEFAULT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "followed_source" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"follower_id" uuid NOT NULL,
|
||||
"source_id" uuid NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "comment" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"article_id" uuid NOT NULL,
|
||||
"content" varchar(512) NOT NULL,
|
||||
"sentiment" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"is_spam" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "refresh_tokens" (
|
||||
"id" integer PRIMARY KEY NOT NULL,
|
||||
"refresh_token" varchar(128) NOT NULL,
|
||||
"username" varchar(255) NOT NULL,
|
||||
"valid" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "article" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"source_id" uuid NOT NULL,
|
||||
"title" varchar(1024) NOT NULL,
|
||||
"body" text NOT NULL,
|
||||
"hash" varchar(32) NOT NULL,
|
||||
"categories" text[],
|
||||
"sentiment" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"metadata" jsonb,
|
||||
"image" varchar(1024) GENERATED ALWAYS AS ((metadata ->> 'image'::text)) STORED,
|
||||
"excerpt" varchar(255) GENERATED ALWAYS AS (("left"(body, 200) || '...'::text)) STORED,
|
||||
"published_at" timestamp(0) NOT NULL,
|
||||
"crawled_at" timestamp(0) NOT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL,
|
||||
"link" varchar(1024) NOT NULL,
|
||||
"bias" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"reliability" varchar(30) DEFAULT 'reliable' NOT NULL,
|
||||
"transparency" varchar(30) DEFAULT 'medium' NOT NULL,
|
||||
"reading_time" integer DEFAULT 1,
|
||||
"tsv" "tsvector" GENERATED ALWAYS AS ((setweight(to_tsvector('french'::regconfig, (COALESCE(title, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char"))) STORED,
|
||||
"token_statistics" jsonb,
|
||||
CONSTRAINT "chk_article_reading_time" CHECK (reading_time >= 0),
|
||||
CONSTRAINT "chk_article_sentiment" CHECK ((sentiment)::text = ANY ((ARRAY['positive'::character varying, 'neutral'::character varying, 'negative'::character varying])::text[])),
|
||||
CONSTRAINT "chk_article_metadata_json" CHECK ((metadata IS NULL) OR (jsonb_typeof(metadata) = ANY (ARRAY['object'::text, 'array'::text])))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "user" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"email" varchar(255) NOT NULL,
|
||||
"password" varchar(512) NOT NULL,
|
||||
"is_locked" boolean DEFAULT false NOT NULL,
|
||||
"is_confirmed" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL,
|
||||
"roles" jsonb NOT NULL,
|
||||
CONSTRAINT "chk_user_roles_json" CHECK (jsonb_typeof(roles) = 'array'::text)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "source" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"url" varchar(255) NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"display_name" varchar(255) DEFAULT NULL,
|
||||
"description" varchar(1024) DEFAULT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL,
|
||||
"bias" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"reliability" varchar(30) DEFAULT 'reliable' NOT NULL,
|
||||
"transparency" varchar(30) DEFAULT 'medium' NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark_article" (
|
||||
"bookmark_id" uuid NOT NULL,
|
||||
"article_id" uuid NOT NULL,
|
||||
CONSTRAINT "bookmark_article_pkey" PRIMARY KEY("bookmark_id","article_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "bookmark" ADD CONSTRAINT "fk_da62921da76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_attempt" ADD CONSTRAINT "fk_8c11c1ba76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_history" ADD CONSTRAINT "fk_37976e36a76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "verification_token" ADD CONSTRAINT "fk_c1cc006ba76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_7a763a3eac24f853" FOREIGN KEY ("follower_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_7a763a3e953c1c61" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_9474526ca76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_9474526c7294869c" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "article" ADD CONSTRAINT "fk_23a0e66953c1c61" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_6fe2655d92741d25" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmark"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_6fe2655d7294869c" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "idx_bookmark_user_created" ON "bookmark" USING btree ("user_id" timestamp_ops,"created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_da62921da76ed395" ON "bookmark" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_8c11c1ba76ed395" ON "login_attempt" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_attempt_created_at" ON "login_attempt" USING btree ("created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_37976e36a76ed395" ON "login_history" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_created_at" ON "login_history" USING btree ("user_id" uuid_ops,"created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_ip_address" ON "login_history" USING btree ("ip_address" inet_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_c1cc006ba76ed395" ON "verification_token" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_verif_token_created_at" ON "verification_token" USING btree ("created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_verif_user_purpose_token" ON "verification_token" USING btree ("user_id" text_ops,"purpose" text_ops) WHERE (token IS NOT NULL);--> statement-breakpoint
|
||||
CREATE INDEX "idx_7a763a3e953c1c61" ON "followed_source" USING btree ("source_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_7a763a3eac24f853" ON "followed_source" USING btree ("follower_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_follower_created" ON "followed_source" USING btree ("follower_id" timestamp_ops,"created_at" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_9474526c7294869c" ON "comment" USING btree ("article_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_9474526ca76ed395" ON "comment" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_article_created" ON "comment" USING btree ("article_id" timestamp_ops,"created_at" uuid_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "uniq_9bace7e1c74f2195" ON "refresh_tokens" USING btree ("refresh_token" text_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_categories" ON "article" USING gin ("categories" array_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_link_trgm" ON "article" USING gin ("link" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_title_trgm" ON "article" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_tsv" ON "article" USING gin ("tsv" tsvector_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_23a0e66953c1c61" ON "article" USING btree ("source_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_article_published_at" ON "article" USING btree ("published_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_article_published_id" ON "article" USING btree ("published_at" timestamp_ops,"id" uuid_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_article_hash" ON "article" USING btree ("hash" text_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_user_email" ON "user" USING btree (lower((email)::text) text_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_name" ON "source" USING btree (lower((name)::text) text_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_url" ON "source" USING btree (lower((url)::text) text_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_6fe2655d7294869c" ON "bookmark_article" USING btree ("article_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_6fe2655d92741d25" ON "bookmark_article" USING btree ("bookmark_id" uuid_ops);
|
||||
*/
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Custom SQL migration file, put your code below! --
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
SET SESSION TIME ZONE 'UTC';
|
||||
@@ -0,0 +1,154 @@
|
||||
CREATE TYPE "public"."bias" AS ENUM('neutral', 'slightly', 'partisan', 'extreme');--> statement-breakpoint
|
||||
CREATE TYPE "public"."reliability" AS ENUM('trusted', 'reliable', 'average', 'low_trust', 'unreliable');--> statement-breakpoint
|
||||
CREATE TYPE "public"."sentiment" AS ENUM('positive', 'neutral', 'negative');--> statement-breakpoint
|
||||
CREATE TYPE "public"."token_purpose" AS ENUM('confirm_account', 'password_reset', 'unlock_account', 'delete_account');--> statement-breakpoint
|
||||
CREATE TYPE "public"."transparency" AS ENUM('high', 'medium', 'low');--> statement-breakpoint
|
||||
CREATE TABLE "article" (
|
||||
"body" text NOT NULL,
|
||||
"categories" text[],
|
||||
"crawled_at" timestamp DEFAULT now() NOT NULL,
|
||||
"credibility" jsonb,
|
||||
"excerpt" varchar(255) GENERATED ALWAYS AS (("left"(body, 200) || '...'::text)) STORED,
|
||||
"hash" varchar(32) NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"image" varchar(1024) GENERATED ALWAYS AS ((metadata ->> 'image'::text)) STORED,
|
||||
"link" varchar(1024) NOT NULL,
|
||||
"metadata" jsonb,
|
||||
"published_at" timestamp NOT NULL,
|
||||
"reading_time" integer DEFAULT 1,
|
||||
"sentiment" "sentiment" NOT NULL,
|
||||
"source_id" uuid NOT NULL,
|
||||
"title" varchar(1024) NOT NULL,
|
||||
"token_statistics" jsonb,
|
||||
"tsv" "tsvector" GENERATED ALWAYS AS ((
|
||||
setweight(to_tsvector('french'::regconfig, COALESCE(title, '')::text), 'A'::"char")
|
||||
|| setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char")
|
||||
)) STORED,
|
||||
"updated_at" timestamp
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"description" varchar(512),
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"is_public" boolean DEFAULT false NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"updated_at" timestamp,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark_article" (
|
||||
"article_id" uuid NOT NULL,
|
||||
"bookmark_id" uuid NOT NULL,
|
||||
CONSTRAINT "bookmark_article_pkey" PRIMARY KEY("bookmark_id","article_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "comment" (
|
||||
"article_id" uuid NOT NULL,
|
||||
"content" varchar(512) NOT NULL,
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"is_spam" boolean DEFAULT false NOT NULL,
|
||||
"sentiment" "sentiment" NOT NULL,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "followed_source" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"follower_id" uuid NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"source_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_attempt" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_history" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"device" jsonb,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"ip_address" "inet",
|
||||
"location" jsonb,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "refresh_token" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"token" varchar(128) NOT NULL,
|
||||
"username" varchar(255) NOT NULL,
|
||||
"valid" timestamp NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "source" (
|
||||
"credibility" jsonb,
|
||||
"description" varchar(1024),
|
||||
"display_name" varchar(255),
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"updated_at" timestamp,
|
||||
"url" varchar(255) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "user" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"email" varchar(255) NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"is_confirmed" boolean DEFAULT false NOT NULL,
|
||||
"is_locked" boolean DEFAULT false NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"password" varchar(512) NOT NULL,
|
||||
"roles" varchar(255)[] DEFAULT '{"ROLE_USER"}' NOT NULL,
|
||||
"updated_at" timestamp
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "verification_token" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"purpose" "token_purpose" NOT NULL,
|
||||
"token" varchar(60),
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "article" ADD CONSTRAINT "fk_article_source_id" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark" ADD CONSTRAINT "fk_bookmark_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_bookmark_article_bookmark_id" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmark"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_bookmark_article_article_id" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_comment_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_comment_article_id" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_followed_source_follower_id" FOREIGN KEY ("follower_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_followed_source_source_id" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_attempt" ADD CONSTRAINT "fk_login_attempt_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_history" ADD CONSTRAINT "fk_login_history_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "verification_token" ADD CONSTRAINT "fk_verification_token_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_categories" ON "article" USING gin ("categories" array_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_link_trgm" ON "article" USING gin ("link" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_title_trgm" ON "article" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_tsv" ON "article" USING gin ("tsv" tsvector_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_article_source_published_id" ON "article" USING btree ("source_id","published_at" DESC NULLS FIRST,"id" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_article_hash" ON "article" USING btree ("hash");--> statement-breakpoint
|
||||
CREATE INDEX "idx_bookmark_user_created" ON "bookmark" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_bookmark_user_name" ON "bookmark" USING btree ("user_id",lower("name"));--> statement-breakpoint
|
||||
CREATE INDEX "idx_bookmark_article_bookmark_id" ON "bookmark_article" USING btree ("bookmark_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_article_id" ON "comment" USING btree ("article_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_user_id" ON "comment" USING btree ("user_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_article_created" ON "comment" USING btree ("article_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_source_id" ON "followed_source" USING btree ("source_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_follower_id" ON "followed_source" USING btree ("follower_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_follower_created" ON "followed_source" USING btree ("follower_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_followed_source_user_source" ON "followed_source" USING btree ("follower_id","source_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_attempt_user_created" ON "login_attempt" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_user_created" ON "login_history" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_ip_address" ON "login_history" USING btree ("ip_address");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "uniq_refresh_token_token" ON "refresh_token" USING btree ("token");--> statement-breakpoint
|
||||
CREATE INDEX "idx_refresh_token_valid" ON "refresh_token" USING btree ("valid");--> statement-breakpoint
|
||||
CREATE INDEX "idx_refresh_token_username" ON "refresh_token" USING btree (lower("username"));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_name" ON "source" USING btree (lower((name)::text));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_url" ON "source" USING btree (lower((url)::text));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_user_email" ON "user" USING btree (lower((email)::text));--> statement-breakpoint
|
||||
CREATE INDEX "idx_user_created_at" ON "user" USING btree (created_at);--> statement-breakpoint
|
||||
CREATE INDEX "idx_verif_token_created_at" ON "verification_token" USING btree ("created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_verif_user_purpose_token" ON "verification_token" USING btree ("user_id","purpose","token") WHERE "verification_token"."token" IS NOT NULL;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_verif_token_token" ON "verification_token" USING btree ("token") WHERE "verification_token"."token" IS NOT NULL;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -4,9 +4,16 @@
|
||||
{
|
||||
"breakpoints": true,
|
||||
"idx": 0,
|
||||
"tag": "0000_aromatic_dorian_gray",
|
||||
"tag": "0000_setup",
|
||||
"version": "7",
|
||||
"when": 1762691204645
|
||||
"when": 1762775141000
|
||||
},
|
||||
{
|
||||
"breakpoints": true,
|
||||
"idx": 1,
|
||||
"tag": "0001_init",
|
||||
"version": "7",
|
||||
"when": 1762775267679
|
||||
}
|
||||
],
|
||||
"version": "7"
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@basango/encryption": "workspace:*",
|
||||
"@basango/logger": "workspace:*",
|
||||
"@date-fns/utc": "^2.1.1",
|
||||
"drizzle-orm": "^0.44.7",
|
||||
"mysql2": "^3.15.3",
|
||||
"pg": "^8.16.3",
|
||||
"snakecase-keys": "^9.0.2"
|
||||
"snakecase-keys": "^9.0.2",
|
||||
"tiktoken": "^1.0.22",
|
||||
"uuid": "^13.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/pg": "^8.15.6",
|
||||
@@ -13,6 +17,7 @@
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./client": "./src/client.ts",
|
||||
"./importer": "./src/importer/index.ts",
|
||||
"./queries": "./src/queries/index.ts",
|
||||
"./schema": "./src/schema.ts",
|
||||
"./utils": "./src/utils/index.ts"
|
||||
@@ -21,6 +26,7 @@
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"clean": "rm -rf .turbo node_modules",
|
||||
"sync:import": "bun ./src/importer/import.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
export class NotFoundError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "NotFoundError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
import { RowDataPacket } from "mysql2/promise";
|
||||
import { Pool, PoolClient } from "pg";
|
||||
|
||||
import { computeReadingTime, computeTokenStatistics } from "@/utils/computed";
|
||||
|
||||
type SourceOptions = {
|
||||
host: string;
|
||||
user: string;
|
||||
password: string;
|
||||
database: string;
|
||||
};
|
||||
|
||||
type TargetOptions = {
|
||||
database: string;
|
||||
batchSize?: number;
|
||||
pageSize?: number;
|
||||
ignoreColumns?: Record<string, string[]>;
|
||||
};
|
||||
|
||||
const DEFAULT_IGNORE: Record<string, string[]> = {
|
||||
article: ["tsv", "image", "excerpt", "bias", "reliability", "transparency"],
|
||||
source: ["bias", "reliability", "transparency"],
|
||||
};
|
||||
|
||||
/**
|
||||
* Engine
|
||||
*
|
||||
* Coordinates copying rows from a MySQL source into a PostgreSQL target in a
|
||||
* controlled, transactional, batched manner.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Establish and manage a connection pool to the target PostgreSQL database.
|
||||
* - Stream rows from a MySQL source (via a temporary pool) using pagination.
|
||||
* - Transform row values to match target expectations (UUID normalization,
|
||||
* timestamp fallback, array parsing for categories/roles, computed JSON
|
||||
* credibility, etc.).
|
||||
* - Filter out ignored columns based on a configurable ignore map.
|
||||
* - Insert rows into the target in configurable batch sizes with transactional
|
||||
* commits every batch to limit long-running transactions.
|
||||
* - Provide a safe reset operation that truncates the target table and manages
|
||||
* session replication role toggling for Postgres.
|
||||
*
|
||||
* @param sourceOptions - connection and authentication options for the MySQL
|
||||
* source (database, host, user, password, etc.).
|
||||
* @param targetOptions - configuration for the Postgres target including
|
||||
* connection string (database), optional pageSize, batchSize and per-table
|
||||
* ignoreColumns map.
|
||||
*/
|
||||
export class Engine {
|
||||
private readonly target: Pool;
|
||||
private readonly ignore: Record<string, string[]>;
|
||||
private readonly pageSize: number;
|
||||
private readonly batchSize: number;
|
||||
|
||||
constructor(
|
||||
private readonly sourceOptions: SourceOptions,
|
||||
private readonly targetOptions: TargetOptions,
|
||||
) {
|
||||
this.target = new Pool({
|
||||
allowExitOnIdle: true,
|
||||
connectionString: this.targetOptions.database,
|
||||
max: 8,
|
||||
});
|
||||
this.ignore = { ...DEFAULT_IGNORE, ...(this.targetOptions.ignoreColumns ?? {}) };
|
||||
this.pageSize = this.targetOptions.pageSize ?? 10_000;
|
||||
this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 1000);
|
||||
console.log(
|
||||
`Engine initialized with pageSize=${this.pageSize} and batchSize=${this.batchSize}`,
|
||||
);
|
||||
}
|
||||
|
||||
async close() {
|
||||
await this.target.end();
|
||||
}
|
||||
|
||||
async import(table: string): Promise<number> {
|
||||
await this.reset(table);
|
||||
return await this.paste(table, this.copy(table));
|
||||
}
|
||||
|
||||
private async *copy(table: string): AsyncGenerator<Record<string, unknown>> {
|
||||
const mysql = await import("mysql2/promise");
|
||||
|
||||
const source = mysql.createPool({
|
||||
database: this.sourceOptions.database,
|
||||
host: this.sourceOptions.host,
|
||||
idleTimeout: 180_000_000,
|
||||
password: this.sourceOptions.password,
|
||||
port: 3306,
|
||||
rowsAsArray: false,
|
||||
user: this.sourceOptions.user,
|
||||
});
|
||||
|
||||
let offset = 0;
|
||||
const size = this.pageSize;
|
||||
try {
|
||||
while (true) {
|
||||
const [rows] = await source.query<RowDataPacket[]>(
|
||||
`SELECT * FROM \`${this.escapeBacktick(table)}\` LIMIT ? OFFSET ?`,
|
||||
[size, offset],
|
||||
);
|
||||
|
||||
if (!rows || rows.length === 0) break;
|
||||
|
||||
for (const row of rows) {
|
||||
yield row as Record<string, unknown>;
|
||||
}
|
||||
|
||||
offset += rows.length;
|
||||
if (rows.length < size) break;
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await source.end();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
private async paste(
|
||||
table: string,
|
||||
rows: AsyncGenerator<Record<string, unknown>>,
|
||||
): Promise<number> {
|
||||
const target = await this.target.connect();
|
||||
let total = 0;
|
||||
let inBatch = 0;
|
||||
let columns: string[] | null = null;
|
||||
let insertSql = "";
|
||||
|
||||
const ignored = this.ignoredColumnsFor(table);
|
||||
const ignoredSet = new Set(ignored);
|
||||
|
||||
try {
|
||||
for await (let row of rows) {
|
||||
if (!columns) {
|
||||
row = this.transformRowForTarget(table, row);
|
||||
// Filter ignored columns and build column order
|
||||
columns = Object.keys(row).filter((c) => !ignoredSet.has(c));
|
||||
|
||||
// If article target has credibility but source not, include computed credibility
|
||||
if (
|
||||
(this.normalizedName(table) === "article" && !columns.includes("credibility")) ||
|
||||
(this.normalizedName(table) === "source" && !columns.includes("credibility"))
|
||||
) {
|
||||
columns.push("credibility");
|
||||
}
|
||||
|
||||
if (this.normalizedName(table) === "article" && !columns.includes("token_statistics")) {
|
||||
columns.push("token_statistics");
|
||||
}
|
||||
|
||||
const colsSql = columns.map((c) => this.quote(c)).join(", ");
|
||||
const placeholders = columns.map((_, i) => `$${i + 1}`).join(", ");
|
||||
insertSql = `INSERT INTO ${this.quote(table)} (${colsSql}) VALUES (${placeholders})`;
|
||||
|
||||
await target.query("BEGIN");
|
||||
}
|
||||
|
||||
// Row transform and params in column order
|
||||
const transformed = this.transformRowForTarget(table, row);
|
||||
const params = columns!.map((c) => this.valueForColumn(c, transformed));
|
||||
|
||||
try {
|
||||
await target.query(insertSql, params);
|
||||
} catch (err: unknown) {
|
||||
const msg = String((err as Error)?.message ?? "");
|
||||
if (msg.includes("invalid input syntax for type timestamp")) {
|
||||
// Fallback: coerce all *_at params to now() and retry once
|
||||
const fixed = columns!.map((c, i) => (c.endsWith("_at") ? new Date() : params[i]));
|
||||
await target.query(insertSql, fixed);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
total++;
|
||||
inBatch++;
|
||||
|
||||
if (inBatch >= this.batchSize) {
|
||||
await target.query("COMMIT");
|
||||
inBatch = 0;
|
||||
await target.query("BEGIN");
|
||||
console.log(`Imported ${total} records into ${table} so far...`);
|
||||
}
|
||||
}
|
||||
|
||||
if (inBatch > 0) {
|
||||
await target.query("COMMIT");
|
||||
}
|
||||
} catch (e) {
|
||||
await safeRollback(target);
|
||||
throw e;
|
||||
} finally {
|
||||
target.release();
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
private normalizedName(table: string): string {
|
||||
return table.replaceAll('"', "").replaceAll("`", "").toLowerCase();
|
||||
}
|
||||
|
||||
private ignoredColumnsFor(table: string): string[] {
|
||||
return this.ignore[this.normalizedName(table)] ?? [];
|
||||
}
|
||||
|
||||
private async reset(table: string) {
|
||||
const client = await this.target.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SET session_replication_role = 'replica'");
|
||||
await client.query(`TRUNCATE TABLE ${this.quote(table)} RESTART IDENTITY CASCADE`);
|
||||
await client.query("SET session_replication_role = 'origin'");
|
||||
await client.query("COMMIT");
|
||||
console.log(`Reset completed for table ${table}`);
|
||||
} catch (e) {
|
||||
await safeRollback(client);
|
||||
throw e;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
private transformRowForTarget(table: string, row: Record<string, unknown>) {
|
||||
const t = this.normalizedName(table);
|
||||
const clone: Record<string, unknown> = { ...row };
|
||||
|
||||
// Normalize UUIDs and timestamps and categories
|
||||
for (const [key, val] of Object.entries(clone)) {
|
||||
if (val == null) continue;
|
||||
|
||||
if (key === "id" || key.endsWith("_id")) {
|
||||
clone[key] = this.normalizeUuidValue(val);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Robust timestamp normalization for *_at columns
|
||||
if (key.endsWith("_at")) {
|
||||
clone[key] = this.normalizeTimestampValue(val);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (key === "categories") {
|
||||
if (Array.isArray(val)) {
|
||||
clone[key] = val;
|
||||
} else if (typeof val === "string") {
|
||||
const raw = val.trim();
|
||||
// Try JSON first
|
||||
if (raw.startsWith("[") && raw.endsWith("]")) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) {
|
||||
clone[key] = parsed;
|
||||
continue;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const parts = raw
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
clone[key] = parts.length ? parts : null;
|
||||
}
|
||||
}
|
||||
|
||||
if (t === "article" && key === "token_statistics") {
|
||||
clone[key] = computeTokenStatistics({
|
||||
body: String(clone.body ?? ""),
|
||||
categories: Array.isArray(clone.categories) ? clone.categories : [],
|
||||
title: String(clone.title ?? ""),
|
||||
});
|
||||
}
|
||||
|
||||
if (t === "article" && key === "reading_time") {
|
||||
clone[key] = Math.max(1, computeReadingTime(String(clone.body ?? "")));
|
||||
}
|
||||
|
||||
if (key === "roles") {
|
||||
if (Array.isArray(val)) {
|
||||
clone[key] = val;
|
||||
} else if (typeof val === "string") {
|
||||
const raw = val.trim();
|
||||
|
||||
// If the value is a JSON array string like '["ROLE_USER","ROLE_ADMIN"]', parse it.
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) {
|
||||
clone[key] = parsed;
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// not JSON, fall back to CSV-like parsing below
|
||||
}
|
||||
|
||||
// Remove surrounding brackets/quotes then split by comma and strip quotes/space
|
||||
const parts = raw
|
||||
.replace(/^\[|\]$/g, "")
|
||||
.split(",")
|
||||
.map((s) => s.replace(/^["']|["']$/g, "").trim())
|
||||
.filter(Boolean);
|
||||
|
||||
clone[key] = parts.length ? parts : ["ROLE_USER"];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compute credibility JSON if bias/reliability/transparency present
|
||||
if (t === "article" || t === "source") {
|
||||
const bias = clone.bias ?? null;
|
||||
const reliability = clone.reliability ?? null;
|
||||
const transparency = clone.transparency ?? null;
|
||||
if (bias || reliability || transparency) {
|
||||
clone.credibility = {
|
||||
bias,
|
||||
reliability,
|
||||
transparency,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure article token_statistics exists (computed on the fly)
|
||||
if (
|
||||
t === "article" &&
|
||||
(clone.token_statistics == null || typeof clone.token_statistics !== "object")
|
||||
) {
|
||||
clone.token_statistics = computeTokenStatistics({
|
||||
body: String(clone.body ?? ""),
|
||||
categories: Array.isArray(clone.categories) ? (clone.categories as string[]) : [],
|
||||
title: String(clone.title ?? ""),
|
||||
});
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
private valueForColumn(col: string, row: Record<string, unknown>) {
|
||||
const v = row[col];
|
||||
// Pass Date objects directly to pg for timestamp columns
|
||||
if (col.endsWith("_at") && v instanceof Date) {
|
||||
return v;
|
||||
}
|
||||
if (col === "credibility" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "token_statistics" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "device" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "location" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "roles" && v) {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "metadata" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
return v ?? null;
|
||||
}
|
||||
|
||||
private normalizeUuidValue(value: unknown): string {
|
||||
if (Buffer.isBuffer(value)) {
|
||||
return bufferToUuid(value);
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
// Already a UUID string or hex; try to format 32-hex into canonical form
|
||||
const hex = value.replace(/-/g, "").toLowerCase();
|
||||
if (/^[0-9a-f]{32}$/.test(hex)) {
|
||||
return (
|
||||
hex.slice(0, 8) +
|
||||
"-" +
|
||||
hex.slice(8, 12) +
|
||||
"-" +
|
||||
hex.slice(12, 16) +
|
||||
"-" +
|
||||
hex.slice(16, 20) +
|
||||
"-" +
|
||||
hex.slice(20)
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
private normalizeTimestampValue(value: unknown): Date {
|
||||
// If it's already a Date, ensure it's valid
|
||||
if (value instanceof Date) {
|
||||
return Number.isNaN(value.getTime()) ? new Date() : value;
|
||||
}
|
||||
|
||||
// Strings: handle common invalid patterns and attempt safe parsing
|
||||
if (typeof value === "string") {
|
||||
const raw = value.trim();
|
||||
if (
|
||||
!raw ||
|
||||
/0000-00-00/.test(raw) ||
|
||||
/NaN/.test(raw) ||
|
||||
raw.toLowerCase() === "invalid date"
|
||||
) {
|
||||
return new Date();
|
||||
}
|
||||
|
||||
// Normalize MySQL-like 'YYYY-MM-DD HH:MM:SS[.ffffff]' to ISO
|
||||
let s = raw.replace(" ", "T");
|
||||
// Reduce microseconds to milliseconds (3 digits) if present
|
||||
s = s.replace(/\.(\d{3})\d+$/, ".$1");
|
||||
// Append Z if there is no timezone info
|
||||
if (!/[zZ]|[+-]\d{2}:?\d{2}$/.test(s)) s += "Z";
|
||||
|
||||
const d = new Date(s);
|
||||
if (!Number.isNaN(d.getTime())) return d;
|
||||
|
||||
// Try numeric string as epoch seconds/millis
|
||||
const n = Number(raw);
|
||||
if (Number.isFinite(n)) {
|
||||
const ms = n > 1e12 ? n : n * 1000;
|
||||
const d2 = new Date(ms);
|
||||
if (!Number.isNaN(d2.getTime())) return d2;
|
||||
}
|
||||
|
||||
return new Date();
|
||||
}
|
||||
|
||||
// Numbers: treat as epoch seconds/millis
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const ms = value > 1e12 ? value : value * 1000;
|
||||
const d = new Date(ms);
|
||||
return Number.isNaN(d.getTime()) ? new Date() : d;
|
||||
}
|
||||
|
||||
// Fallback: now
|
||||
return new Date();
|
||||
}
|
||||
|
||||
private quote(id: string) {
|
||||
const norm = this.normalizedName(id);
|
||||
return `"${norm.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
private escapeBacktick(id: string) {
|
||||
return id.replaceAll("`", "``");
|
||||
}
|
||||
}
|
||||
|
||||
function bufferToUuid(buf: Buffer): string {
|
||||
if (buf.length !== 16) return buf.toString("hex");
|
||||
const hex = buf.toString("hex");
|
||||
return (
|
||||
hex.slice(0, 8) +
|
||||
"-" +
|
||||
hex.slice(8, 12) +
|
||||
"-" +
|
||||
hex.slice(12, 16) +
|
||||
"-" +
|
||||
hex.slice(16, 20) +
|
||||
"-" +
|
||||
hex.slice(20)
|
||||
);
|
||||
}
|
||||
|
||||
async function safeRollback(client: PoolClient) {
|
||||
try {
|
||||
await client.query("ROLLBACK");
|
||||
} catch {}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bun
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
|
||||
import { createEnvAccessor } from "@devscast/config";
|
||||
|
||||
import { Engine } from "@/importer";
|
||||
|
||||
const env = createEnvAccessor([
|
||||
"BASANGO_SOURCE_DATABASE_HOST",
|
||||
"BASANGO_SOURCE_DATABASE_USER",
|
||||
"BASANGO_SOURCE_DATABASE_PASS",
|
||||
"BASANGO_SOURCE_DATABASE_NAME",
|
||||
"BASANGO_DATABASE_URL",
|
||||
]);
|
||||
|
||||
async function promptConfirm(question: string, def = false) {
|
||||
const rl = createInterface({ input, output });
|
||||
const suffix = def ? "[Y/n]" : "[y/N]";
|
||||
const answer = await rl.question(`${question} ${suffix} `);
|
||||
rl.close();
|
||||
const v = String(answer || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (v === "y" || v === "yes") return true;
|
||||
if (v === "n" || v === "no") return false;
|
||||
return def;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const ok = await promptConfirm("Do you want to continue?", false);
|
||||
if (!ok) {
|
||||
console.warn("Process aborted");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const engine = new Engine(
|
||||
{
|
||||
database: env("BASANGO_SOURCE_DATABASE_NAME"),
|
||||
host: env("BASANGO_SOURCE_DATABASE_HOST"),
|
||||
password: env("BASANGO_SOURCE_DATABASE_PASS"),
|
||||
user: env("BASANGO_SOURCE_DATABASE_USER"),
|
||||
},
|
||||
{
|
||||
database: env("BASANGO_DATABASE_URL"),
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const tables = process.argv.slice(2);
|
||||
if (tables.length === 0) tables.push("user", "source", "article");
|
||||
for (const t of tables) {
|
||||
const count = await engine.import(t);
|
||||
console.log(`Imported ${count} records into ${t} table.`);
|
||||
}
|
||||
console.log("Import completed successfully");
|
||||
} finally {
|
||||
await engine.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err?.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from "./engine";
|
||||
export * from "./import";
|
||||
@@ -0,0 +1,65 @@
|
||||
import { md5 } from "@basango/encryption";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { v7 as uuidV7 } from "uuid";
|
||||
|
||||
import { Database } from "@/client";
|
||||
import { ArticleMetadata, Sentiment, TokenStatistics, article } from "@/schema";
|
||||
import { computeReadingTime, computeTokenStatistics } from "@/utils/computed";
|
||||
|
||||
import { getSourceIdByName } from "./sources";
|
||||
|
||||
export type CreateArticleParams = {
|
||||
title: string;
|
||||
body: string;
|
||||
categories: string[];
|
||||
link: string;
|
||||
sourceId: string;
|
||||
publishedAt: Date;
|
||||
sentiment?: Sentiment;
|
||||
tokenStatistics?: TokenStatistics;
|
||||
readingTime?: number;
|
||||
metadata?: ArticleMetadata;
|
||||
};
|
||||
|
||||
export async function createArticle(db: Database, params: CreateArticleParams) {
|
||||
const data = {
|
||||
...params,
|
||||
hash: md5(params.link),
|
||||
readingTime: computeReadingTime(params.body),
|
||||
sentiment: "neutral" as Sentiment,
|
||||
sourceId: await getSourceIdByName(db, params.sourceId),
|
||||
tokenStatistics: computeTokenStatistics({
|
||||
body: params.body,
|
||||
categories: params.categories,
|
||||
title: params.title,
|
||||
}),
|
||||
};
|
||||
|
||||
const duplicated = await getArticleByHash(db, data.hash);
|
||||
if (duplicated !== undefined) {
|
||||
return {
|
||||
id: duplicated.id,
|
||||
sourceId: duplicated.sourceId,
|
||||
};
|
||||
}
|
||||
|
||||
const [result] = await db
|
||||
.insert(article)
|
||||
.values({ id: uuidV7(), ...data })
|
||||
.returning({
|
||||
id: article.id,
|
||||
sourceId: article.sourceId,
|
||||
});
|
||||
|
||||
if (result === undefined) {
|
||||
throw new Error("Failed to create article");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function getArticleByHash(db: Database, hash: string) {
|
||||
return db.query.article.findFirst({
|
||||
where: eq(article.hash, hash),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { v7 as uuidV7 } from "uuid";
|
||||
|
||||
import { Database } from "@/client";
|
||||
import { NotFoundError } from "@/errors";
|
||||
import { Credibility, source } from "@/schema";
|
||||
|
||||
export type CreateSourceParams = {
|
||||
name: string;
|
||||
url: string;
|
||||
displayName?: string;
|
||||
description?: string;
|
||||
credibility: Credibility;
|
||||
updatedAt?: Date;
|
||||
};
|
||||
|
||||
export async function createSource(db: Database, params: CreateSourceParams) {
|
||||
const [result] = await db
|
||||
.insert(source)
|
||||
.values({ id: uuidV7(), ...params })
|
||||
.returning();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export type DeleteSourceParams = {
|
||||
id: string;
|
||||
};
|
||||
|
||||
export async function deleteSource(db: Database, params: DeleteSourceParams) {
|
||||
const [result] = await db.delete(source).where(eq(source.id, params.id)).returning();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function getSourceByName(db: Database, name: string) {
|
||||
return db.query.source.findFirst({
|
||||
where: eq(source.name, name),
|
||||
});
|
||||
}
|
||||
|
||||
export async function getSourceIdByName(db: Database, name: string): Promise<string> {
|
||||
const result = await db.query.source.findFirst({
|
||||
columns: {
|
||||
id: true,
|
||||
},
|
||||
where: eq(source.name, name),
|
||||
});
|
||||
|
||||
if (!result) {
|
||||
throw new NotFoundError(`Source with name "${name}" not found`);
|
||||
}
|
||||
|
||||
return result.id;
|
||||
}
|
||||
|
||||
export type GetSourceByIdParams = {
|
||||
id: string;
|
||||
};
|
||||
|
||||
export async function getSourceById(db: Database, params: GetSourceByIdParams) {
|
||||
return db.query.source.findFirst({
|
||||
where: eq(source.id, params.id),
|
||||
});
|
||||
}
|
||||
|
||||
+334
-253
@@ -1,15 +1,14 @@
|
||||
import { relations, sql } from "drizzle-orm";
|
||||
import { check } from "drizzle-orm/gel-core";
|
||||
import {
|
||||
boolean,
|
||||
check,
|
||||
customType,
|
||||
doublePrecision,
|
||||
foreignKey,
|
||||
index,
|
||||
inet,
|
||||
integer,
|
||||
jsonb,
|
||||
pgSequence,
|
||||
pgEnum,
|
||||
pgTable,
|
||||
primaryKey,
|
||||
text,
|
||||
@@ -19,242 +18,181 @@ import {
|
||||
varchar,
|
||||
} from "drizzle-orm/pg-core";
|
||||
|
||||
const tsvector = customType<{ data: string; driverData: string }>({
|
||||
/* -------------------------------------------------------------------------- */
|
||||
/* Types */
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
export const tsvector = customType<{ data: string; driverData: string }>({
|
||||
dataType() {
|
||||
return "tsvector";
|
||||
},
|
||||
});
|
||||
|
||||
export const refreshTokensIdSeq = pgSequence("refresh_tokens_id_seq", {
|
||||
cache: "1",
|
||||
cycle: false,
|
||||
increment: "1",
|
||||
maxValue: "9223372036854775807",
|
||||
minValue: "1",
|
||||
startWith: "1",
|
||||
});
|
||||
export const customJsonType = <T>() =>
|
||||
customType<{ data: T }>({
|
||||
dataType() {
|
||||
return "jsonb";
|
||||
},
|
||||
fromDriver(value) {
|
||||
return value as T;
|
||||
},
|
||||
toDriver(value) {
|
||||
return value; // JSONB → just pass the object
|
||||
},
|
||||
});
|
||||
|
||||
// legacy table for doctrine migrations
|
||||
export const doctrineMigrationVersions = pgTable("doctrine_migration_versions", {
|
||||
executedAt: timestamp("executed_at", { mode: "string" }).default(sql`NULL`),
|
||||
executionTime: integer("execution_time"),
|
||||
version: varchar({ length: 191 }).primaryKey().notNull(),
|
||||
});
|
||||
export const biasEnum = pgEnum("bias", ["neutral", "slightly", "partisan", "extreme"]);
|
||||
export const reliabilityEnum = pgEnum("reliability", [
|
||||
"trusted",
|
||||
"reliable",
|
||||
"average",
|
||||
"low_trust",
|
||||
"unreliable",
|
||||
]);
|
||||
export const sentimentEnum = pgEnum("sentiment", ["positive", "neutral", "negative"]);
|
||||
export const transparencyEnum = pgEnum("transparency", ["high", "medium", "low"]);
|
||||
export const tokenPurposeEnum = pgEnum("token_purpose", [
|
||||
"confirm_account",
|
||||
"password_reset",
|
||||
"unlock_account",
|
||||
"delete_account",
|
||||
]);
|
||||
|
||||
export const bookmark = pgTable(
|
||||
"bookmark",
|
||||
export type EmailAddress = string;
|
||||
export type Link = string;
|
||||
export type ReadingTime = number;
|
||||
|
||||
export type Role = "ROLE_USER" | "ROLE_ADMIN";
|
||||
export type Roles = Role[];
|
||||
|
||||
export type Bias = (typeof biasEnum.enumValues)[number];
|
||||
export type Reliability = (typeof reliabilityEnum.enumValues)[number];
|
||||
export type Sentiment = (typeof sentimentEnum.enumValues)[number];
|
||||
export type Transparency = (typeof transparencyEnum.enumValues)[number];
|
||||
export type TokenPurpose = (typeof tokenPurposeEnum.enumValues)[number];
|
||||
|
||||
export type Credibility = {
|
||||
bias: Bias;
|
||||
reliability: Reliability;
|
||||
transparency: Transparency;
|
||||
};
|
||||
|
||||
export type TokenStatistics = {
|
||||
title: number;
|
||||
body: number;
|
||||
categories: number;
|
||||
excerpt: number;
|
||||
total: number;
|
||||
};
|
||||
|
||||
export type Device = {
|
||||
operatingSystem?: string;
|
||||
client?: string;
|
||||
device?: string;
|
||||
isBot: boolean;
|
||||
};
|
||||
|
||||
export type GeoLocation = {
|
||||
country?: string;
|
||||
city?: string;
|
||||
timeZone?: string;
|
||||
longitude?: number;
|
||||
latitude?: number;
|
||||
accuracyRadius?: number;
|
||||
};
|
||||
|
||||
export type ClientProfile = {
|
||||
userIp?: string;
|
||||
userAgent?: string;
|
||||
hints: unknown[];
|
||||
};
|
||||
|
||||
export type ArticleMetadata = {
|
||||
title?: string;
|
||||
description?: string;
|
||||
image?: string;
|
||||
};
|
||||
|
||||
export type DateRange = {
|
||||
start: number; // unix timestamp (seconds)
|
||||
end: number; // unix timestamp (seconds)
|
||||
};
|
||||
|
||||
// Secrets
|
||||
export type GeneratedToken = string;
|
||||
export type GeneratedCode = string;
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
/* Tables */
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
export const user = pgTable(
|
||||
"user",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
description: varchar({ length: 512 }).default(sql`NULL`),
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
email: varchar({ length: 255 }).$type<EmailAddress>().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isPublic: boolean("is_public").default(false).notNull(),
|
||||
isConfirmed: boolean("is_confirmed").default(false).notNull(),
|
||||
isLocked: boolean("is_locked").default(false).notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
userId: uuid("user_id").notNull(),
|
||||
password: varchar({ length: 512 }).notNull(),
|
||||
roles: varchar("roles", { length: 255 })
|
||||
.$type<Roles>()
|
||||
.array()
|
||||
.notNull()
|
||||
.default(["ROLE_USER"]),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_bookmark_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast().op("timestamp_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
index("idx_da62921da76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_da62921da76ed395",
|
||||
}).onDelete("cascade"),
|
||||
(_table) => [
|
||||
uniqueIndex("unq_user_email").using("btree", sql`lower((email)::text)`),
|
||||
index("idx_user_created_at").using("btree", sql`created_at`),
|
||||
sql`CONSTRAINT "chk_user_roles_json" CHECK (jsonb_typeof(roles) = 'array')`,
|
||||
],
|
||||
);
|
||||
|
||||
export const loginAttempt = pgTable(
|
||||
"login_attempt",
|
||||
export const source = pgTable(
|
||||
"source",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
credibility: jsonb("credibility").$type<Credibility>(),
|
||||
description: varchar({ length: 1024 }),
|
||||
displayName: varchar("display_name", { length: 255 }),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
url: varchar({ length: 255 }).notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_8c11c1ba76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_login_attempt_created_at").using(
|
||||
"btree",
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_8c11c1ba76ed395",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const loginHistory = pgTable(
|
||||
"login_history",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
deviceClient: varchar("device_client", { length: 255 }).default(sql`NULL`),
|
||||
deviceDevice: varchar("device_device", { length: 255 }).default(sql`NULL`),
|
||||
deviceIsBot: boolean("device_is_bot").default(false).notNull(),
|
||||
deviceOperatingSystem: varchar("device_operating_system", { length: 255 }).default(sql`NULL`),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
ipAddress: inet("ip_address"),
|
||||
locationAccuracyRadius: integer("location_accuracy_radius"),
|
||||
locationLatitude: doublePrecision("location_latitude"),
|
||||
locationLongitude: doublePrecision("location_longitude"),
|
||||
locationTimeZone: varchar("location_time_zone", { length: 255 }).default(sql`NULL`),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_37976e36a76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_login_history_created_at").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast().op("uuid_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
index("idx_login_history_ip_address").using(
|
||||
"btree",
|
||||
table.ipAddress.asc().nullsLast().op("inet_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_37976e36a76ed395",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const verificationToken = pgTable(
|
||||
"verification_token",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
purpose: varchar({ length: 255 }).notNull(),
|
||||
token: varchar({ length: 60 }).default(sql`NULL`),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_c1cc006ba76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_verif_token_created_at").using(
|
||||
"btree",
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
uniqueIndex("unq_verif_user_purpose_token")
|
||||
.using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast().op("text_ops"),
|
||||
table.purpose.asc().nullsLast().op("text_ops"),
|
||||
)
|
||||
.where(sql`(token IS NOT NULL)`),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_c1cc006ba76ed395",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const followedSource = pgTable(
|
||||
"followed_source",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
followerId: uuid("follower_id").notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
sourceId: uuid("source_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_7a763a3e953c1c61").using("btree", table.sourceId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_7a763a3eac24f853").using("btree", table.followerId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_followed_source_follower_created").using(
|
||||
"btree",
|
||||
table.followerId.asc().nullsLast().op("timestamp_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("uuid_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.followerId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_7a763a3eac24f853",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.sourceId],
|
||||
foreignColumns: [source.id],
|
||||
name: "fk_7a763a3e953c1c61",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const comment = pgTable(
|
||||
"comment",
|
||||
{
|
||||
articleId: uuid("article_id").notNull(),
|
||||
content: varchar({ length: 512 }).notNull(),
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isSpam: boolean("is_spam").default(false).notNull(),
|
||||
sentiment: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_9474526c7294869c").using("btree", table.articleId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_9474526ca76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_comment_article_created").using(
|
||||
"btree",
|
||||
table.articleId.asc().nullsLast().op("timestamp_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("uuid_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_9474526ca76ed395",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.articleId],
|
||||
foreignColumns: [article.id],
|
||||
name: "fk_9474526c7294869c",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const refreshTokens = pgTable(
|
||||
"refresh_tokens",
|
||||
{
|
||||
id: integer().primaryKey().notNull(),
|
||||
refreshToken: varchar("refresh_token", { length: 128 }).notNull(),
|
||||
username: varchar({ length: 255 }).notNull(),
|
||||
valid: timestamp({ mode: "string" }).notNull(),
|
||||
},
|
||||
(table) => [
|
||||
uniqueIndex("uniq_9bace7e1c74f2195").using(
|
||||
"btree",
|
||||
table.refreshToken.asc().nullsLast().op("text_ops"),
|
||||
),
|
||||
(_table) => [
|
||||
uniqueIndex("unq_source_name").using("btree", sql`lower((name)::text)`),
|
||||
uniqueIndex("unq_source_url").using("btree", sql`lower((url)::text)`),
|
||||
],
|
||||
);
|
||||
|
||||
export const article = pgTable(
|
||||
"article",
|
||||
{
|
||||
bias: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
body: text().notNull(),
|
||||
categories: text().array(),
|
||||
crawledAt: timestamp("crawled_at", { mode: "string" }).notNull(),
|
||||
crawledAt: timestamp("crawled_at").defaultNow().notNull(),
|
||||
credibility: jsonb("credibility").$type<Credibility>(),
|
||||
excerpt: varchar({ length: 255 }).generatedAlwaysAs(sql`("left"(body, 200) || '...'::text)`),
|
||||
hash: varchar({ length: 32 }).notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
image: varchar({ length: 1024 }).generatedAlwaysAs(sql`(metadata ->> 'image'::text)`),
|
||||
link: varchar({ length: 1024 }).notNull(),
|
||||
metadata: jsonb(),
|
||||
publishedAt: timestamp("published_at", { mode: "string" }).notNull(),
|
||||
metadata: jsonb("metadata").$type<ArticleMetadata>(),
|
||||
publishedAt: timestamp("published_at").notNull(),
|
||||
readingTime: integer("reading_time").default(1),
|
||||
reliability: varchar({ length: 30 }).default("reliable").notNull(),
|
||||
sentiment: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
sentiment: sentimentEnum("sentiment").notNull(),
|
||||
sourceId: uuid("source_id").notNull(),
|
||||
title: varchar({ length: 1024 }).notNull(),
|
||||
tokenStatistics: jsonb("token_statistics"),
|
||||
transparency: varchar({ length: 30 }).default("medium").notNull(),
|
||||
tokenStatistics: jsonb("token_statistics").$type<TokenStatistics>(),
|
||||
tsv: tsvector("tsv").generatedAlwaysAs(
|
||||
sql`(setweight(to_tsvector('french'::regconfig, (COALESCE(title, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char"))`,
|
||||
sql`(
|
||||
setweight(to_tsvector('french'::regconfig, COALESCE(title, '')::text), 'A'::"char")
|
||||
|| setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char")
|
||||
)`,
|
||||
),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
},
|
||||
(table) => [
|
||||
index("gin_article_categories").using(
|
||||
@@ -264,69 +202,57 @@ export const article = pgTable(
|
||||
index("gin_article_link_trgm").using("gin", table.link.asc().nullsLast().op("gin_trgm_ops")),
|
||||
index("gin_article_title_trgm").using("gin", table.title.asc().nullsLast().op("gin_trgm_ops")),
|
||||
index("gin_article_tsv").using("gin", table.tsv.asc().nullsLast().op("tsvector_ops")),
|
||||
index("idx_23a0e66953c1c61").using("btree", table.sourceId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_article_published_at").using(
|
||||
index("idx_article_source_published_id").using(
|
||||
"btree",
|
||||
table.publishedAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
table.sourceId.asc().nullsLast(),
|
||||
table.publishedAt.desc().nullsFirst(),
|
||||
table.id.desc().nullsFirst(),
|
||||
),
|
||||
index("idx_article_published_id").using(
|
||||
"btree",
|
||||
table.publishedAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
table.id.desc().nullsFirst().op("uuid_ops"),
|
||||
),
|
||||
uniqueIndex("unq_article_hash").using("btree", table.hash.asc().nullsLast().op("text_ops")),
|
||||
uniqueIndex("unq_article_hash").using("btree", table.hash.asc().nullsLast()),
|
||||
foreignKey({
|
||||
columns: [table.sourceId],
|
||||
foreignColumns: [source.id],
|
||||
name: "fk_23a0e66953c1c61",
|
||||
name: "fk_article_source_id",
|
||||
}).onDelete("cascade"),
|
||||
check("chk_article_reading_time", sql`reading_time >= 0`),
|
||||
check("chk_article_reading_time", sql`(reading_time >= 0)`),
|
||||
check(
|
||||
"chk_article_sentiment",
|
||||
sql`(sentiment)::text = ANY ((ARRAY['positive'::character varying, 'neutral'::character varying, 'negative'::character varying])::text[])`,
|
||||
sql`((sentiment)::text = ANY (ARRAY['positive'::text,'neutral'::text,'negative'::text]))`,
|
||||
),
|
||||
check(
|
||||
"chk_article_metadata_json",
|
||||
sql`(metadata IS NULL) OR (jsonb_typeof(metadata) = ANY (ARRAY['object'::text, 'array'::text]))`,
|
||||
sql`((metadata IS NULL) OR (jsonb_typeof(metadata) IN ('object'::text,'array'::text)))`,
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
export const user = pgTable(
|
||||
"user",
|
||||
export const bookmark = pgTable(
|
||||
"bookmark",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
email: varchar({ length: 255 }).notNull(),
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
description: varchar({ length: 512 }),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isConfirmed: boolean("is_confirmed").default(false).notNull(),
|
||||
isLocked: boolean("is_locked").default(false).notNull(),
|
||||
isPublic: boolean("is_public").default(false).notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
password: varchar({ length: 512 }).notNull(),
|
||||
roles: jsonb().notNull(),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(_table) => [
|
||||
uniqueIndex("unq_user_email").using("btree", sql`lower((email)::text)`),
|
||||
check("chk_user_roles_json", sql`jsonb_typeof(roles) = 'array'::text`),
|
||||
],
|
||||
);
|
||||
|
||||
export const source = pgTable(
|
||||
"source",
|
||||
{
|
||||
bias: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
description: varchar({ length: 1024 }).default(sql`NULL`),
|
||||
displayName: varchar("display_name", { length: 255 }).default(sql`NULL`),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
reliability: varchar({ length: 30 }).default("reliable").notNull(),
|
||||
transparency: varchar({ length: 30 }).default("medium").notNull(),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
url: varchar({ length: 255 }).notNull(),
|
||||
},
|
||||
(_table) => [
|
||||
uniqueIndex("unq_source_name").using("btree", sql`lower((name)::text)`),
|
||||
uniqueIndex("unq_source_url").using("btree", sql`lower((url)::text)`),
|
||||
(table) => [
|
||||
index("idx_bookmark_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
uniqueIndex("unq_bookmark_user_name").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
sql`lower(${table.name})`,
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_bookmark_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -337,22 +263,177 @@ export const bookmarkArticle = pgTable(
|
||||
bookmarkId: uuid("bookmark_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_6fe2655d7294869c").using("btree", table.articleId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_6fe2655d92741d25").using("btree", table.bookmarkId.asc().nullsLast().op("uuid_ops")),
|
||||
primaryKey({ columns: [table.bookmarkId, table.articleId], name: "bookmark_article_pkey" }),
|
||||
index("idx_bookmark_article_bookmark_id").using("btree", table.bookmarkId.asc().nullsLast()),
|
||||
foreignKey({
|
||||
columns: [table.bookmarkId],
|
||||
foreignColumns: [bookmark.id],
|
||||
name: "fk_6fe2655d92741d25",
|
||||
name: "fk_bookmark_article_bookmark_id",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.articleId],
|
||||
foreignColumns: [article.id],
|
||||
name: "fk_6fe2655d7294869c",
|
||||
name: "fk_bookmark_article_article_id",
|
||||
}).onDelete("cascade"),
|
||||
primaryKey({ columns: [table.bookmarkId, table.articleId], name: "bookmark_article_pkey" }),
|
||||
],
|
||||
);
|
||||
|
||||
export const loginAttempt = pgTable(
|
||||
"login_attempt",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_login_attempt_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_login_attempt_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const loginHistory = pgTable(
|
||||
"login_history",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
device: jsonb("device").$type<Device>(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
ipAddress: inet("ip_address"),
|
||||
location: jsonb("location").$type<GeoLocation>(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_login_history_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
index("idx_login_history_ip_address").using("btree", table.ipAddress.asc().nullsLast()),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_login_history_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const verificationToken = pgTable(
|
||||
"verification_token",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
purpose: tokenPurposeEnum("purpose").notNull(),
|
||||
token: varchar({ length: 60 }), // nullable if you support "reservations" before issue
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_verif_token_created_at").using("btree", table.createdAt.desc().nullsFirst()),
|
||||
uniqueIndex("unq_verif_user_purpose_token")
|
||||
.using("btree", table.userId, table.purpose, table.token)
|
||||
.where(sql`${table.token} IS NOT NULL`),
|
||||
uniqueIndex("unq_verif_token_token")
|
||||
.using("btree", table.token)
|
||||
.where(sql`${table.token} IS NOT NULL`),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_verification_token_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const followedSource = pgTable(
|
||||
"followed_source",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
followerId: uuid("follower_id").notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
sourceId: uuid("source_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_followed_source_source_id").using("btree", table.sourceId.asc().nullsLast()),
|
||||
index("idx_followed_source_follower_id").using("btree", table.followerId.asc().nullsLast()),
|
||||
index("idx_followed_source_follower_created").using(
|
||||
"btree",
|
||||
table.followerId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
uniqueIndex("unq_followed_source_user_source").using(
|
||||
"btree",
|
||||
table.followerId.asc().nullsLast(),
|
||||
table.sourceId.asc().nullsLast(),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.followerId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_followed_source_follower_id",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.sourceId],
|
||||
foreignColumns: [source.id],
|
||||
name: "fk_followed_source_source_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const comment = pgTable(
|
||||
"comment",
|
||||
{
|
||||
articleId: uuid("article_id").notNull(),
|
||||
content: varchar({ length: 512 }).notNull(),
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isSpam: boolean("is_spam").default(false).notNull(),
|
||||
sentiment: sentimentEnum("sentiment").notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_comment_article_id").using("btree", table.articleId.asc().nullsLast()),
|
||||
index("idx_comment_user_id").using("btree", table.userId.asc().nullsLast()),
|
||||
index("idx_comment_article_created").using(
|
||||
"btree",
|
||||
table.articleId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_comment_user_id",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.articleId],
|
||||
foreignColumns: [article.id],
|
||||
name: "fk_comment_article_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const refreshToken = pgTable(
|
||||
"refresh_token",
|
||||
{
|
||||
id: uuid().primaryKey().notNull(),
|
||||
token: varchar("token", { length: 128 }).notNull(),
|
||||
username: varchar({ length: 255 }).notNull(),
|
||||
valid: timestamp().notNull(),
|
||||
},
|
||||
(table) => [
|
||||
uniqueIndex("uniq_refresh_token_token").using("btree", table.token.asc().nullsLast()),
|
||||
index("idx_refresh_token_valid").using("btree", table.valid.asc().nullsLast()),
|
||||
index("idx_refresh_token_username").using("btree", sql`lower(${table.username})`),
|
||||
],
|
||||
);
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
/* Relations */
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
export const bookmarkRelations = relations(bookmark, ({ one, many }) => ({
|
||||
bookmarkArticles: many(bookmarkArticle),
|
||||
user: one(user, {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { TiktokenEncoding, get_encoding } from "tiktoken";
|
||||
|
||||
import { TokenStatistics } from "@/schema";
|
||||
|
||||
/**
|
||||
* Count the number of tokens in the given text using the specified encoding.
|
||||
* @param text - The input text
|
||||
* @param encoding - The token encoding (default: "cl100k_base")
|
||||
*/
|
||||
export const computeTokenCount = (
|
||||
text: string,
|
||||
encoding: TiktokenEncoding = "cl100k_base",
|
||||
): number => {
|
||||
try {
|
||||
const encoder = get_encoding(encoding);
|
||||
const tokens = encoder.encode(text);
|
||||
encoder.free();
|
||||
return tokens.length;
|
||||
} catch {
|
||||
return text.length;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Create token statistics for the given data.
|
||||
* @param data - The input data containing title, body, and categories
|
||||
* @returns TokenStatistics object
|
||||
*/
|
||||
export const computeTokenStatistics = (data: {
|
||||
title: string;
|
||||
body: string;
|
||||
categories: string[];
|
||||
}): TokenStatistics => {
|
||||
const title = computeTokenCount(data.title);
|
||||
const body = computeTokenCount(data.body);
|
||||
const categories = computeTokenCount(data.categories.join(","));
|
||||
const excerpt = computeTokenCount(data.body.substring(0, 200));
|
||||
|
||||
return {
|
||||
body,
|
||||
categories,
|
||||
excerpt,
|
||||
title,
|
||||
total: title + body + categories + excerpt,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute the estimated reading time for the given text.
|
||||
* @param text - The input text
|
||||
* @param wordsPerMinute - The reading speed in words per minute (default: 200)
|
||||
* @returns The estimated reading time in minutes
|
||||
*/
|
||||
export const computeReadingTime = (text: string, wordsPerMinute = 200): number => {
|
||||
const words = text.trim().split(/\s+/).length;
|
||||
return Math.ceil(words / wordsPerMinute);
|
||||
};
|
||||
@@ -6,8 +6,6 @@ import {
|
||||
PAGINATION_MAX_LIMIT,
|
||||
} from "@/constants";
|
||||
|
||||
export type SortDirection = "asc" | "desc";
|
||||
|
||||
export interface PageRequest {
|
||||
page?: number;
|
||||
limit?: number;
|
||||
|
||||
Reference in New Issue
Block a user