feat(db): migration and database setup

This commit is contained in:
2025-11-10 16:57:27 +02:00
parent 594b08a2d1
commit fbca02bec6
31 changed files with 2854 additions and 1928 deletions
+6 -1
View File
@@ -1 +1,6 @@
BASANGO_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/app?serverVersion=16&charset=utf8"
BASANGO_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/app?serverVersion=16&charset=utf8"
BASANGO_SOURCE_DATABASE_HOST="localhost"
BASANGO_SOURCE_DATABASE_PASS="root"
BASANGO_SOURCE_DATABASE_NAME="app"
BASANGO_SOURCE_DATABASE_USER="root"
@@ -1,172 +0,0 @@
-- Current sql file was generated after introspecting the database
-- If you want to run this migration please uncomment this code before executing migrations
/*
CREATE SEQUENCE "public"."refresh_tokens_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1;--> statement-breakpoint
CREATE TABLE "doctrine_migration_versions" (
"version" varchar(191) PRIMARY KEY NOT NULL,
"executed_at" timestamp(0) DEFAULT NULL,
"execution_time" integer
);
--> statement-breakpoint
CREATE TABLE "bookmark" (
"id" uuid PRIMARY KEY NOT NULL,
"user_id" uuid NOT NULL,
"name" varchar(255) NOT NULL,
"description" varchar(512) DEFAULT NULL,
"is_public" boolean DEFAULT false NOT NULL,
"created_at" timestamp(0) NOT NULL,
"updated_at" timestamp(0) DEFAULT NULL
);
--> statement-breakpoint
CREATE TABLE "login_attempt" (
"id" uuid PRIMARY KEY NOT NULL,
"user_id" uuid NOT NULL,
"created_at" timestamp(0) NOT NULL
);
--> statement-breakpoint
CREATE TABLE "login_history" (
"id" uuid PRIMARY KEY NOT NULL,
"user_id" uuid NOT NULL,
"ip_address" "inet",
"created_at" timestamp(0) NOT NULL,
"device_operating_system" varchar(255) DEFAULT NULL,
"device_client" varchar(255) DEFAULT NULL,
"device_device" varchar(255) DEFAULT NULL,
"device_is_bot" boolean DEFAULT false NOT NULL,
"location_time_zone" varchar(255) DEFAULT NULL,
"location_longitude" double precision,
"location_latitude" double precision,
"location_accuracy_radius" integer
);
--> statement-breakpoint
CREATE TABLE "verification_token" (
"id" uuid PRIMARY KEY NOT NULL,
"user_id" uuid NOT NULL,
"purpose" varchar(255) NOT NULL,
"created_at" timestamp(0) NOT NULL,
"token" varchar(60) DEFAULT NULL
);
--> statement-breakpoint
CREATE TABLE "followed_source" (
"id" uuid PRIMARY KEY NOT NULL,
"follower_id" uuid NOT NULL,
"source_id" uuid NOT NULL,
"created_at" timestamp(0) NOT NULL
);
--> statement-breakpoint
CREATE TABLE "comment" (
"id" uuid PRIMARY KEY NOT NULL,
"user_id" uuid NOT NULL,
"article_id" uuid NOT NULL,
"content" varchar(512) NOT NULL,
"sentiment" varchar(30) DEFAULT 'neutral' NOT NULL,
"is_spam" boolean DEFAULT false NOT NULL,
"created_at" timestamp(0) NOT NULL
);
--> statement-breakpoint
CREATE TABLE "refresh_tokens" (
"id" integer PRIMARY KEY NOT NULL,
"refresh_token" varchar(128) NOT NULL,
"username" varchar(255) NOT NULL,
"valid" timestamp(0) NOT NULL
);
--> statement-breakpoint
CREATE TABLE "article" (
"id" uuid PRIMARY KEY NOT NULL,
"source_id" uuid NOT NULL,
"title" varchar(1024) NOT NULL,
"body" text NOT NULL,
"hash" varchar(32) NOT NULL,
"categories" text[],
"sentiment" varchar(30) DEFAULT 'neutral' NOT NULL,
"metadata" jsonb,
"image" varchar(1024) GENERATED ALWAYS AS ((metadata ->> 'image'::text)) STORED,
"excerpt" varchar(255) GENERATED ALWAYS AS (("left"(body, 200) || '...'::text)) STORED,
"published_at" timestamp(0) NOT NULL,
"crawled_at" timestamp(0) NOT NULL,
"updated_at" timestamp(0) DEFAULT NULL,
"link" varchar(1024) NOT NULL,
"bias" varchar(30) DEFAULT 'neutral' NOT NULL,
"reliability" varchar(30) DEFAULT 'reliable' NOT NULL,
"transparency" varchar(30) DEFAULT 'medium' NOT NULL,
"reading_time" integer DEFAULT 1,
"tsv" "tsvector" GENERATED ALWAYS AS ((setweight(to_tsvector('french'::regconfig, (COALESCE(title, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char"))) STORED,
"token_statistics" jsonb,
CONSTRAINT "chk_article_reading_time" CHECK (reading_time >= 0),
CONSTRAINT "chk_article_sentiment" CHECK ((sentiment)::text = ANY ((ARRAY['positive'::character varying, 'neutral'::character varying, 'negative'::character varying])::text[])),
CONSTRAINT "chk_article_metadata_json" CHECK ((metadata IS NULL) OR (jsonb_typeof(metadata) = ANY (ARRAY['object'::text, 'array'::text])))
);
--> statement-breakpoint
CREATE TABLE "user" (
"id" uuid PRIMARY KEY NOT NULL,
"name" varchar(255) NOT NULL,
"email" varchar(255) NOT NULL,
"password" varchar(512) NOT NULL,
"is_locked" boolean DEFAULT false NOT NULL,
"is_confirmed" boolean DEFAULT false NOT NULL,
"created_at" timestamp(0) NOT NULL,
"updated_at" timestamp(0) DEFAULT NULL,
"roles" jsonb NOT NULL,
CONSTRAINT "chk_user_roles_json" CHECK (jsonb_typeof(roles) = 'array'::text)
);
--> statement-breakpoint
CREATE TABLE "source" (
"id" uuid PRIMARY KEY NOT NULL,
"url" varchar(255) NOT NULL,
"name" varchar(255) NOT NULL,
"display_name" varchar(255) DEFAULT NULL,
"description" varchar(1024) DEFAULT NULL,
"updated_at" timestamp(0) DEFAULT NULL,
"bias" varchar(30) DEFAULT 'neutral' NOT NULL,
"reliability" varchar(30) DEFAULT 'reliable' NOT NULL,
"transparency" varchar(30) DEFAULT 'medium' NOT NULL
);
--> statement-breakpoint
CREATE TABLE "bookmark_article" (
"bookmark_id" uuid NOT NULL,
"article_id" uuid NOT NULL,
CONSTRAINT "bookmark_article_pkey" PRIMARY KEY("bookmark_id","article_id")
);
--> statement-breakpoint
ALTER TABLE "bookmark" ADD CONSTRAINT "fk_da62921da76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "login_attempt" ADD CONSTRAINT "fk_8c11c1ba76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "login_history" ADD CONSTRAINT "fk_37976e36a76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "verification_token" ADD CONSTRAINT "fk_c1cc006ba76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_7a763a3eac24f853" FOREIGN KEY ("follower_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_7a763a3e953c1c61" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "comment" ADD CONSTRAINT "fk_9474526ca76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "comment" ADD CONSTRAINT "fk_9474526c7294869c" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "article" ADD CONSTRAINT "fk_23a0e66953c1c61" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_6fe2655d92741d25" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmark"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_6fe2655d7294869c" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "idx_bookmark_user_created" ON "bookmark" USING btree ("user_id" timestamp_ops,"created_at" timestamp_ops);--> statement-breakpoint
CREATE INDEX "idx_da62921da76ed395" ON "bookmark" USING btree ("user_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_8c11c1ba76ed395" ON "login_attempt" USING btree ("user_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_login_attempt_created_at" ON "login_attempt" USING btree ("created_at" timestamp_ops);--> statement-breakpoint
CREATE INDEX "idx_37976e36a76ed395" ON "login_history" USING btree ("user_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_login_history_created_at" ON "login_history" USING btree ("user_id" uuid_ops,"created_at" timestamp_ops);--> statement-breakpoint
CREATE INDEX "idx_login_history_ip_address" ON "login_history" USING btree ("ip_address" inet_ops);--> statement-breakpoint
CREATE INDEX "idx_c1cc006ba76ed395" ON "verification_token" USING btree ("user_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_verif_token_created_at" ON "verification_token" USING btree ("created_at" timestamp_ops);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_verif_user_purpose_token" ON "verification_token" USING btree ("user_id" text_ops,"purpose" text_ops) WHERE (token IS NOT NULL);--> statement-breakpoint
CREATE INDEX "idx_7a763a3e953c1c61" ON "followed_source" USING btree ("source_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_7a763a3eac24f853" ON "followed_source" USING btree ("follower_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_followed_source_follower_created" ON "followed_source" USING btree ("follower_id" timestamp_ops,"created_at" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_9474526c7294869c" ON "comment" USING btree ("article_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_9474526ca76ed395" ON "comment" USING btree ("user_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_comment_article_created" ON "comment" USING btree ("article_id" timestamp_ops,"created_at" uuid_ops);--> statement-breakpoint
CREATE UNIQUE INDEX "uniq_9bace7e1c74f2195" ON "refresh_tokens" USING btree ("refresh_token" text_ops);--> statement-breakpoint
CREATE INDEX "gin_article_categories" ON "article" USING gin ("categories" array_ops);--> statement-breakpoint
CREATE INDEX "gin_article_link_trgm" ON "article" USING gin ("link" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "gin_article_title_trgm" ON "article" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "gin_article_tsv" ON "article" USING gin ("tsv" tsvector_ops);--> statement-breakpoint
CREATE INDEX "idx_23a0e66953c1c61" ON "article" USING btree ("source_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_article_published_at" ON "article" USING btree ("published_at" timestamp_ops);--> statement-breakpoint
CREATE INDEX "idx_article_published_id" ON "article" USING btree ("published_at" timestamp_ops,"id" uuid_ops);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_article_hash" ON "article" USING btree ("hash" text_ops);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_user_email" ON "user" USING btree (lower((email)::text) text_ops);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_source_name" ON "source" USING btree (lower((name)::text) text_ops);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_source_url" ON "source" USING btree (lower((url)::text) text_ops);--> statement-breakpoint
CREATE INDEX "idx_6fe2655d7294869c" ON "bookmark_article" USING btree ("article_id" uuid_ops);--> statement-breakpoint
CREATE INDEX "idx_6fe2655d92741d25" ON "bookmark_article" USING btree ("bookmark_id" uuid_ops);
*/
+3
View File
@@ -0,0 +1,3 @@
-- Custom SQL migration file, put your code below! --
CREATE EXTENSION IF NOT EXISTS pg_trgm;
SET SESSION TIME ZONE 'UTC';
+154
View File
@@ -0,0 +1,154 @@
CREATE TYPE "public"."bias" AS ENUM('neutral', 'slightly', 'partisan', 'extreme');--> statement-breakpoint
CREATE TYPE "public"."reliability" AS ENUM('trusted', 'reliable', 'average', 'low_trust', 'unreliable');--> statement-breakpoint
CREATE TYPE "public"."sentiment" AS ENUM('positive', 'neutral', 'negative');--> statement-breakpoint
CREATE TYPE "public"."token_purpose" AS ENUM('confirm_account', 'password_reset', 'unlock_account', 'delete_account');--> statement-breakpoint
CREATE TYPE "public"."transparency" AS ENUM('high', 'medium', 'low');--> statement-breakpoint
CREATE TABLE "article" (
"body" text NOT NULL,
"categories" text[],
"crawled_at" timestamp DEFAULT now() NOT NULL,
"credibility" jsonb,
"excerpt" varchar(255) GENERATED ALWAYS AS (("left"(body, 200) || '...'::text)) STORED,
"hash" varchar(32) NOT NULL,
"id" uuid PRIMARY KEY NOT NULL,
"image" varchar(1024) GENERATED ALWAYS AS ((metadata ->> 'image'::text)) STORED,
"link" varchar(1024) NOT NULL,
"metadata" jsonb,
"published_at" timestamp NOT NULL,
"reading_time" integer DEFAULT 1,
"sentiment" "sentiment" NOT NULL,
"source_id" uuid NOT NULL,
"title" varchar(1024) NOT NULL,
"token_statistics" jsonb,
"tsv" "tsvector" GENERATED ALWAYS AS ((
setweight(to_tsvector('french'::regconfig, COALESCE(title, '')::text), 'A'::"char")
|| setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char")
)) STORED,
"updated_at" timestamp
);
--> statement-breakpoint
CREATE TABLE "bookmark" (
"created_at" timestamp DEFAULT now() NOT NULL,
"description" varchar(512),
"id" uuid PRIMARY KEY NOT NULL,
"is_public" boolean DEFAULT false NOT NULL,
"name" varchar(255) NOT NULL,
"updated_at" timestamp,
"user_id" uuid NOT NULL
);
--> statement-breakpoint
CREATE TABLE "bookmark_article" (
"article_id" uuid NOT NULL,
"bookmark_id" uuid NOT NULL,
CONSTRAINT "bookmark_article_pkey" PRIMARY KEY("bookmark_id","article_id")
);
--> statement-breakpoint
CREATE TABLE "comment" (
"article_id" uuid NOT NULL,
"content" varchar(512) NOT NULL,
"created_at" timestamp DEFAULT now() NOT NULL,
"id" uuid PRIMARY KEY NOT NULL,
"is_spam" boolean DEFAULT false NOT NULL,
"sentiment" "sentiment" NOT NULL,
"user_id" uuid NOT NULL
);
--> statement-breakpoint
CREATE TABLE "followed_source" (
"created_at" timestamp DEFAULT now() NOT NULL,
"follower_id" uuid NOT NULL,
"id" uuid PRIMARY KEY NOT NULL,
"source_id" uuid NOT NULL
);
--> statement-breakpoint
CREATE TABLE "login_attempt" (
"created_at" timestamp DEFAULT now() NOT NULL,
"id" uuid PRIMARY KEY NOT NULL,
"user_id" uuid NOT NULL
);
--> statement-breakpoint
CREATE TABLE "login_history" (
"created_at" timestamp DEFAULT now() NOT NULL,
"device" jsonb,
"id" uuid PRIMARY KEY NOT NULL,
"ip_address" "inet",
"location" jsonb,
"user_id" uuid NOT NULL
);
--> statement-breakpoint
CREATE TABLE "refresh_token" (
"id" uuid PRIMARY KEY NOT NULL,
"token" varchar(128) NOT NULL,
"username" varchar(255) NOT NULL,
"valid" timestamp NOT NULL
);
--> statement-breakpoint
CREATE TABLE "source" (
"credibility" jsonb,
"description" varchar(1024),
"display_name" varchar(255),
"id" uuid PRIMARY KEY NOT NULL,
"name" varchar(255) NOT NULL,
"updated_at" timestamp,
"url" varchar(255) NOT NULL
);
--> statement-breakpoint
CREATE TABLE "user" (
"created_at" timestamp DEFAULT now() NOT NULL,
"email" varchar(255) NOT NULL,
"id" uuid PRIMARY KEY NOT NULL,
"is_confirmed" boolean DEFAULT false NOT NULL,
"is_locked" boolean DEFAULT false NOT NULL,
"name" varchar(255) NOT NULL,
"password" varchar(512) NOT NULL,
"roles" varchar(255)[] DEFAULT '{"ROLE_USER"}' NOT NULL,
"updated_at" timestamp
);
--> statement-breakpoint
CREATE TABLE "verification_token" (
"created_at" timestamp DEFAULT now() NOT NULL,
"id" uuid PRIMARY KEY NOT NULL,
"purpose" "token_purpose" NOT NULL,
"token" varchar(60),
"user_id" uuid NOT NULL
);
--> statement-breakpoint
ALTER TABLE "article" ADD CONSTRAINT "fk_article_source_id" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "bookmark" ADD CONSTRAINT "fk_bookmark_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_bookmark_article_bookmark_id" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmark"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_bookmark_article_article_id" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "comment" ADD CONSTRAINT "fk_comment_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "comment" ADD CONSTRAINT "fk_comment_article_id" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_followed_source_follower_id" FOREIGN KEY ("follower_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_followed_source_source_id" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "login_attempt" ADD CONSTRAINT "fk_login_attempt_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "login_history" ADD CONSTRAINT "fk_login_history_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
ALTER TABLE "verification_token" ADD CONSTRAINT "fk_verification_token_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
CREATE INDEX "gin_article_categories" ON "article" USING gin ("categories" array_ops);--> statement-breakpoint
CREATE INDEX "gin_article_link_trgm" ON "article" USING gin ("link" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "gin_article_title_trgm" ON "article" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
CREATE INDEX "gin_article_tsv" ON "article" USING gin ("tsv" tsvector_ops);--> statement-breakpoint
CREATE INDEX "idx_article_source_published_id" ON "article" USING btree ("source_id","published_at" DESC NULLS FIRST,"id" DESC NULLS FIRST);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_article_hash" ON "article" USING btree ("hash");--> statement-breakpoint
CREATE INDEX "idx_bookmark_user_created" ON "bookmark" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_bookmark_user_name" ON "bookmark" USING btree ("user_id",lower("name"));--> statement-breakpoint
CREATE INDEX "idx_bookmark_article_bookmark_id" ON "bookmark_article" USING btree ("bookmark_id");--> statement-breakpoint
CREATE INDEX "idx_comment_article_id" ON "comment" USING btree ("article_id");--> statement-breakpoint
CREATE INDEX "idx_comment_user_id" ON "comment" USING btree ("user_id");--> statement-breakpoint
CREATE INDEX "idx_comment_article_created" ON "comment" USING btree ("article_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
CREATE INDEX "idx_followed_source_source_id" ON "followed_source" USING btree ("source_id");--> statement-breakpoint
CREATE INDEX "idx_followed_source_follower_id" ON "followed_source" USING btree ("follower_id");--> statement-breakpoint
CREATE INDEX "idx_followed_source_follower_created" ON "followed_source" USING btree ("follower_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_followed_source_user_source" ON "followed_source" USING btree ("follower_id","source_id");--> statement-breakpoint
CREATE INDEX "idx_login_attempt_user_created" ON "login_attempt" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
CREATE INDEX "idx_login_history_user_created" ON "login_history" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
CREATE INDEX "idx_login_history_ip_address" ON "login_history" USING btree ("ip_address");--> statement-breakpoint
CREATE UNIQUE INDEX "uniq_refresh_token_token" ON "refresh_token" USING btree ("token");--> statement-breakpoint
CREATE INDEX "idx_refresh_token_valid" ON "refresh_token" USING btree ("valid");--> statement-breakpoint
CREATE INDEX "idx_refresh_token_username" ON "refresh_token" USING btree (lower("username"));--> statement-breakpoint
CREATE UNIQUE INDEX "unq_source_name" ON "source" USING btree (lower((name)::text));--> statement-breakpoint
CREATE UNIQUE INDEX "unq_source_url" ON "source" USING btree (lower((url)::text));--> statement-breakpoint
CREATE UNIQUE INDEX "unq_user_email" ON "user" USING btree (lower((email)::text));--> statement-breakpoint
CREATE INDEX "idx_user_created_at" ON "user" USING btree (created_at);--> statement-breakpoint
CREATE INDEX "idx_verif_token_created_at" ON "verification_token" USING btree ("created_at" DESC NULLS FIRST);--> statement-breakpoint
CREATE UNIQUE INDEX "unq_verif_user_purpose_token" ON "verification_token" USING btree ("user_id","purpose","token") WHERE "verification_token"."token" IS NOT NULL;--> statement-breakpoint
CREATE UNIQUE INDEX "unq_verif_token_token" ON "verification_token" USING btree ("token") WHERE "verification_token"."token" IS NOT NULL;
File diff suppressed because it is too large Load Diff
File diff suppressed because it is too large Load Diff
+9 -2
View File
@@ -4,9 +4,16 @@
{
"breakpoints": true,
"idx": 0,
"tag": "0000_aromatic_dorian_gray",
"tag": "0000_setup",
"version": "7",
"when": 1762691204645
"when": 1762775141000
},
{
"breakpoints": true,
"idx": 1,
"tag": "0001_init",
"version": "7",
"when": 1762775267679
}
],
"version": "7"
+7 -1
View File
@@ -1,10 +1,14 @@
{
"dependencies": {
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"@date-fns/utc": "^2.1.1",
"drizzle-orm": "^0.44.7",
"mysql2": "^3.15.3",
"pg": "^8.16.3",
"snakecase-keys": "^9.0.2"
"snakecase-keys": "^9.0.2",
"tiktoken": "^1.0.22",
"uuid": "^13.0.0"
},
"devDependencies": {
"@types/pg": "^8.15.6",
@@ -13,6 +17,7 @@
"exports": {
".": "./src/index.ts",
"./client": "./src/client.ts",
"./importer": "./src/importer/index.ts",
"./queries": "./src/queries/index.ts",
"./schema": "./src/schema.ts",
"./utils": "./src/utils/index.ts"
@@ -21,6 +26,7 @@
"private": true,
"scripts": {
"clean": "rm -rf .turbo node_modules",
"sync:import": "bun ./src/importer/import.ts",
"typecheck": "tsc --noEmit"
}
}
+6
View File
@@ -0,0 +1,6 @@
export class NotFoundError extends Error {
constructor(message: string) {
super(message);
this.name = "NotFoundError";
}
}
+468
View File
@@ -0,0 +1,468 @@
import { RowDataPacket } from "mysql2/promise";
import { Pool, PoolClient } from "pg";
import { computeReadingTime, computeTokenStatistics } from "@/utils/computed";
type SourceOptions = {
host: string;
user: string;
password: string;
database: string;
};
type TargetOptions = {
database: string;
batchSize?: number;
pageSize?: number;
ignoreColumns?: Record<string, string[]>;
};
const DEFAULT_IGNORE: Record<string, string[]> = {
article: ["tsv", "image", "excerpt", "bias", "reliability", "transparency"],
source: ["bias", "reliability", "transparency"],
};
/**
* Engine
*
* Coordinates copying rows from a MySQL source into a PostgreSQL target in a
* controlled, transactional, batched manner.
*
* Responsibilities:
* - Establish and manage a connection pool to the target PostgreSQL database.
* - Stream rows from a MySQL source (via a temporary pool) using pagination.
* - Transform row values to match target expectations (UUID normalization,
* timestamp fallback, array parsing for categories/roles, computed JSON
* credibility, etc.).
* - Filter out ignored columns based on a configurable ignore map.
* - Insert rows into the target in configurable batch sizes with transactional
* commits every batch to limit long-running transactions.
* - Provide a safe reset operation that truncates the target table and manages
* session replication role toggling for Postgres.
*
* @param sourceOptions - connection and authentication options for the MySQL
* source (database, host, user, password, etc.).
* @param targetOptions - configuration for the Postgres target including
* connection string (database), optional pageSize, batchSize and per-table
* ignoreColumns map.
*/
export class Engine {
private readonly target: Pool;
private readonly ignore: Record<string, string[]>;
private readonly pageSize: number;
private readonly batchSize: number;
constructor(
private readonly sourceOptions: SourceOptions,
private readonly targetOptions: TargetOptions,
) {
this.target = new Pool({
allowExitOnIdle: true,
connectionString: this.targetOptions.database,
max: 8,
});
this.ignore = { ...DEFAULT_IGNORE, ...(this.targetOptions.ignoreColumns ?? {}) };
this.pageSize = this.targetOptions.pageSize ?? 10_000;
this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 1000);
console.log(
`Engine initialized with pageSize=${this.pageSize} and batchSize=${this.batchSize}`,
);
}
async close() {
await this.target.end();
}
async import(table: string): Promise<number> {
await this.reset(table);
return await this.paste(table, this.copy(table));
}
private async *copy(table: string): AsyncGenerator<Record<string, unknown>> {
const mysql = await import("mysql2/promise");
const source = mysql.createPool({
database: this.sourceOptions.database,
host: this.sourceOptions.host,
idleTimeout: 180_000_000,
password: this.sourceOptions.password,
port: 3306,
rowsAsArray: false,
user: this.sourceOptions.user,
});
let offset = 0;
const size = this.pageSize;
try {
while (true) {
const [rows] = await source.query<RowDataPacket[]>(
`SELECT * FROM \`${this.escapeBacktick(table)}\` LIMIT ? OFFSET ?`,
[size, offset],
);
if (!rows || rows.length === 0) break;
for (const row of rows) {
yield row as Record<string, unknown>;
}
offset += rows.length;
if (rows.length < size) break;
}
} finally {
try {
await source.end();
} catch {}
}
}
private async paste(
table: string,
rows: AsyncGenerator<Record<string, unknown>>,
): Promise<number> {
const target = await this.target.connect();
let total = 0;
let inBatch = 0;
let columns: string[] | null = null;
let insertSql = "";
const ignored = this.ignoredColumnsFor(table);
const ignoredSet = new Set(ignored);
try {
for await (let row of rows) {
if (!columns) {
row = this.transformRowForTarget(table, row);
// Filter ignored columns and build column order
columns = Object.keys(row).filter((c) => !ignoredSet.has(c));
// If article target has credibility but source not, include computed credibility
if (
(this.normalizedName(table) === "article" && !columns.includes("credibility")) ||
(this.normalizedName(table) === "source" && !columns.includes("credibility"))
) {
columns.push("credibility");
}
if (this.normalizedName(table) === "article" && !columns.includes("token_statistics")) {
columns.push("token_statistics");
}
const colsSql = columns.map((c) => this.quote(c)).join(", ");
const placeholders = columns.map((_, i) => `$${i + 1}`).join(", ");
insertSql = `INSERT INTO ${this.quote(table)} (${colsSql}) VALUES (${placeholders})`;
await target.query("BEGIN");
}
// Row transform and params in column order
const transformed = this.transformRowForTarget(table, row);
const params = columns!.map((c) => this.valueForColumn(c, transformed));
try {
await target.query(insertSql, params);
} catch (err: unknown) {
const msg = String((err as Error)?.message ?? "");
if (msg.includes("invalid input syntax for type timestamp")) {
// Fallback: coerce all *_at params to now() and retry once
const fixed = columns!.map((c, i) => (c.endsWith("_at") ? new Date() : params[i]));
await target.query(insertSql, fixed);
} else {
throw err;
}
}
total++;
inBatch++;
if (inBatch >= this.batchSize) {
await target.query("COMMIT");
inBatch = 0;
await target.query("BEGIN");
console.log(`Imported ${total} records into ${table} so far...`);
}
}
if (inBatch > 0) {
await target.query("COMMIT");
}
} catch (e) {
await safeRollback(target);
throw e;
} finally {
target.release();
}
return total;
}
private normalizedName(table: string): string {
return table.replaceAll('"', "").replaceAll("`", "").toLowerCase();
}
private ignoredColumnsFor(table: string): string[] {
return this.ignore[this.normalizedName(table)] ?? [];
}
private async reset(table: string) {
const client = await this.target.connect();
try {
await client.query("BEGIN");
await client.query("SET session_replication_role = 'replica'");
await client.query(`TRUNCATE TABLE ${this.quote(table)} RESTART IDENTITY CASCADE`);
await client.query("SET session_replication_role = 'origin'");
await client.query("COMMIT");
console.log(`Reset completed for table ${table}`);
} catch (e) {
await safeRollback(client);
throw e;
} finally {
client.release();
}
}
private transformRowForTarget(table: string, row: Record<string, unknown>) {
const t = this.normalizedName(table);
const clone: Record<string, unknown> = { ...row };
// Normalize UUIDs and timestamps and categories
for (const [key, val] of Object.entries(clone)) {
if (val == null) continue;
if (key === "id" || key.endsWith("_id")) {
clone[key] = this.normalizeUuidValue(val);
continue;
}
// Robust timestamp normalization for *_at columns
if (key.endsWith("_at")) {
clone[key] = this.normalizeTimestampValue(val);
continue;
}
if (key === "categories") {
if (Array.isArray(val)) {
clone[key] = val;
} else if (typeof val === "string") {
const raw = val.trim();
// Try JSON first
if (raw.startsWith("[") && raw.endsWith("]")) {
try {
const parsed = JSON.parse(raw);
if (Array.isArray(parsed)) {
clone[key] = parsed;
continue;
}
} catch {}
}
const parts = raw
.split(",")
.map((s) => s.trim())
.filter(Boolean);
clone[key] = parts.length ? parts : null;
}
}
if (t === "article" && key === "token_statistics") {
clone[key] = computeTokenStatistics({
body: String(clone.body ?? ""),
categories: Array.isArray(clone.categories) ? clone.categories : [],
title: String(clone.title ?? ""),
});
}
if (t === "article" && key === "reading_time") {
clone[key] = Math.max(1, computeReadingTime(String(clone.body ?? "")));
}
if (key === "roles") {
if (Array.isArray(val)) {
clone[key] = val;
} else if (typeof val === "string") {
const raw = val.trim();
// If the value is a JSON array string like '["ROLE_USER","ROLE_ADMIN"]', parse it.
try {
const parsed = JSON.parse(raw);
if (Array.isArray(parsed)) {
clone[key] = parsed;
continue;
}
} catch {
// not JSON, fall back to CSV-like parsing below
}
// Remove surrounding brackets/quotes then split by comma and strip quotes/space
const parts = raw
.replace(/^\[|\]$/g, "")
.split(",")
.map((s) => s.replace(/^["']|["']$/g, "").trim())
.filter(Boolean);
clone[key] = parts.length ? parts : ["ROLE_USER"];
}
}
}
// compute credibility JSON if bias/reliability/transparency present
if (t === "article" || t === "source") {
const bias = clone.bias ?? null;
const reliability = clone.reliability ?? null;
const transparency = clone.transparency ?? null;
if (bias || reliability || transparency) {
clone.credibility = {
bias,
reliability,
transparency,
};
}
}
// Ensure article token_statistics exists (computed on the fly)
if (
t === "article" &&
(clone.token_statistics == null || typeof clone.token_statistics !== "object")
) {
clone.token_statistics = computeTokenStatistics({
body: String(clone.body ?? ""),
categories: Array.isArray(clone.categories) ? (clone.categories as string[]) : [],
title: String(clone.title ?? ""),
});
}
return clone;
}
private valueForColumn(col: string, row: Record<string, unknown>) {
const v = row[col];
// Pass Date objects directly to pg for timestamp columns
if (col.endsWith("_at") && v instanceof Date) {
return v;
}
if (col === "credibility" && v && typeof v === "object") {
return JSON.stringify(v);
}
if (col === "token_statistics" && v && typeof v === "object") {
return JSON.stringify(v);
}
if (col === "device" && v && typeof v === "object") {
return JSON.stringify(v);
}
if (col === "location" && v && typeof v === "object") {
return JSON.stringify(v);
}
if (col === "roles" && v) {
return JSON.stringify(v);
}
if (col === "metadata" && v && typeof v === "object") {
return JSON.stringify(v);
}
return v ?? null;
}
private normalizeUuidValue(value: unknown): string {
if (Buffer.isBuffer(value)) {
return bufferToUuid(value);
}
if (typeof value === "string") {
// Already a UUID string or hex; try to format 32-hex into canonical form
const hex = value.replace(/-/g, "").toLowerCase();
if (/^[0-9a-f]{32}$/.test(hex)) {
return (
hex.slice(0, 8) +
"-" +
hex.slice(8, 12) +
"-" +
hex.slice(12, 16) +
"-" +
hex.slice(16, 20) +
"-" +
hex.slice(20)
);
}
return value;
}
return String(value);
}
private normalizeTimestampValue(value: unknown): Date {
// If it's already a Date, ensure it's valid
if (value instanceof Date) {
return Number.isNaN(value.getTime()) ? new Date() : value;
}
// Strings: handle common invalid patterns and attempt safe parsing
if (typeof value === "string") {
const raw = value.trim();
if (
!raw ||
/0000-00-00/.test(raw) ||
/NaN/.test(raw) ||
raw.toLowerCase() === "invalid date"
) {
return new Date();
}
// Normalize MySQL-like 'YYYY-MM-DD HH:MM:SS[.ffffff]' to ISO
let s = raw.replace(" ", "T");
// Reduce microseconds to milliseconds (3 digits) if present
s = s.replace(/\.(\d{3})\d+$/, ".$1");
// Append Z if there is no timezone info
if (!/[zZ]|[+-]\d{2}:?\d{2}$/.test(s)) s += "Z";
const d = new Date(s);
if (!Number.isNaN(d.getTime())) return d;
// Try numeric string as epoch seconds/millis
const n = Number(raw);
if (Number.isFinite(n)) {
const ms = n > 1e12 ? n : n * 1000;
const d2 = new Date(ms);
if (!Number.isNaN(d2.getTime())) return d2;
}
return new Date();
}
// Numbers: treat as epoch seconds/millis
if (typeof value === "number" && Number.isFinite(value)) {
const ms = value > 1e12 ? value : value * 1000;
const d = new Date(ms);
return Number.isNaN(d.getTime()) ? new Date() : d;
}
// Fallback: now
return new Date();
}
private quote(id: string) {
const norm = this.normalizedName(id);
return `"${norm.replaceAll('"', '""')}"`;
}
private escapeBacktick(id: string) {
return id.replaceAll("`", "``");
}
}
function bufferToUuid(buf: Buffer): string {
if (buf.length !== 16) return buf.toString("hex");
const hex = buf.toString("hex");
return (
hex.slice(0, 8) +
"-" +
hex.slice(8, 12) +
"-" +
hex.slice(12, 16) +
"-" +
hex.slice(16, 20) +
"-" +
hex.slice(20)
);
}
async function safeRollback(client: PoolClient) {
try {
await client.query("ROLLBACK");
} catch {}
}
+65
View File
@@ -0,0 +1,65 @@
#!/usr/bin/env bun
import { stdin as input, stdout as output } from "node:process";
import { createInterface } from "node:readline/promises";
import { createEnvAccessor } from "@devscast/config";
import { Engine } from "@/importer";
const env = createEnvAccessor([
"BASANGO_SOURCE_DATABASE_HOST",
"BASANGO_SOURCE_DATABASE_USER",
"BASANGO_SOURCE_DATABASE_PASS",
"BASANGO_SOURCE_DATABASE_NAME",
"BASANGO_DATABASE_URL",
]);
async function promptConfirm(question: string, def = false) {
const rl = createInterface({ input, output });
const suffix = def ? "[Y/n]" : "[y/N]";
const answer = await rl.question(`${question} ${suffix} `);
rl.close();
const v = String(answer || "")
.trim()
.toLowerCase();
if (v === "y" || v === "yes") return true;
if (v === "n" || v === "no") return false;
return def;
}
async function main() {
const ok = await promptConfirm("Do you want to continue?", false);
if (!ok) {
console.warn("Process aborted");
process.exit(1);
}
const engine = new Engine(
{
database: env("BASANGO_SOURCE_DATABASE_NAME"),
host: env("BASANGO_SOURCE_DATABASE_HOST"),
password: env("BASANGO_SOURCE_DATABASE_PASS"),
user: env("BASANGO_SOURCE_DATABASE_USER"),
},
{
database: env("BASANGO_DATABASE_URL"),
},
);
try {
const tables = process.argv.slice(2);
if (tables.length === 0) tables.push("user", "source", "article");
for (const t of tables) {
const count = await engine.import(t);
console.log(`Imported ${count} records into ${t} table.`);
}
console.log("Import completed successfully");
} finally {
await engine.close();
}
}
main().catch((err) => {
console.error(err?.message ?? err);
process.exit(1);
});
+2
View File
@@ -0,0 +1,2 @@
export * from "./engine";
export * from "./import";
+65
View File
@@ -0,0 +1,65 @@
import { md5 } from "@basango/encryption";
import { eq } from "drizzle-orm";
import { v7 as uuidV7 } from "uuid";
import { Database } from "@/client";
import { ArticleMetadata, Sentiment, TokenStatistics, article } from "@/schema";
import { computeReadingTime, computeTokenStatistics } from "@/utils/computed";
import { getSourceIdByName } from "./sources";
export type CreateArticleParams = {
title: string;
body: string;
categories: string[];
link: string;
sourceId: string;
publishedAt: Date;
sentiment?: Sentiment;
tokenStatistics?: TokenStatistics;
readingTime?: number;
metadata?: ArticleMetadata;
};
export async function createArticle(db: Database, params: CreateArticleParams) {
const data = {
...params,
hash: md5(params.link),
readingTime: computeReadingTime(params.body),
sentiment: "neutral" as Sentiment,
sourceId: await getSourceIdByName(db, params.sourceId),
tokenStatistics: computeTokenStatistics({
body: params.body,
categories: params.categories,
title: params.title,
}),
};
const duplicated = await getArticleByHash(db, data.hash);
if (duplicated !== undefined) {
return {
id: duplicated.id,
sourceId: duplicated.sourceId,
};
}
const [result] = await db
.insert(article)
.values({ id: uuidV7(), ...data })
.returning({
id: article.id,
sourceId: article.sourceId,
});
if (result === undefined) {
throw new Error("Failed to create article");
}
return result;
}
export async function getArticleByHash(db: Database, hash: string) {
return db.query.article.findFirst({
where: eq(article.hash, hash),
});
}
+65
View File
@@ -0,0 +1,65 @@
import { eq } from "drizzle-orm";
import { v7 as uuidV7 } from "uuid";
import { Database } from "@/client";
import { NotFoundError } from "@/errors";
import { Credibility, source } from "@/schema";
export type CreateSourceParams = {
name: string;
url: string;
displayName?: string;
description?: string;
credibility: Credibility;
updatedAt?: Date;
};
export async function createSource(db: Database, params: CreateSourceParams) {
const [result] = await db
.insert(source)
.values({ id: uuidV7(), ...params })
.returning();
return result;
}
export type DeleteSourceParams = {
id: string;
};
export async function deleteSource(db: Database, params: DeleteSourceParams) {
const [result] = await db.delete(source).where(eq(source.id, params.id)).returning();
return result;
}
export async function getSourceByName(db: Database, name: string) {
return db.query.source.findFirst({
where: eq(source.name, name),
});
}
export async function getSourceIdByName(db: Database, name: string): Promise<string> {
const result = await db.query.source.findFirst({
columns: {
id: true,
},
where: eq(source.name, name),
});
if (!result) {
throw new NotFoundError(`Source with name "${name}" not found`);
}
return result.id;
}
export type GetSourceByIdParams = {
id: string;
};
export async function getSourceById(db: Database, params: GetSourceByIdParams) {
return db.query.source.findFirst({
where: eq(source.id, params.id),
});
}
+334 -253
View File
@@ -1,15 +1,14 @@
import { relations, sql } from "drizzle-orm";
import { check } from "drizzle-orm/gel-core";
import {
boolean,
check,
customType,
doublePrecision,
foreignKey,
index,
inet,
integer,
jsonb,
pgSequence,
pgEnum,
pgTable,
primaryKey,
text,
@@ -19,242 +18,181 @@ import {
varchar,
} from "drizzle-orm/pg-core";
const tsvector = customType<{ data: string; driverData: string }>({
/* -------------------------------------------------------------------------- */
/* Types */
/* -------------------------------------------------------------------------- */
export const tsvector = customType<{ data: string; driverData: string }>({
dataType() {
return "tsvector";
},
});
export const refreshTokensIdSeq = pgSequence("refresh_tokens_id_seq", {
cache: "1",
cycle: false,
increment: "1",
maxValue: "9223372036854775807",
minValue: "1",
startWith: "1",
});
export const customJsonType = <T>() =>
customType<{ data: T }>({
dataType() {
return "jsonb";
},
fromDriver(value) {
return value as T;
},
toDriver(value) {
return value; // JSONB → just pass the object
},
});
// legacy table for doctrine migrations
export const doctrineMigrationVersions = pgTable("doctrine_migration_versions", {
executedAt: timestamp("executed_at", { mode: "string" }).default(sql`NULL`),
executionTime: integer("execution_time"),
version: varchar({ length: 191 }).primaryKey().notNull(),
});
export const biasEnum = pgEnum("bias", ["neutral", "slightly", "partisan", "extreme"]);
export const reliabilityEnum = pgEnum("reliability", [
"trusted",
"reliable",
"average",
"low_trust",
"unreliable",
]);
export const sentimentEnum = pgEnum("sentiment", ["positive", "neutral", "negative"]);
export const transparencyEnum = pgEnum("transparency", ["high", "medium", "low"]);
export const tokenPurposeEnum = pgEnum("token_purpose", [
"confirm_account",
"password_reset",
"unlock_account",
"delete_account",
]);
export const bookmark = pgTable(
"bookmark",
export type EmailAddress = string;
export type Link = string;
export type ReadingTime = number;
export type Role = "ROLE_USER" | "ROLE_ADMIN";
export type Roles = Role[];
export type Bias = (typeof biasEnum.enumValues)[number];
export type Reliability = (typeof reliabilityEnum.enumValues)[number];
export type Sentiment = (typeof sentimentEnum.enumValues)[number];
export type Transparency = (typeof transparencyEnum.enumValues)[number];
export type TokenPurpose = (typeof tokenPurposeEnum.enumValues)[number];
export type Credibility = {
bias: Bias;
reliability: Reliability;
transparency: Transparency;
};
export type TokenStatistics = {
title: number;
body: number;
categories: number;
excerpt: number;
total: number;
};
export type Device = {
operatingSystem?: string;
client?: string;
device?: string;
isBot: boolean;
};
export type GeoLocation = {
country?: string;
city?: string;
timeZone?: string;
longitude?: number;
latitude?: number;
accuracyRadius?: number;
};
export type ClientProfile = {
userIp?: string;
userAgent?: string;
hints: unknown[];
};
export type ArticleMetadata = {
title?: string;
description?: string;
image?: string;
};
export type DateRange = {
start: number; // unix timestamp (seconds)
end: number; // unix timestamp (seconds)
};
// Secrets
export type GeneratedToken = string;
export type GeneratedCode = string;
/* -------------------------------------------------------------------------- */
/* Tables */
/* -------------------------------------------------------------------------- */
export const user = pgTable(
"user",
{
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
description: varchar({ length: 512 }).default(sql`NULL`),
createdAt: timestamp("created_at").defaultNow().notNull(),
email: varchar({ length: 255 }).$type<EmailAddress>().notNull(),
id: uuid().primaryKey().notNull(),
isPublic: boolean("is_public").default(false).notNull(),
isConfirmed: boolean("is_confirmed").default(false).notNull(),
isLocked: boolean("is_locked").default(false).notNull(),
name: varchar({ length: 255 }).notNull(),
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
userId: uuid("user_id").notNull(),
password: varchar({ length: 512 }).notNull(),
roles: varchar("roles", { length: 255 })
.$type<Roles>()
.array()
.notNull()
.default(["ROLE_USER"]),
updatedAt: timestamp("updated_at"),
},
(table) => [
index("idx_bookmark_user_created").using(
"btree",
table.userId.asc().nullsLast().op("timestamp_ops"),
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
),
index("idx_da62921da76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_da62921da76ed395",
}).onDelete("cascade"),
(_table) => [
uniqueIndex("unq_user_email").using("btree", sql`lower((email)::text)`),
index("idx_user_created_at").using("btree", sql`created_at`),
sql`CONSTRAINT "chk_user_roles_json" CHECK (jsonb_typeof(roles) = 'array')`,
],
);
export const loginAttempt = pgTable(
"login_attempt",
export const source = pgTable(
"source",
{
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
credibility: jsonb("credibility").$type<Credibility>(),
description: varchar({ length: 1024 }),
displayName: varchar("display_name", { length: 255 }),
id: uuid().primaryKey().notNull(),
userId: uuid("user_id").notNull(),
name: varchar({ length: 255 }).notNull(),
updatedAt: timestamp("updated_at"),
url: varchar({ length: 255 }).notNull(),
},
(table) => [
index("idx_8c11c1ba76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
index("idx_login_attempt_created_at").using(
"btree",
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_8c11c1ba76ed395",
}).onDelete("cascade"),
],
);
export const loginHistory = pgTable(
"login_history",
{
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
deviceClient: varchar("device_client", { length: 255 }).default(sql`NULL`),
deviceDevice: varchar("device_device", { length: 255 }).default(sql`NULL`),
deviceIsBot: boolean("device_is_bot").default(false).notNull(),
deviceOperatingSystem: varchar("device_operating_system", { length: 255 }).default(sql`NULL`),
id: uuid().primaryKey().notNull(),
ipAddress: inet("ip_address"),
locationAccuracyRadius: integer("location_accuracy_radius"),
locationLatitude: doublePrecision("location_latitude"),
locationLongitude: doublePrecision("location_longitude"),
locationTimeZone: varchar("location_time_zone", { length: 255 }).default(sql`NULL`),
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_37976e36a76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
index("idx_login_history_created_at").using(
"btree",
table.userId.asc().nullsLast().op("uuid_ops"),
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
),
index("idx_login_history_ip_address").using(
"btree",
table.ipAddress.asc().nullsLast().op("inet_ops"),
),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_37976e36a76ed395",
}).onDelete("cascade"),
],
);
export const verificationToken = pgTable(
"verification_token",
{
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
id: uuid().primaryKey().notNull(),
purpose: varchar({ length: 255 }).notNull(),
token: varchar({ length: 60 }).default(sql`NULL`),
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_c1cc006ba76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
index("idx_verif_token_created_at").using(
"btree",
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
),
uniqueIndex("unq_verif_user_purpose_token")
.using(
"btree",
table.userId.asc().nullsLast().op("text_ops"),
table.purpose.asc().nullsLast().op("text_ops"),
)
.where(sql`(token IS NOT NULL)`),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_c1cc006ba76ed395",
}).onDelete("cascade"),
],
);
export const followedSource = pgTable(
"followed_source",
{
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
followerId: uuid("follower_id").notNull(),
id: uuid().primaryKey().notNull(),
sourceId: uuid("source_id").notNull(),
},
(table) => [
index("idx_7a763a3e953c1c61").using("btree", table.sourceId.asc().nullsLast().op("uuid_ops")),
index("idx_7a763a3eac24f853").using("btree", table.followerId.asc().nullsLast().op("uuid_ops")),
index("idx_followed_source_follower_created").using(
"btree",
table.followerId.asc().nullsLast().op("timestamp_ops"),
table.createdAt.desc().nullsFirst().op("uuid_ops"),
),
foreignKey({
columns: [table.followerId],
foreignColumns: [user.id],
name: "fk_7a763a3eac24f853",
}).onDelete("cascade"),
foreignKey({
columns: [table.sourceId],
foreignColumns: [source.id],
name: "fk_7a763a3e953c1c61",
}).onDelete("cascade"),
],
);
export const comment = pgTable(
"comment",
{
articleId: uuid("article_id").notNull(),
content: varchar({ length: 512 }).notNull(),
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
id: uuid().primaryKey().notNull(),
isSpam: boolean("is_spam").default(false).notNull(),
sentiment: varchar({ length: 30 }).default("neutral").notNull(),
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_9474526c7294869c").using("btree", table.articleId.asc().nullsLast().op("uuid_ops")),
index("idx_9474526ca76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
index("idx_comment_article_created").using(
"btree",
table.articleId.asc().nullsLast().op("timestamp_ops"),
table.createdAt.desc().nullsFirst().op("uuid_ops"),
),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_9474526ca76ed395",
}).onDelete("cascade"),
foreignKey({
columns: [table.articleId],
foreignColumns: [article.id],
name: "fk_9474526c7294869c",
}).onDelete("cascade"),
],
);
export const refreshTokens = pgTable(
"refresh_tokens",
{
id: integer().primaryKey().notNull(),
refreshToken: varchar("refresh_token", { length: 128 }).notNull(),
username: varchar({ length: 255 }).notNull(),
valid: timestamp({ mode: "string" }).notNull(),
},
(table) => [
uniqueIndex("uniq_9bace7e1c74f2195").using(
"btree",
table.refreshToken.asc().nullsLast().op("text_ops"),
),
(_table) => [
uniqueIndex("unq_source_name").using("btree", sql`lower((name)::text)`),
uniqueIndex("unq_source_url").using("btree", sql`lower((url)::text)`),
],
);
export const article = pgTable(
"article",
{
bias: varchar({ length: 30 }).default("neutral").notNull(),
body: text().notNull(),
categories: text().array(),
crawledAt: timestamp("crawled_at", { mode: "string" }).notNull(),
crawledAt: timestamp("crawled_at").defaultNow().notNull(),
credibility: jsonb("credibility").$type<Credibility>(),
excerpt: varchar({ length: 255 }).generatedAlwaysAs(sql`("left"(body, 200) || '...'::text)`),
hash: varchar({ length: 32 }).notNull(),
id: uuid().primaryKey().notNull(),
image: varchar({ length: 1024 }).generatedAlwaysAs(sql`(metadata ->> 'image'::text)`),
link: varchar({ length: 1024 }).notNull(),
metadata: jsonb(),
publishedAt: timestamp("published_at", { mode: "string" }).notNull(),
metadata: jsonb("metadata").$type<ArticleMetadata>(),
publishedAt: timestamp("published_at").notNull(),
readingTime: integer("reading_time").default(1),
reliability: varchar({ length: 30 }).default("reliable").notNull(),
sentiment: varchar({ length: 30 }).default("neutral").notNull(),
sentiment: sentimentEnum("sentiment").notNull(),
sourceId: uuid("source_id").notNull(),
title: varchar({ length: 1024 }).notNull(),
tokenStatistics: jsonb("token_statistics"),
transparency: varchar({ length: 30 }).default("medium").notNull(),
tokenStatistics: jsonb("token_statistics").$type<TokenStatistics>(),
tsv: tsvector("tsv").generatedAlwaysAs(
sql`(setweight(to_tsvector('french'::regconfig, (COALESCE(title, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char"))`,
sql`(
setweight(to_tsvector('french'::regconfig, COALESCE(title, '')::text), 'A'::"char")
|| setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char")
)`,
),
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
updatedAt: timestamp("updated_at"),
},
(table) => [
index("gin_article_categories").using(
@@ -264,69 +202,57 @@ export const article = pgTable(
index("gin_article_link_trgm").using("gin", table.link.asc().nullsLast().op("gin_trgm_ops")),
index("gin_article_title_trgm").using("gin", table.title.asc().nullsLast().op("gin_trgm_ops")),
index("gin_article_tsv").using("gin", table.tsv.asc().nullsLast().op("tsvector_ops")),
index("idx_23a0e66953c1c61").using("btree", table.sourceId.asc().nullsLast().op("uuid_ops")),
index("idx_article_published_at").using(
index("idx_article_source_published_id").using(
"btree",
table.publishedAt.desc().nullsFirst().op("timestamp_ops"),
table.sourceId.asc().nullsLast(),
table.publishedAt.desc().nullsFirst(),
table.id.desc().nullsFirst(),
),
index("idx_article_published_id").using(
"btree",
table.publishedAt.desc().nullsFirst().op("timestamp_ops"),
table.id.desc().nullsFirst().op("uuid_ops"),
),
uniqueIndex("unq_article_hash").using("btree", table.hash.asc().nullsLast().op("text_ops")),
uniqueIndex("unq_article_hash").using("btree", table.hash.asc().nullsLast()),
foreignKey({
columns: [table.sourceId],
foreignColumns: [source.id],
name: "fk_23a0e66953c1c61",
name: "fk_article_source_id",
}).onDelete("cascade"),
check("chk_article_reading_time", sql`reading_time >= 0`),
check("chk_article_reading_time", sql`(reading_time >= 0)`),
check(
"chk_article_sentiment",
sql`(sentiment)::text = ANY ((ARRAY['positive'::character varying, 'neutral'::character varying, 'negative'::character varying])::text[])`,
sql`((sentiment)::text = ANY (ARRAY['positive'::text,'neutral'::text,'negative'::text]))`,
),
check(
"chk_article_metadata_json",
sql`(metadata IS NULL) OR (jsonb_typeof(metadata) = ANY (ARRAY['object'::text, 'array'::text]))`,
sql`((metadata IS NULL) OR (jsonb_typeof(metadata) IN ('object'::text,'array'::text)))`,
),
],
);
export const user = pgTable(
"user",
export const bookmark = pgTable(
"bookmark",
{
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
email: varchar({ length: 255 }).notNull(),
createdAt: timestamp("created_at").defaultNow().notNull(),
description: varchar({ length: 512 }),
id: uuid().primaryKey().notNull(),
isConfirmed: boolean("is_confirmed").default(false).notNull(),
isLocked: boolean("is_locked").default(false).notNull(),
isPublic: boolean("is_public").default(false).notNull(),
name: varchar({ length: 255 }).notNull(),
password: varchar({ length: 512 }).notNull(),
roles: jsonb().notNull(),
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
updatedAt: timestamp("updated_at"),
userId: uuid("user_id").notNull(),
},
(_table) => [
uniqueIndex("unq_user_email").using("btree", sql`lower((email)::text)`),
check("chk_user_roles_json", sql`jsonb_typeof(roles) = 'array'::text`),
],
);
export const source = pgTable(
"source",
{
bias: varchar({ length: 30 }).default("neutral").notNull(),
description: varchar({ length: 1024 }).default(sql`NULL`),
displayName: varchar("display_name", { length: 255 }).default(sql`NULL`),
id: uuid().primaryKey().notNull(),
name: varchar({ length: 255 }).notNull(),
reliability: varchar({ length: 30 }).default("reliable").notNull(),
transparency: varchar({ length: 30 }).default("medium").notNull(),
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
url: varchar({ length: 255 }).notNull(),
},
(_table) => [
uniqueIndex("unq_source_name").using("btree", sql`lower((name)::text)`),
uniqueIndex("unq_source_url").using("btree", sql`lower((url)::text)`),
(table) => [
index("idx_bookmark_user_created").using(
"btree",
table.userId.asc().nullsLast(),
table.createdAt.desc().nullsFirst(),
),
uniqueIndex("unq_bookmark_user_name").using(
"btree",
table.userId.asc().nullsLast(),
sql`lower(${table.name})`,
),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_bookmark_user_id",
}).onDelete("cascade"),
],
);
@@ -337,22 +263,177 @@ export const bookmarkArticle = pgTable(
bookmarkId: uuid("bookmark_id").notNull(),
},
(table) => [
index("idx_6fe2655d7294869c").using("btree", table.articleId.asc().nullsLast().op("uuid_ops")),
index("idx_6fe2655d92741d25").using("btree", table.bookmarkId.asc().nullsLast().op("uuid_ops")),
primaryKey({ columns: [table.bookmarkId, table.articleId], name: "bookmark_article_pkey" }),
index("idx_bookmark_article_bookmark_id").using("btree", table.bookmarkId.asc().nullsLast()),
foreignKey({
columns: [table.bookmarkId],
foreignColumns: [bookmark.id],
name: "fk_6fe2655d92741d25",
name: "fk_bookmark_article_bookmark_id",
}).onDelete("cascade"),
foreignKey({
columns: [table.articleId],
foreignColumns: [article.id],
name: "fk_6fe2655d7294869c",
name: "fk_bookmark_article_article_id",
}).onDelete("cascade"),
primaryKey({ columns: [table.bookmarkId, table.articleId], name: "bookmark_article_pkey" }),
],
);
export const loginAttempt = pgTable(
"login_attempt",
{
createdAt: timestamp("created_at").defaultNow().notNull(),
id: uuid().primaryKey().notNull(),
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_login_attempt_user_created").using(
"btree",
table.userId.asc().nullsLast(),
table.createdAt.desc().nullsFirst(),
),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_login_attempt_user_id",
}).onDelete("cascade"),
],
);
export const loginHistory = pgTable(
"login_history",
{
createdAt: timestamp("created_at").defaultNow().notNull(),
device: jsonb("device").$type<Device>(),
id: uuid().primaryKey().notNull(),
ipAddress: inet("ip_address"),
location: jsonb("location").$type<GeoLocation>(),
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_login_history_user_created").using(
"btree",
table.userId.asc().nullsLast(),
table.createdAt.desc().nullsFirst(),
),
index("idx_login_history_ip_address").using("btree", table.ipAddress.asc().nullsLast()),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_login_history_user_id",
}).onDelete("cascade"),
],
);
export const verificationToken = pgTable(
"verification_token",
{
createdAt: timestamp("created_at").defaultNow().notNull(),
id: uuid().primaryKey().notNull(),
purpose: tokenPurposeEnum("purpose").notNull(),
token: varchar({ length: 60 }), // nullable if you support "reservations" before issue
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_verif_token_created_at").using("btree", table.createdAt.desc().nullsFirst()),
uniqueIndex("unq_verif_user_purpose_token")
.using("btree", table.userId, table.purpose, table.token)
.where(sql`${table.token} IS NOT NULL`),
uniqueIndex("unq_verif_token_token")
.using("btree", table.token)
.where(sql`${table.token} IS NOT NULL`),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_verification_token_user_id",
}).onDelete("cascade"),
],
);
export const followedSource = pgTable(
"followed_source",
{
createdAt: timestamp("created_at").defaultNow().notNull(),
followerId: uuid("follower_id").notNull(),
id: uuid().primaryKey().notNull(),
sourceId: uuid("source_id").notNull(),
},
(table) => [
index("idx_followed_source_source_id").using("btree", table.sourceId.asc().nullsLast()),
index("idx_followed_source_follower_id").using("btree", table.followerId.asc().nullsLast()),
index("idx_followed_source_follower_created").using(
"btree",
table.followerId.asc().nullsLast(),
table.createdAt.desc().nullsFirst(),
),
uniqueIndex("unq_followed_source_user_source").using(
"btree",
table.followerId.asc().nullsLast(),
table.sourceId.asc().nullsLast(),
),
foreignKey({
columns: [table.followerId],
foreignColumns: [user.id],
name: "fk_followed_source_follower_id",
}).onDelete("cascade"),
foreignKey({
columns: [table.sourceId],
foreignColumns: [source.id],
name: "fk_followed_source_source_id",
}).onDelete("cascade"),
],
);
export const comment = pgTable(
"comment",
{
articleId: uuid("article_id").notNull(),
content: varchar({ length: 512 }).notNull(),
createdAt: timestamp("created_at").defaultNow().notNull(),
id: uuid().primaryKey().notNull(),
isSpam: boolean("is_spam").default(false).notNull(),
sentiment: sentimentEnum("sentiment").notNull(),
userId: uuid("user_id").notNull(),
},
(table) => [
index("idx_comment_article_id").using("btree", table.articleId.asc().nullsLast()),
index("idx_comment_user_id").using("btree", table.userId.asc().nullsLast()),
index("idx_comment_article_created").using(
"btree",
table.articleId.asc().nullsLast(),
table.createdAt.desc().nullsFirst(),
),
foreignKey({
columns: [table.userId],
foreignColumns: [user.id],
name: "fk_comment_user_id",
}).onDelete("cascade"),
foreignKey({
columns: [table.articleId],
foreignColumns: [article.id],
name: "fk_comment_article_id",
}).onDelete("cascade"),
],
);
export const refreshToken = pgTable(
"refresh_token",
{
id: uuid().primaryKey().notNull(),
token: varchar("token", { length: 128 }).notNull(),
username: varchar({ length: 255 }).notNull(),
valid: timestamp().notNull(),
},
(table) => [
uniqueIndex("uniq_refresh_token_token").using("btree", table.token.asc().nullsLast()),
index("idx_refresh_token_valid").using("btree", table.valid.asc().nullsLast()),
index("idx_refresh_token_username").using("btree", sql`lower(${table.username})`),
],
);
/* -------------------------------------------------------------------------- */
/* Relations */
/* -------------------------------------------------------------------------- */
export const bookmarkRelations = relations(bookmark, ({ one, many }) => ({
bookmarkArticles: many(bookmarkArticle),
user: one(user, {
+57
View File
@@ -0,0 +1,57 @@
import { TiktokenEncoding, get_encoding } from "tiktoken";
import { TokenStatistics } from "@/schema";
/**
* Count the number of tokens in the given text using the specified encoding.
* @param text - The input text
* @param encoding - The token encoding (default: "cl100k_base")
*/
export const computeTokenCount = (
text: string,
encoding: TiktokenEncoding = "cl100k_base",
): number => {
try {
const encoder = get_encoding(encoding);
const tokens = encoder.encode(text);
encoder.free();
return tokens.length;
} catch {
return text.length;
}
};
/**
* Create token statistics for the given data.
* @param data - The input data containing title, body, and categories
* @returns TokenStatistics object
*/
export const computeTokenStatistics = (data: {
title: string;
body: string;
categories: string[];
}): TokenStatistics => {
const title = computeTokenCount(data.title);
const body = computeTokenCount(data.body);
const categories = computeTokenCount(data.categories.join(","));
const excerpt = computeTokenCount(data.body.substring(0, 200));
return {
body,
categories,
excerpt,
title,
total: title + body + categories + excerpt,
};
};
/**
* Compute the estimated reading time for the given text.
* @param text - The input text
* @param wordsPerMinute - The reading speed in words per minute (default: 200)
* @returns The estimated reading time in minutes
*/
export const computeReadingTime = (text: string, wordsPerMinute = 200): number => {
const words = text.trim().split(/\s+/).length;
return Math.ceil(words / wordsPerMinute);
};
-2
View File
@@ -6,8 +6,6 @@ import {
PAGINATION_MAX_LIMIT,
} from "@/constants";
export type SortDirection = "asc" | "desc";
export interface PageRequest {
page?: number;
limit?: number;