feat(db): migration and database setup
This commit is contained in:
+6
-1
@@ -1 +1,6 @@
|
||||
BASANGO_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/app?serverVersion=16&charset=utf8"
|
||||
BASANGO_DATABASE_URL="postgresql://postgres:postgres@localhost:5432/app?serverVersion=16&charset=utf8"
|
||||
|
||||
BASANGO_SOURCE_DATABASE_HOST="localhost"
|
||||
BASANGO_SOURCE_DATABASE_PASS="root"
|
||||
BASANGO_SOURCE_DATABASE_NAME="app"
|
||||
BASANGO_SOURCE_DATABASE_USER="root"
|
||||
|
||||
@@ -1,172 +0,0 @@
|
||||
-- Current sql file was generated after introspecting the database
|
||||
-- If you want to run this migration please uncomment this code before executing migrations
|
||||
/*
|
||||
CREATE SEQUENCE "public"."refresh_tokens_id_seq" INCREMENT BY 1 MINVALUE 1 MAXVALUE 9223372036854775807 START WITH 1 CACHE 1;--> statement-breakpoint
|
||||
CREATE TABLE "doctrine_migration_versions" (
|
||||
"version" varchar(191) PRIMARY KEY NOT NULL,
|
||||
"executed_at" timestamp(0) DEFAULT NULL,
|
||||
"execution_time" integer
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"description" varchar(512) DEFAULT NULL,
|
||||
"is_public" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_attempt" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_history" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"ip_address" "inet",
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"device_operating_system" varchar(255) DEFAULT NULL,
|
||||
"device_client" varchar(255) DEFAULT NULL,
|
||||
"device_device" varchar(255) DEFAULT NULL,
|
||||
"device_is_bot" boolean DEFAULT false NOT NULL,
|
||||
"location_time_zone" varchar(255) DEFAULT NULL,
|
||||
"location_longitude" double precision,
|
||||
"location_latitude" double precision,
|
||||
"location_accuracy_radius" integer
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "verification_token" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"purpose" varchar(255) NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"token" varchar(60) DEFAULT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "followed_source" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"follower_id" uuid NOT NULL,
|
||||
"source_id" uuid NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "comment" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL,
|
||||
"article_id" uuid NOT NULL,
|
||||
"content" varchar(512) NOT NULL,
|
||||
"sentiment" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"is_spam" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "refresh_tokens" (
|
||||
"id" integer PRIMARY KEY NOT NULL,
|
||||
"refresh_token" varchar(128) NOT NULL,
|
||||
"username" varchar(255) NOT NULL,
|
||||
"valid" timestamp(0) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "article" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"source_id" uuid NOT NULL,
|
||||
"title" varchar(1024) NOT NULL,
|
||||
"body" text NOT NULL,
|
||||
"hash" varchar(32) NOT NULL,
|
||||
"categories" text[],
|
||||
"sentiment" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"metadata" jsonb,
|
||||
"image" varchar(1024) GENERATED ALWAYS AS ((metadata ->> 'image'::text)) STORED,
|
||||
"excerpt" varchar(255) GENERATED ALWAYS AS (("left"(body, 200) || '...'::text)) STORED,
|
||||
"published_at" timestamp(0) NOT NULL,
|
||||
"crawled_at" timestamp(0) NOT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL,
|
||||
"link" varchar(1024) NOT NULL,
|
||||
"bias" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"reliability" varchar(30) DEFAULT 'reliable' NOT NULL,
|
||||
"transparency" varchar(30) DEFAULT 'medium' NOT NULL,
|
||||
"reading_time" integer DEFAULT 1,
|
||||
"tsv" "tsvector" GENERATED ALWAYS AS ((setweight(to_tsvector('french'::regconfig, (COALESCE(title, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char"))) STORED,
|
||||
"token_statistics" jsonb,
|
||||
CONSTRAINT "chk_article_reading_time" CHECK (reading_time >= 0),
|
||||
CONSTRAINT "chk_article_sentiment" CHECK ((sentiment)::text = ANY ((ARRAY['positive'::character varying, 'neutral'::character varying, 'negative'::character varying])::text[])),
|
||||
CONSTRAINT "chk_article_metadata_json" CHECK ((metadata IS NULL) OR (jsonb_typeof(metadata) = ANY (ARRAY['object'::text, 'array'::text])))
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "user" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"email" varchar(255) NOT NULL,
|
||||
"password" varchar(512) NOT NULL,
|
||||
"is_locked" boolean DEFAULT false NOT NULL,
|
||||
"is_confirmed" boolean DEFAULT false NOT NULL,
|
||||
"created_at" timestamp(0) NOT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL,
|
||||
"roles" jsonb NOT NULL,
|
||||
CONSTRAINT "chk_user_roles_json" CHECK (jsonb_typeof(roles) = 'array'::text)
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "source" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"url" varchar(255) NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"display_name" varchar(255) DEFAULT NULL,
|
||||
"description" varchar(1024) DEFAULT NULL,
|
||||
"updated_at" timestamp(0) DEFAULT NULL,
|
||||
"bias" varchar(30) DEFAULT 'neutral' NOT NULL,
|
||||
"reliability" varchar(30) DEFAULT 'reliable' NOT NULL,
|
||||
"transparency" varchar(30) DEFAULT 'medium' NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark_article" (
|
||||
"bookmark_id" uuid NOT NULL,
|
||||
"article_id" uuid NOT NULL,
|
||||
CONSTRAINT "bookmark_article_pkey" PRIMARY KEY("bookmark_id","article_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "bookmark" ADD CONSTRAINT "fk_da62921da76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_attempt" ADD CONSTRAINT "fk_8c11c1ba76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_history" ADD CONSTRAINT "fk_37976e36a76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "verification_token" ADD CONSTRAINT "fk_c1cc006ba76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_7a763a3eac24f853" FOREIGN KEY ("follower_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_7a763a3e953c1c61" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_9474526ca76ed395" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_9474526c7294869c" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "article" ADD CONSTRAINT "fk_23a0e66953c1c61" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_6fe2655d92741d25" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmark"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_6fe2655d7294869c" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "idx_bookmark_user_created" ON "bookmark" USING btree ("user_id" timestamp_ops,"created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_da62921da76ed395" ON "bookmark" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_8c11c1ba76ed395" ON "login_attempt" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_attempt_created_at" ON "login_attempt" USING btree ("created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_37976e36a76ed395" ON "login_history" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_created_at" ON "login_history" USING btree ("user_id" uuid_ops,"created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_ip_address" ON "login_history" USING btree ("ip_address" inet_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_c1cc006ba76ed395" ON "verification_token" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_verif_token_created_at" ON "verification_token" USING btree ("created_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_verif_user_purpose_token" ON "verification_token" USING btree ("user_id" text_ops,"purpose" text_ops) WHERE (token IS NOT NULL);--> statement-breakpoint
|
||||
CREATE INDEX "idx_7a763a3e953c1c61" ON "followed_source" USING btree ("source_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_7a763a3eac24f853" ON "followed_source" USING btree ("follower_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_follower_created" ON "followed_source" USING btree ("follower_id" timestamp_ops,"created_at" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_9474526c7294869c" ON "comment" USING btree ("article_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_9474526ca76ed395" ON "comment" USING btree ("user_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_article_created" ON "comment" USING btree ("article_id" timestamp_ops,"created_at" uuid_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "uniq_9bace7e1c74f2195" ON "refresh_tokens" USING btree ("refresh_token" text_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_categories" ON "article" USING gin ("categories" array_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_link_trgm" ON "article" USING gin ("link" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_title_trgm" ON "article" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_tsv" ON "article" USING gin ("tsv" tsvector_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_23a0e66953c1c61" ON "article" USING btree ("source_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_article_published_at" ON "article" USING btree ("published_at" timestamp_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_article_published_id" ON "article" USING btree ("published_at" timestamp_ops,"id" uuid_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_article_hash" ON "article" USING btree ("hash" text_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_user_email" ON "user" USING btree (lower((email)::text) text_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_name" ON "source" USING btree (lower((name)::text) text_ops);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_url" ON "source" USING btree (lower((url)::text) text_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_6fe2655d7294869c" ON "bookmark_article" USING btree ("article_id" uuid_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_6fe2655d92741d25" ON "bookmark_article" USING btree ("bookmark_id" uuid_ops);
|
||||
*/
|
||||
@@ -0,0 +1,3 @@
|
||||
-- Custom SQL migration file, put your code below! --
|
||||
CREATE EXTENSION IF NOT EXISTS pg_trgm;
|
||||
SET SESSION TIME ZONE 'UTC';
|
||||
@@ -0,0 +1,154 @@
|
||||
CREATE TYPE "public"."bias" AS ENUM('neutral', 'slightly', 'partisan', 'extreme');--> statement-breakpoint
|
||||
CREATE TYPE "public"."reliability" AS ENUM('trusted', 'reliable', 'average', 'low_trust', 'unreliable');--> statement-breakpoint
|
||||
CREATE TYPE "public"."sentiment" AS ENUM('positive', 'neutral', 'negative');--> statement-breakpoint
|
||||
CREATE TYPE "public"."token_purpose" AS ENUM('confirm_account', 'password_reset', 'unlock_account', 'delete_account');--> statement-breakpoint
|
||||
CREATE TYPE "public"."transparency" AS ENUM('high', 'medium', 'low');--> statement-breakpoint
|
||||
CREATE TABLE "article" (
|
||||
"body" text NOT NULL,
|
||||
"categories" text[],
|
||||
"crawled_at" timestamp DEFAULT now() NOT NULL,
|
||||
"credibility" jsonb,
|
||||
"excerpt" varchar(255) GENERATED ALWAYS AS (("left"(body, 200) || '...'::text)) STORED,
|
||||
"hash" varchar(32) NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"image" varchar(1024) GENERATED ALWAYS AS ((metadata ->> 'image'::text)) STORED,
|
||||
"link" varchar(1024) NOT NULL,
|
||||
"metadata" jsonb,
|
||||
"published_at" timestamp NOT NULL,
|
||||
"reading_time" integer DEFAULT 1,
|
||||
"sentiment" "sentiment" NOT NULL,
|
||||
"source_id" uuid NOT NULL,
|
||||
"title" varchar(1024) NOT NULL,
|
||||
"token_statistics" jsonb,
|
||||
"tsv" "tsvector" GENERATED ALWAYS AS ((
|
||||
setweight(to_tsvector('french'::regconfig, COALESCE(title, '')::text), 'A'::"char")
|
||||
|| setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char")
|
||||
)) STORED,
|
||||
"updated_at" timestamp
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"description" varchar(512),
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"is_public" boolean DEFAULT false NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"updated_at" timestamp,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "bookmark_article" (
|
||||
"article_id" uuid NOT NULL,
|
||||
"bookmark_id" uuid NOT NULL,
|
||||
CONSTRAINT "bookmark_article_pkey" PRIMARY KEY("bookmark_id","article_id")
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "comment" (
|
||||
"article_id" uuid NOT NULL,
|
||||
"content" varchar(512) NOT NULL,
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"is_spam" boolean DEFAULT false NOT NULL,
|
||||
"sentiment" "sentiment" NOT NULL,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "followed_source" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"follower_id" uuid NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"source_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_attempt" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "login_history" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"device" jsonb,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"ip_address" "inet",
|
||||
"location" jsonb,
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "refresh_token" (
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"token" varchar(128) NOT NULL,
|
||||
"username" varchar(255) NOT NULL,
|
||||
"valid" timestamp NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "source" (
|
||||
"credibility" jsonb,
|
||||
"description" varchar(1024),
|
||||
"display_name" varchar(255),
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"updated_at" timestamp,
|
||||
"url" varchar(255) NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "user" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"email" varchar(255) NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"is_confirmed" boolean DEFAULT false NOT NULL,
|
||||
"is_locked" boolean DEFAULT false NOT NULL,
|
||||
"name" varchar(255) NOT NULL,
|
||||
"password" varchar(512) NOT NULL,
|
||||
"roles" varchar(255)[] DEFAULT '{"ROLE_USER"}' NOT NULL,
|
||||
"updated_at" timestamp
|
||||
);
|
||||
--> statement-breakpoint
|
||||
CREATE TABLE "verification_token" (
|
||||
"created_at" timestamp DEFAULT now() NOT NULL,
|
||||
"id" uuid PRIMARY KEY NOT NULL,
|
||||
"purpose" "token_purpose" NOT NULL,
|
||||
"token" varchar(60),
|
||||
"user_id" uuid NOT NULL
|
||||
);
|
||||
--> statement-breakpoint
|
||||
ALTER TABLE "article" ADD CONSTRAINT "fk_article_source_id" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark" ADD CONSTRAINT "fk_bookmark_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_bookmark_article_bookmark_id" FOREIGN KEY ("bookmark_id") REFERENCES "public"."bookmark"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "bookmark_article" ADD CONSTRAINT "fk_bookmark_article_article_id" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_comment_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "comment" ADD CONSTRAINT "fk_comment_article_id" FOREIGN KEY ("article_id") REFERENCES "public"."article"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_followed_source_follower_id" FOREIGN KEY ("follower_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "followed_source" ADD CONSTRAINT "fk_followed_source_source_id" FOREIGN KEY ("source_id") REFERENCES "public"."source"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_attempt" ADD CONSTRAINT "fk_login_attempt_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "login_history" ADD CONSTRAINT "fk_login_history_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
ALTER TABLE "verification_token" ADD CONSTRAINT "fk_verification_token_user_id" FOREIGN KEY ("user_id") REFERENCES "public"."user"("id") ON DELETE cascade ON UPDATE no action;--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_categories" ON "article" USING gin ("categories" array_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_link_trgm" ON "article" USING gin ("link" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_title_trgm" ON "article" USING gin ("title" gin_trgm_ops);--> statement-breakpoint
|
||||
CREATE INDEX "gin_article_tsv" ON "article" USING gin ("tsv" tsvector_ops);--> statement-breakpoint
|
||||
CREATE INDEX "idx_article_source_published_id" ON "article" USING btree ("source_id","published_at" DESC NULLS FIRST,"id" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_article_hash" ON "article" USING btree ("hash");--> statement-breakpoint
|
||||
CREATE INDEX "idx_bookmark_user_created" ON "bookmark" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_bookmark_user_name" ON "bookmark" USING btree ("user_id",lower("name"));--> statement-breakpoint
|
||||
CREATE INDEX "idx_bookmark_article_bookmark_id" ON "bookmark_article" USING btree ("bookmark_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_article_id" ON "comment" USING btree ("article_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_user_id" ON "comment" USING btree ("user_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_comment_article_created" ON "comment" USING btree ("article_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_source_id" ON "followed_source" USING btree ("source_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_follower_id" ON "followed_source" USING btree ("follower_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_followed_source_follower_created" ON "followed_source" USING btree ("follower_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_followed_source_user_source" ON "followed_source" USING btree ("follower_id","source_id");--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_attempt_user_created" ON "login_attempt" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_user_created" ON "login_history" USING btree ("user_id","created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE INDEX "idx_login_history_ip_address" ON "login_history" USING btree ("ip_address");--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "uniq_refresh_token_token" ON "refresh_token" USING btree ("token");--> statement-breakpoint
|
||||
CREATE INDEX "idx_refresh_token_valid" ON "refresh_token" USING btree ("valid");--> statement-breakpoint
|
||||
CREATE INDEX "idx_refresh_token_username" ON "refresh_token" USING btree (lower("username"));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_name" ON "source" USING btree (lower((name)::text));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_source_url" ON "source" USING btree (lower((url)::text));--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_user_email" ON "user" USING btree (lower((email)::text));--> statement-breakpoint
|
||||
CREATE INDEX "idx_user_created_at" ON "user" USING btree (created_at);--> statement-breakpoint
|
||||
CREATE INDEX "idx_verif_token_created_at" ON "verification_token" USING btree ("created_at" DESC NULLS FIRST);--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_verif_user_purpose_token" ON "verification_token" USING btree ("user_id","purpose","token") WHERE "verification_token"."token" IS NOT NULL;--> statement-breakpoint
|
||||
CREATE UNIQUE INDEX "unq_verif_token_token" ON "verification_token" USING btree ("token") WHERE "verification_token"."token" IS NOT NULL;
|
||||
File diff suppressed because it is too large
Load Diff
File diff suppressed because it is too large
Load Diff
@@ -4,9 +4,16 @@
|
||||
{
|
||||
"breakpoints": true,
|
||||
"idx": 0,
|
||||
"tag": "0000_aromatic_dorian_gray",
|
||||
"tag": "0000_setup",
|
||||
"version": "7",
|
||||
"when": 1762691204645
|
||||
"when": 1762775141000
|
||||
},
|
||||
{
|
||||
"breakpoints": true,
|
||||
"idx": 1,
|
||||
"tag": "0001_init",
|
||||
"version": "7",
|
||||
"when": 1762775267679
|
||||
}
|
||||
],
|
||||
"version": "7"
|
||||
|
||||
@@ -1,10 +1,14 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@basango/encryption": "workspace:*",
|
||||
"@basango/logger": "workspace:*",
|
||||
"@date-fns/utc": "^2.1.1",
|
||||
"drizzle-orm": "^0.44.7",
|
||||
"mysql2": "^3.15.3",
|
||||
"pg": "^8.16.3",
|
||||
"snakecase-keys": "^9.0.2"
|
||||
"snakecase-keys": "^9.0.2",
|
||||
"tiktoken": "^1.0.22",
|
||||
"uuid": "^13.0.0"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/pg": "^8.15.6",
|
||||
@@ -13,6 +17,7 @@
|
||||
"exports": {
|
||||
".": "./src/index.ts",
|
||||
"./client": "./src/client.ts",
|
||||
"./importer": "./src/importer/index.ts",
|
||||
"./queries": "./src/queries/index.ts",
|
||||
"./schema": "./src/schema.ts",
|
||||
"./utils": "./src/utils/index.ts"
|
||||
@@ -21,6 +26,7 @@
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"clean": "rm -rf .turbo node_modules",
|
||||
"sync:import": "bun ./src/importer/import.ts",
|
||||
"typecheck": "tsc --noEmit"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
export class NotFoundError extends Error {
|
||||
constructor(message: string) {
|
||||
super(message);
|
||||
this.name = "NotFoundError";
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,468 @@
|
||||
import { RowDataPacket } from "mysql2/promise";
|
||||
import { Pool, PoolClient } from "pg";
|
||||
|
||||
import { computeReadingTime, computeTokenStatistics } from "@/utils/computed";
|
||||
|
||||
type SourceOptions = {
|
||||
host: string;
|
||||
user: string;
|
||||
password: string;
|
||||
database: string;
|
||||
};
|
||||
|
||||
type TargetOptions = {
|
||||
database: string;
|
||||
batchSize?: number;
|
||||
pageSize?: number;
|
||||
ignoreColumns?: Record<string, string[]>;
|
||||
};
|
||||
|
||||
const DEFAULT_IGNORE: Record<string, string[]> = {
|
||||
article: ["tsv", "image", "excerpt", "bias", "reliability", "transparency"],
|
||||
source: ["bias", "reliability", "transparency"],
|
||||
};
|
||||
|
||||
/**
|
||||
* Engine
|
||||
*
|
||||
* Coordinates copying rows from a MySQL source into a PostgreSQL target in a
|
||||
* controlled, transactional, batched manner.
|
||||
*
|
||||
* Responsibilities:
|
||||
* - Establish and manage a connection pool to the target PostgreSQL database.
|
||||
* - Stream rows from a MySQL source (via a temporary pool) using pagination.
|
||||
* - Transform row values to match target expectations (UUID normalization,
|
||||
* timestamp fallback, array parsing for categories/roles, computed JSON
|
||||
* credibility, etc.).
|
||||
* - Filter out ignored columns based on a configurable ignore map.
|
||||
* - Insert rows into the target in configurable batch sizes with transactional
|
||||
* commits every batch to limit long-running transactions.
|
||||
* - Provide a safe reset operation that truncates the target table and manages
|
||||
* session replication role toggling for Postgres.
|
||||
*
|
||||
* @param sourceOptions - connection and authentication options for the MySQL
|
||||
* source (database, host, user, password, etc.).
|
||||
* @param targetOptions - configuration for the Postgres target including
|
||||
* connection string (database), optional pageSize, batchSize and per-table
|
||||
* ignoreColumns map.
|
||||
*/
|
||||
export class Engine {
|
||||
private readonly target: Pool;
|
||||
private readonly ignore: Record<string, string[]>;
|
||||
private readonly pageSize: number;
|
||||
private readonly batchSize: number;
|
||||
|
||||
constructor(
|
||||
private readonly sourceOptions: SourceOptions,
|
||||
private readonly targetOptions: TargetOptions,
|
||||
) {
|
||||
this.target = new Pool({
|
||||
allowExitOnIdle: true,
|
||||
connectionString: this.targetOptions.database,
|
||||
max: 8,
|
||||
});
|
||||
this.ignore = { ...DEFAULT_IGNORE, ...(this.targetOptions.ignoreColumns ?? {}) };
|
||||
this.pageSize = this.targetOptions.pageSize ?? 10_000;
|
||||
this.batchSize = Math.max(1, this.targetOptions.batchSize ?? 1000);
|
||||
console.log(
|
||||
`Engine initialized with pageSize=${this.pageSize} and batchSize=${this.batchSize}`,
|
||||
);
|
||||
}
|
||||
|
||||
async close() {
|
||||
await this.target.end();
|
||||
}
|
||||
|
||||
async import(table: string): Promise<number> {
|
||||
await this.reset(table);
|
||||
return await this.paste(table, this.copy(table));
|
||||
}
|
||||
|
||||
private async *copy(table: string): AsyncGenerator<Record<string, unknown>> {
|
||||
const mysql = await import("mysql2/promise");
|
||||
|
||||
const source = mysql.createPool({
|
||||
database: this.sourceOptions.database,
|
||||
host: this.sourceOptions.host,
|
||||
idleTimeout: 180_000_000,
|
||||
password: this.sourceOptions.password,
|
||||
port: 3306,
|
||||
rowsAsArray: false,
|
||||
user: this.sourceOptions.user,
|
||||
});
|
||||
|
||||
let offset = 0;
|
||||
const size = this.pageSize;
|
||||
try {
|
||||
while (true) {
|
||||
const [rows] = await source.query<RowDataPacket[]>(
|
||||
`SELECT * FROM \`${this.escapeBacktick(table)}\` LIMIT ? OFFSET ?`,
|
||||
[size, offset],
|
||||
);
|
||||
|
||||
if (!rows || rows.length === 0) break;
|
||||
|
||||
for (const row of rows) {
|
||||
yield row as Record<string, unknown>;
|
||||
}
|
||||
|
||||
offset += rows.length;
|
||||
if (rows.length < size) break;
|
||||
}
|
||||
} finally {
|
||||
try {
|
||||
await source.end();
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
private async paste(
|
||||
table: string,
|
||||
rows: AsyncGenerator<Record<string, unknown>>,
|
||||
): Promise<number> {
|
||||
const target = await this.target.connect();
|
||||
let total = 0;
|
||||
let inBatch = 0;
|
||||
let columns: string[] | null = null;
|
||||
let insertSql = "";
|
||||
|
||||
const ignored = this.ignoredColumnsFor(table);
|
||||
const ignoredSet = new Set(ignored);
|
||||
|
||||
try {
|
||||
for await (let row of rows) {
|
||||
if (!columns) {
|
||||
row = this.transformRowForTarget(table, row);
|
||||
// Filter ignored columns and build column order
|
||||
columns = Object.keys(row).filter((c) => !ignoredSet.has(c));
|
||||
|
||||
// If article target has credibility but source not, include computed credibility
|
||||
if (
|
||||
(this.normalizedName(table) === "article" && !columns.includes("credibility")) ||
|
||||
(this.normalizedName(table) === "source" && !columns.includes("credibility"))
|
||||
) {
|
||||
columns.push("credibility");
|
||||
}
|
||||
|
||||
if (this.normalizedName(table) === "article" && !columns.includes("token_statistics")) {
|
||||
columns.push("token_statistics");
|
||||
}
|
||||
|
||||
const colsSql = columns.map((c) => this.quote(c)).join(", ");
|
||||
const placeholders = columns.map((_, i) => `$${i + 1}`).join(", ");
|
||||
insertSql = `INSERT INTO ${this.quote(table)} (${colsSql}) VALUES (${placeholders})`;
|
||||
|
||||
await target.query("BEGIN");
|
||||
}
|
||||
|
||||
// Row transform and params in column order
|
||||
const transformed = this.transformRowForTarget(table, row);
|
||||
const params = columns!.map((c) => this.valueForColumn(c, transformed));
|
||||
|
||||
try {
|
||||
await target.query(insertSql, params);
|
||||
} catch (err: unknown) {
|
||||
const msg = String((err as Error)?.message ?? "");
|
||||
if (msg.includes("invalid input syntax for type timestamp")) {
|
||||
// Fallback: coerce all *_at params to now() and retry once
|
||||
const fixed = columns!.map((c, i) => (c.endsWith("_at") ? new Date() : params[i]));
|
||||
await target.query(insertSql, fixed);
|
||||
} else {
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
total++;
|
||||
inBatch++;
|
||||
|
||||
if (inBatch >= this.batchSize) {
|
||||
await target.query("COMMIT");
|
||||
inBatch = 0;
|
||||
await target.query("BEGIN");
|
||||
console.log(`Imported ${total} records into ${table} so far...`);
|
||||
}
|
||||
}
|
||||
|
||||
if (inBatch > 0) {
|
||||
await target.query("COMMIT");
|
||||
}
|
||||
} catch (e) {
|
||||
await safeRollback(target);
|
||||
throw e;
|
||||
} finally {
|
||||
target.release();
|
||||
}
|
||||
|
||||
return total;
|
||||
}
|
||||
|
||||
private normalizedName(table: string): string {
|
||||
return table.replaceAll('"', "").replaceAll("`", "").toLowerCase();
|
||||
}
|
||||
|
||||
private ignoredColumnsFor(table: string): string[] {
|
||||
return this.ignore[this.normalizedName(table)] ?? [];
|
||||
}
|
||||
|
||||
private async reset(table: string) {
|
||||
const client = await this.target.connect();
|
||||
try {
|
||||
await client.query("BEGIN");
|
||||
await client.query("SET session_replication_role = 'replica'");
|
||||
await client.query(`TRUNCATE TABLE ${this.quote(table)} RESTART IDENTITY CASCADE`);
|
||||
await client.query("SET session_replication_role = 'origin'");
|
||||
await client.query("COMMIT");
|
||||
console.log(`Reset completed for table ${table}`);
|
||||
} catch (e) {
|
||||
await safeRollback(client);
|
||||
throw e;
|
||||
} finally {
|
||||
client.release();
|
||||
}
|
||||
}
|
||||
|
||||
private transformRowForTarget(table: string, row: Record<string, unknown>) {
|
||||
const t = this.normalizedName(table);
|
||||
const clone: Record<string, unknown> = { ...row };
|
||||
|
||||
// Normalize UUIDs and timestamps and categories
|
||||
for (const [key, val] of Object.entries(clone)) {
|
||||
if (val == null) continue;
|
||||
|
||||
if (key === "id" || key.endsWith("_id")) {
|
||||
clone[key] = this.normalizeUuidValue(val);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Robust timestamp normalization for *_at columns
|
||||
if (key.endsWith("_at")) {
|
||||
clone[key] = this.normalizeTimestampValue(val);
|
||||
continue;
|
||||
}
|
||||
|
||||
if (key === "categories") {
|
||||
if (Array.isArray(val)) {
|
||||
clone[key] = val;
|
||||
} else if (typeof val === "string") {
|
||||
const raw = val.trim();
|
||||
// Try JSON first
|
||||
if (raw.startsWith("[") && raw.endsWith("]")) {
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) {
|
||||
clone[key] = parsed;
|
||||
continue;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
const parts = raw
|
||||
.split(",")
|
||||
.map((s) => s.trim())
|
||||
.filter(Boolean);
|
||||
clone[key] = parts.length ? parts : null;
|
||||
}
|
||||
}
|
||||
|
||||
if (t === "article" && key === "token_statistics") {
|
||||
clone[key] = computeTokenStatistics({
|
||||
body: String(clone.body ?? ""),
|
||||
categories: Array.isArray(clone.categories) ? clone.categories : [],
|
||||
title: String(clone.title ?? ""),
|
||||
});
|
||||
}
|
||||
|
||||
if (t === "article" && key === "reading_time") {
|
||||
clone[key] = Math.max(1, computeReadingTime(String(clone.body ?? "")));
|
||||
}
|
||||
|
||||
if (key === "roles") {
|
||||
if (Array.isArray(val)) {
|
||||
clone[key] = val;
|
||||
} else if (typeof val === "string") {
|
||||
const raw = val.trim();
|
||||
|
||||
// If the value is a JSON array string like '["ROLE_USER","ROLE_ADMIN"]', parse it.
|
||||
try {
|
||||
const parsed = JSON.parse(raw);
|
||||
if (Array.isArray(parsed)) {
|
||||
clone[key] = parsed;
|
||||
continue;
|
||||
}
|
||||
} catch {
|
||||
// not JSON, fall back to CSV-like parsing below
|
||||
}
|
||||
|
||||
// Remove surrounding brackets/quotes then split by comma and strip quotes/space
|
||||
const parts = raw
|
||||
.replace(/^\[|\]$/g, "")
|
||||
.split(",")
|
||||
.map((s) => s.replace(/^["']|["']$/g, "").trim())
|
||||
.filter(Boolean);
|
||||
|
||||
clone[key] = parts.length ? parts : ["ROLE_USER"];
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// compute credibility JSON if bias/reliability/transparency present
|
||||
if (t === "article" || t === "source") {
|
||||
const bias = clone.bias ?? null;
|
||||
const reliability = clone.reliability ?? null;
|
||||
const transparency = clone.transparency ?? null;
|
||||
if (bias || reliability || transparency) {
|
||||
clone.credibility = {
|
||||
bias,
|
||||
reliability,
|
||||
transparency,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure article token_statistics exists (computed on the fly)
|
||||
if (
|
||||
t === "article" &&
|
||||
(clone.token_statistics == null || typeof clone.token_statistics !== "object")
|
||||
) {
|
||||
clone.token_statistics = computeTokenStatistics({
|
||||
body: String(clone.body ?? ""),
|
||||
categories: Array.isArray(clone.categories) ? (clone.categories as string[]) : [],
|
||||
title: String(clone.title ?? ""),
|
||||
});
|
||||
}
|
||||
|
||||
return clone;
|
||||
}
|
||||
|
||||
private valueForColumn(col: string, row: Record<string, unknown>) {
|
||||
const v = row[col];
|
||||
// Pass Date objects directly to pg for timestamp columns
|
||||
if (col.endsWith("_at") && v instanceof Date) {
|
||||
return v;
|
||||
}
|
||||
if (col === "credibility" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "token_statistics" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "device" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "location" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "roles" && v) {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
if (col === "metadata" && v && typeof v === "object") {
|
||||
return JSON.stringify(v);
|
||||
}
|
||||
return v ?? null;
|
||||
}
|
||||
|
||||
private normalizeUuidValue(value: unknown): string {
|
||||
if (Buffer.isBuffer(value)) {
|
||||
return bufferToUuid(value);
|
||||
}
|
||||
if (typeof value === "string") {
|
||||
// Already a UUID string or hex; try to format 32-hex into canonical form
|
||||
const hex = value.replace(/-/g, "").toLowerCase();
|
||||
if (/^[0-9a-f]{32}$/.test(hex)) {
|
||||
return (
|
||||
hex.slice(0, 8) +
|
||||
"-" +
|
||||
hex.slice(8, 12) +
|
||||
"-" +
|
||||
hex.slice(12, 16) +
|
||||
"-" +
|
||||
hex.slice(16, 20) +
|
||||
"-" +
|
||||
hex.slice(20)
|
||||
);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
return String(value);
|
||||
}
|
||||
|
||||
private normalizeTimestampValue(value: unknown): Date {
|
||||
// If it's already a Date, ensure it's valid
|
||||
if (value instanceof Date) {
|
||||
return Number.isNaN(value.getTime()) ? new Date() : value;
|
||||
}
|
||||
|
||||
// Strings: handle common invalid patterns and attempt safe parsing
|
||||
if (typeof value === "string") {
|
||||
const raw = value.trim();
|
||||
if (
|
||||
!raw ||
|
||||
/0000-00-00/.test(raw) ||
|
||||
/NaN/.test(raw) ||
|
||||
raw.toLowerCase() === "invalid date"
|
||||
) {
|
||||
return new Date();
|
||||
}
|
||||
|
||||
// Normalize MySQL-like 'YYYY-MM-DD HH:MM:SS[.ffffff]' to ISO
|
||||
let s = raw.replace(" ", "T");
|
||||
// Reduce microseconds to milliseconds (3 digits) if present
|
||||
s = s.replace(/\.(\d{3})\d+$/, ".$1");
|
||||
// Append Z if there is no timezone info
|
||||
if (!/[zZ]|[+-]\d{2}:?\d{2}$/.test(s)) s += "Z";
|
||||
|
||||
const d = new Date(s);
|
||||
if (!Number.isNaN(d.getTime())) return d;
|
||||
|
||||
// Try numeric string as epoch seconds/millis
|
||||
const n = Number(raw);
|
||||
if (Number.isFinite(n)) {
|
||||
const ms = n > 1e12 ? n : n * 1000;
|
||||
const d2 = new Date(ms);
|
||||
if (!Number.isNaN(d2.getTime())) return d2;
|
||||
}
|
||||
|
||||
return new Date();
|
||||
}
|
||||
|
||||
// Numbers: treat as epoch seconds/millis
|
||||
if (typeof value === "number" && Number.isFinite(value)) {
|
||||
const ms = value > 1e12 ? value : value * 1000;
|
||||
const d = new Date(ms);
|
||||
return Number.isNaN(d.getTime()) ? new Date() : d;
|
||||
}
|
||||
|
||||
// Fallback: now
|
||||
return new Date();
|
||||
}
|
||||
|
||||
private quote(id: string) {
|
||||
const norm = this.normalizedName(id);
|
||||
return `"${norm.replaceAll('"', '""')}"`;
|
||||
}
|
||||
|
||||
private escapeBacktick(id: string) {
|
||||
return id.replaceAll("`", "``");
|
||||
}
|
||||
}
|
||||
|
||||
function bufferToUuid(buf: Buffer): string {
|
||||
if (buf.length !== 16) return buf.toString("hex");
|
||||
const hex = buf.toString("hex");
|
||||
return (
|
||||
hex.slice(0, 8) +
|
||||
"-" +
|
||||
hex.slice(8, 12) +
|
||||
"-" +
|
||||
hex.slice(12, 16) +
|
||||
"-" +
|
||||
hex.slice(16, 20) +
|
||||
"-" +
|
||||
hex.slice(20)
|
||||
);
|
||||
}
|
||||
|
||||
async function safeRollback(client: PoolClient) {
|
||||
try {
|
||||
await client.query("ROLLBACK");
|
||||
} catch {}
|
||||
}
|
||||
@@ -0,0 +1,65 @@
|
||||
#!/usr/bin/env bun
|
||||
import { stdin as input, stdout as output } from "node:process";
|
||||
import { createInterface } from "node:readline/promises";
|
||||
|
||||
import { createEnvAccessor } from "@devscast/config";
|
||||
|
||||
import { Engine } from "@/importer";
|
||||
|
||||
const env = createEnvAccessor([
|
||||
"BASANGO_SOURCE_DATABASE_HOST",
|
||||
"BASANGO_SOURCE_DATABASE_USER",
|
||||
"BASANGO_SOURCE_DATABASE_PASS",
|
||||
"BASANGO_SOURCE_DATABASE_NAME",
|
||||
"BASANGO_DATABASE_URL",
|
||||
]);
|
||||
|
||||
async function promptConfirm(question: string, def = false) {
|
||||
const rl = createInterface({ input, output });
|
||||
const suffix = def ? "[Y/n]" : "[y/N]";
|
||||
const answer = await rl.question(`${question} ${suffix} `);
|
||||
rl.close();
|
||||
const v = String(answer || "")
|
||||
.trim()
|
||||
.toLowerCase();
|
||||
if (v === "y" || v === "yes") return true;
|
||||
if (v === "n" || v === "no") return false;
|
||||
return def;
|
||||
}
|
||||
|
||||
async function main() {
|
||||
const ok = await promptConfirm("Do you want to continue?", false);
|
||||
if (!ok) {
|
||||
console.warn("Process aborted");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const engine = new Engine(
|
||||
{
|
||||
database: env("BASANGO_SOURCE_DATABASE_NAME"),
|
||||
host: env("BASANGO_SOURCE_DATABASE_HOST"),
|
||||
password: env("BASANGO_SOURCE_DATABASE_PASS"),
|
||||
user: env("BASANGO_SOURCE_DATABASE_USER"),
|
||||
},
|
||||
{
|
||||
database: env("BASANGO_DATABASE_URL"),
|
||||
},
|
||||
);
|
||||
|
||||
try {
|
||||
const tables = process.argv.slice(2);
|
||||
if (tables.length === 0) tables.push("user", "source", "article");
|
||||
for (const t of tables) {
|
||||
const count = await engine.import(t);
|
||||
console.log(`Imported ${count} records into ${t} table.`);
|
||||
}
|
||||
console.log("Import completed successfully");
|
||||
} finally {
|
||||
await engine.close();
|
||||
}
|
||||
}
|
||||
|
||||
main().catch((err) => {
|
||||
console.error(err?.message ?? err);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,2 @@
|
||||
export * from "./engine";
|
||||
export * from "./import";
|
||||
@@ -0,0 +1,65 @@
|
||||
import { md5 } from "@basango/encryption";
|
||||
import { eq } from "drizzle-orm";
|
||||
import { v7 as uuidV7 } from "uuid";
|
||||
|
||||
import { Database } from "@/client";
|
||||
import { ArticleMetadata, Sentiment, TokenStatistics, article } from "@/schema";
|
||||
import { computeReadingTime, computeTokenStatistics } from "@/utils/computed";
|
||||
|
||||
import { getSourceIdByName } from "./sources";
|
||||
|
||||
export type CreateArticleParams = {
|
||||
title: string;
|
||||
body: string;
|
||||
categories: string[];
|
||||
link: string;
|
||||
sourceId: string;
|
||||
publishedAt: Date;
|
||||
sentiment?: Sentiment;
|
||||
tokenStatistics?: TokenStatistics;
|
||||
readingTime?: number;
|
||||
metadata?: ArticleMetadata;
|
||||
};
|
||||
|
||||
export async function createArticle(db: Database, params: CreateArticleParams) {
|
||||
const data = {
|
||||
...params,
|
||||
hash: md5(params.link),
|
||||
readingTime: computeReadingTime(params.body),
|
||||
sentiment: "neutral" as Sentiment,
|
||||
sourceId: await getSourceIdByName(db, params.sourceId),
|
||||
tokenStatistics: computeTokenStatistics({
|
||||
body: params.body,
|
||||
categories: params.categories,
|
||||
title: params.title,
|
||||
}),
|
||||
};
|
||||
|
||||
const duplicated = await getArticleByHash(db, data.hash);
|
||||
if (duplicated !== undefined) {
|
||||
return {
|
||||
id: duplicated.id,
|
||||
sourceId: duplicated.sourceId,
|
||||
};
|
||||
}
|
||||
|
||||
const [result] = await db
|
||||
.insert(article)
|
||||
.values({ id: uuidV7(), ...data })
|
||||
.returning({
|
||||
id: article.id,
|
||||
sourceId: article.sourceId,
|
||||
});
|
||||
|
||||
if (result === undefined) {
|
||||
throw new Error("Failed to create article");
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function getArticleByHash(db: Database, hash: string) {
|
||||
return db.query.article.findFirst({
|
||||
where: eq(article.hash, hash),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,65 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
import { v7 as uuidV7 } from "uuid";
|
||||
|
||||
import { Database } from "@/client";
|
||||
import { NotFoundError } from "@/errors";
|
||||
import { Credibility, source } from "@/schema";
|
||||
|
||||
export type CreateSourceParams = {
|
||||
name: string;
|
||||
url: string;
|
||||
displayName?: string;
|
||||
description?: string;
|
||||
credibility: Credibility;
|
||||
updatedAt?: Date;
|
||||
};
|
||||
|
||||
export async function createSource(db: Database, params: CreateSourceParams) {
|
||||
const [result] = await db
|
||||
.insert(source)
|
||||
.values({ id: uuidV7(), ...params })
|
||||
.returning();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export type DeleteSourceParams = {
|
||||
id: string;
|
||||
};
|
||||
|
||||
export async function deleteSource(db: Database, params: DeleteSourceParams) {
|
||||
const [result] = await db.delete(source).where(eq(source.id, params.id)).returning();
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
export async function getSourceByName(db: Database, name: string) {
|
||||
return db.query.source.findFirst({
|
||||
where: eq(source.name, name),
|
||||
});
|
||||
}
|
||||
|
||||
export async function getSourceIdByName(db: Database, name: string): Promise<string> {
|
||||
const result = await db.query.source.findFirst({
|
||||
columns: {
|
||||
id: true,
|
||||
},
|
||||
where: eq(source.name, name),
|
||||
});
|
||||
|
||||
if (!result) {
|
||||
throw new NotFoundError(`Source with name "${name}" not found`);
|
||||
}
|
||||
|
||||
return result.id;
|
||||
}
|
||||
|
||||
export type GetSourceByIdParams = {
|
||||
id: string;
|
||||
};
|
||||
|
||||
export async function getSourceById(db: Database, params: GetSourceByIdParams) {
|
||||
return db.query.source.findFirst({
|
||||
where: eq(source.id, params.id),
|
||||
});
|
||||
}
|
||||
|
||||
+334
-253
@@ -1,15 +1,14 @@
|
||||
import { relations, sql } from "drizzle-orm";
|
||||
import { check } from "drizzle-orm/gel-core";
|
||||
import {
|
||||
boolean,
|
||||
check,
|
||||
customType,
|
||||
doublePrecision,
|
||||
foreignKey,
|
||||
index,
|
||||
inet,
|
||||
integer,
|
||||
jsonb,
|
||||
pgSequence,
|
||||
pgEnum,
|
||||
pgTable,
|
||||
primaryKey,
|
||||
text,
|
||||
@@ -19,242 +18,181 @@ import {
|
||||
varchar,
|
||||
} from "drizzle-orm/pg-core";
|
||||
|
||||
const tsvector = customType<{ data: string; driverData: string }>({
|
||||
/* -------------------------------------------------------------------------- */
|
||||
/* Types */
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
export const tsvector = customType<{ data: string; driverData: string }>({
|
||||
dataType() {
|
||||
return "tsvector";
|
||||
},
|
||||
});
|
||||
|
||||
export const refreshTokensIdSeq = pgSequence("refresh_tokens_id_seq", {
|
||||
cache: "1",
|
||||
cycle: false,
|
||||
increment: "1",
|
||||
maxValue: "9223372036854775807",
|
||||
minValue: "1",
|
||||
startWith: "1",
|
||||
});
|
||||
export const customJsonType = <T>() =>
|
||||
customType<{ data: T }>({
|
||||
dataType() {
|
||||
return "jsonb";
|
||||
},
|
||||
fromDriver(value) {
|
||||
return value as T;
|
||||
},
|
||||
toDriver(value) {
|
||||
return value; // JSONB → just pass the object
|
||||
},
|
||||
});
|
||||
|
||||
// legacy table for doctrine migrations
|
||||
export const doctrineMigrationVersions = pgTable("doctrine_migration_versions", {
|
||||
executedAt: timestamp("executed_at", { mode: "string" }).default(sql`NULL`),
|
||||
executionTime: integer("execution_time"),
|
||||
version: varchar({ length: 191 }).primaryKey().notNull(),
|
||||
});
|
||||
export const biasEnum = pgEnum("bias", ["neutral", "slightly", "partisan", "extreme"]);
|
||||
export const reliabilityEnum = pgEnum("reliability", [
|
||||
"trusted",
|
||||
"reliable",
|
||||
"average",
|
||||
"low_trust",
|
||||
"unreliable",
|
||||
]);
|
||||
export const sentimentEnum = pgEnum("sentiment", ["positive", "neutral", "negative"]);
|
||||
export const transparencyEnum = pgEnum("transparency", ["high", "medium", "low"]);
|
||||
export const tokenPurposeEnum = pgEnum("token_purpose", [
|
||||
"confirm_account",
|
||||
"password_reset",
|
||||
"unlock_account",
|
||||
"delete_account",
|
||||
]);
|
||||
|
||||
export const bookmark = pgTable(
|
||||
"bookmark",
|
||||
export type EmailAddress = string;
|
||||
export type Link = string;
|
||||
export type ReadingTime = number;
|
||||
|
||||
export type Role = "ROLE_USER" | "ROLE_ADMIN";
|
||||
export type Roles = Role[];
|
||||
|
||||
export type Bias = (typeof biasEnum.enumValues)[number];
|
||||
export type Reliability = (typeof reliabilityEnum.enumValues)[number];
|
||||
export type Sentiment = (typeof sentimentEnum.enumValues)[number];
|
||||
export type Transparency = (typeof transparencyEnum.enumValues)[number];
|
||||
export type TokenPurpose = (typeof tokenPurposeEnum.enumValues)[number];
|
||||
|
||||
export type Credibility = {
|
||||
bias: Bias;
|
||||
reliability: Reliability;
|
||||
transparency: Transparency;
|
||||
};
|
||||
|
||||
export type TokenStatistics = {
|
||||
title: number;
|
||||
body: number;
|
||||
categories: number;
|
||||
excerpt: number;
|
||||
total: number;
|
||||
};
|
||||
|
||||
export type Device = {
|
||||
operatingSystem?: string;
|
||||
client?: string;
|
||||
device?: string;
|
||||
isBot: boolean;
|
||||
};
|
||||
|
||||
export type GeoLocation = {
|
||||
country?: string;
|
||||
city?: string;
|
||||
timeZone?: string;
|
||||
longitude?: number;
|
||||
latitude?: number;
|
||||
accuracyRadius?: number;
|
||||
};
|
||||
|
||||
export type ClientProfile = {
|
||||
userIp?: string;
|
||||
userAgent?: string;
|
||||
hints: unknown[];
|
||||
};
|
||||
|
||||
export type ArticleMetadata = {
|
||||
title?: string;
|
||||
description?: string;
|
||||
image?: string;
|
||||
};
|
||||
|
||||
export type DateRange = {
|
||||
start: number; // unix timestamp (seconds)
|
||||
end: number; // unix timestamp (seconds)
|
||||
};
|
||||
|
||||
// Secrets
|
||||
export type GeneratedToken = string;
|
||||
export type GeneratedCode = string;
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
/* Tables */
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
export const user = pgTable(
|
||||
"user",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
description: varchar({ length: 512 }).default(sql`NULL`),
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
email: varchar({ length: 255 }).$type<EmailAddress>().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isPublic: boolean("is_public").default(false).notNull(),
|
||||
isConfirmed: boolean("is_confirmed").default(false).notNull(),
|
||||
isLocked: boolean("is_locked").default(false).notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
userId: uuid("user_id").notNull(),
|
||||
password: varchar({ length: 512 }).notNull(),
|
||||
roles: varchar("roles", { length: 255 })
|
||||
.$type<Roles>()
|
||||
.array()
|
||||
.notNull()
|
||||
.default(["ROLE_USER"]),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_bookmark_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast().op("timestamp_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
index("idx_da62921da76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_da62921da76ed395",
|
||||
}).onDelete("cascade"),
|
||||
(_table) => [
|
||||
uniqueIndex("unq_user_email").using("btree", sql`lower((email)::text)`),
|
||||
index("idx_user_created_at").using("btree", sql`created_at`),
|
||||
sql`CONSTRAINT "chk_user_roles_json" CHECK (jsonb_typeof(roles) = 'array')`,
|
||||
],
|
||||
);
|
||||
|
||||
export const loginAttempt = pgTable(
|
||||
"login_attempt",
|
||||
export const source = pgTable(
|
||||
"source",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
credibility: jsonb("credibility").$type<Credibility>(),
|
||||
description: varchar({ length: 1024 }),
|
||||
displayName: varchar("display_name", { length: 255 }),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
url: varchar({ length: 255 }).notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_8c11c1ba76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_login_attempt_created_at").using(
|
||||
"btree",
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_8c11c1ba76ed395",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const loginHistory = pgTable(
|
||||
"login_history",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
deviceClient: varchar("device_client", { length: 255 }).default(sql`NULL`),
|
||||
deviceDevice: varchar("device_device", { length: 255 }).default(sql`NULL`),
|
||||
deviceIsBot: boolean("device_is_bot").default(false).notNull(),
|
||||
deviceOperatingSystem: varchar("device_operating_system", { length: 255 }).default(sql`NULL`),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
ipAddress: inet("ip_address"),
|
||||
locationAccuracyRadius: integer("location_accuracy_radius"),
|
||||
locationLatitude: doublePrecision("location_latitude"),
|
||||
locationLongitude: doublePrecision("location_longitude"),
|
||||
locationTimeZone: varchar("location_time_zone", { length: 255 }).default(sql`NULL`),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_37976e36a76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_login_history_created_at").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast().op("uuid_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
index("idx_login_history_ip_address").using(
|
||||
"btree",
|
||||
table.ipAddress.asc().nullsLast().op("inet_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_37976e36a76ed395",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const verificationToken = pgTable(
|
||||
"verification_token",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
purpose: varchar({ length: 255 }).notNull(),
|
||||
token: varchar({ length: 60 }).default(sql`NULL`),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_c1cc006ba76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_verif_token_created_at").using(
|
||||
"btree",
|
||||
table.createdAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
),
|
||||
uniqueIndex("unq_verif_user_purpose_token")
|
||||
.using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast().op("text_ops"),
|
||||
table.purpose.asc().nullsLast().op("text_ops"),
|
||||
)
|
||||
.where(sql`(token IS NOT NULL)`),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_c1cc006ba76ed395",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const followedSource = pgTable(
|
||||
"followed_source",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
followerId: uuid("follower_id").notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
sourceId: uuid("source_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_7a763a3e953c1c61").using("btree", table.sourceId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_7a763a3eac24f853").using("btree", table.followerId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_followed_source_follower_created").using(
|
||||
"btree",
|
||||
table.followerId.asc().nullsLast().op("timestamp_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("uuid_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.followerId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_7a763a3eac24f853",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.sourceId],
|
||||
foreignColumns: [source.id],
|
||||
name: "fk_7a763a3e953c1c61",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const comment = pgTable(
|
||||
"comment",
|
||||
{
|
||||
articleId: uuid("article_id").notNull(),
|
||||
content: varchar({ length: 512 }).notNull(),
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isSpam: boolean("is_spam").default(false).notNull(),
|
||||
sentiment: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_9474526c7294869c").using("btree", table.articleId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_9474526ca76ed395").using("btree", table.userId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_comment_article_created").using(
|
||||
"btree",
|
||||
table.articleId.asc().nullsLast().op("timestamp_ops"),
|
||||
table.createdAt.desc().nullsFirst().op("uuid_ops"),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_9474526ca76ed395",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.articleId],
|
||||
foreignColumns: [article.id],
|
||||
name: "fk_9474526c7294869c",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const refreshTokens = pgTable(
|
||||
"refresh_tokens",
|
||||
{
|
||||
id: integer().primaryKey().notNull(),
|
||||
refreshToken: varchar("refresh_token", { length: 128 }).notNull(),
|
||||
username: varchar({ length: 255 }).notNull(),
|
||||
valid: timestamp({ mode: "string" }).notNull(),
|
||||
},
|
||||
(table) => [
|
||||
uniqueIndex("uniq_9bace7e1c74f2195").using(
|
||||
"btree",
|
||||
table.refreshToken.asc().nullsLast().op("text_ops"),
|
||||
),
|
||||
(_table) => [
|
||||
uniqueIndex("unq_source_name").using("btree", sql`lower((name)::text)`),
|
||||
uniqueIndex("unq_source_url").using("btree", sql`lower((url)::text)`),
|
||||
],
|
||||
);
|
||||
|
||||
export const article = pgTable(
|
||||
"article",
|
||||
{
|
||||
bias: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
body: text().notNull(),
|
||||
categories: text().array(),
|
||||
crawledAt: timestamp("crawled_at", { mode: "string" }).notNull(),
|
||||
crawledAt: timestamp("crawled_at").defaultNow().notNull(),
|
||||
credibility: jsonb("credibility").$type<Credibility>(),
|
||||
excerpt: varchar({ length: 255 }).generatedAlwaysAs(sql`("left"(body, 200) || '...'::text)`),
|
||||
hash: varchar({ length: 32 }).notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
image: varchar({ length: 1024 }).generatedAlwaysAs(sql`(metadata ->> 'image'::text)`),
|
||||
link: varchar({ length: 1024 }).notNull(),
|
||||
metadata: jsonb(),
|
||||
publishedAt: timestamp("published_at", { mode: "string" }).notNull(),
|
||||
metadata: jsonb("metadata").$type<ArticleMetadata>(),
|
||||
publishedAt: timestamp("published_at").notNull(),
|
||||
readingTime: integer("reading_time").default(1),
|
||||
reliability: varchar({ length: 30 }).default("reliable").notNull(),
|
||||
sentiment: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
sentiment: sentimentEnum("sentiment").notNull(),
|
||||
sourceId: uuid("source_id").notNull(),
|
||||
title: varchar({ length: 1024 }).notNull(),
|
||||
tokenStatistics: jsonb("token_statistics"),
|
||||
transparency: varchar({ length: 30 }).default("medium").notNull(),
|
||||
tokenStatistics: jsonb("token_statistics").$type<TokenStatistics>(),
|
||||
tsv: tsvector("tsv").generatedAlwaysAs(
|
||||
sql`(setweight(to_tsvector('french'::regconfig, (COALESCE(title, ''::character varying))::text), 'A'::"char") || setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char"))`,
|
||||
sql`(
|
||||
setweight(to_tsvector('french'::regconfig, COALESCE(title, '')::text), 'A'::"char")
|
||||
|| setweight(to_tsvector('french'::regconfig, COALESCE(body, ''::text)), 'B'::"char")
|
||||
)`,
|
||||
),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
},
|
||||
(table) => [
|
||||
index("gin_article_categories").using(
|
||||
@@ -264,69 +202,57 @@ export const article = pgTable(
|
||||
index("gin_article_link_trgm").using("gin", table.link.asc().nullsLast().op("gin_trgm_ops")),
|
||||
index("gin_article_title_trgm").using("gin", table.title.asc().nullsLast().op("gin_trgm_ops")),
|
||||
index("gin_article_tsv").using("gin", table.tsv.asc().nullsLast().op("tsvector_ops")),
|
||||
index("idx_23a0e66953c1c61").using("btree", table.sourceId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_article_published_at").using(
|
||||
index("idx_article_source_published_id").using(
|
||||
"btree",
|
||||
table.publishedAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
table.sourceId.asc().nullsLast(),
|
||||
table.publishedAt.desc().nullsFirst(),
|
||||
table.id.desc().nullsFirst(),
|
||||
),
|
||||
index("idx_article_published_id").using(
|
||||
"btree",
|
||||
table.publishedAt.desc().nullsFirst().op("timestamp_ops"),
|
||||
table.id.desc().nullsFirst().op("uuid_ops"),
|
||||
),
|
||||
uniqueIndex("unq_article_hash").using("btree", table.hash.asc().nullsLast().op("text_ops")),
|
||||
uniqueIndex("unq_article_hash").using("btree", table.hash.asc().nullsLast()),
|
||||
foreignKey({
|
||||
columns: [table.sourceId],
|
||||
foreignColumns: [source.id],
|
||||
name: "fk_23a0e66953c1c61",
|
||||
name: "fk_article_source_id",
|
||||
}).onDelete("cascade"),
|
||||
check("chk_article_reading_time", sql`reading_time >= 0`),
|
||||
check("chk_article_reading_time", sql`(reading_time >= 0)`),
|
||||
check(
|
||||
"chk_article_sentiment",
|
||||
sql`(sentiment)::text = ANY ((ARRAY['positive'::character varying, 'neutral'::character varying, 'negative'::character varying])::text[])`,
|
||||
sql`((sentiment)::text = ANY (ARRAY['positive'::text,'neutral'::text,'negative'::text]))`,
|
||||
),
|
||||
check(
|
||||
"chk_article_metadata_json",
|
||||
sql`(metadata IS NULL) OR (jsonb_typeof(metadata) = ANY (ARRAY['object'::text, 'array'::text]))`,
|
||||
sql`((metadata IS NULL) OR (jsonb_typeof(metadata) IN ('object'::text,'array'::text)))`,
|
||||
),
|
||||
],
|
||||
);
|
||||
|
||||
export const user = pgTable(
|
||||
"user",
|
||||
export const bookmark = pgTable(
|
||||
"bookmark",
|
||||
{
|
||||
createdAt: timestamp("created_at", { mode: "string" }).notNull(),
|
||||
email: varchar({ length: 255 }).notNull(),
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
description: varchar({ length: 512 }),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isConfirmed: boolean("is_confirmed").default(false).notNull(),
|
||||
isLocked: boolean("is_locked").default(false).notNull(),
|
||||
isPublic: boolean("is_public").default(false).notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
password: varchar({ length: 512 }).notNull(),
|
||||
roles: jsonb().notNull(),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
updatedAt: timestamp("updated_at"),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(_table) => [
|
||||
uniqueIndex("unq_user_email").using("btree", sql`lower((email)::text)`),
|
||||
check("chk_user_roles_json", sql`jsonb_typeof(roles) = 'array'::text`),
|
||||
],
|
||||
);
|
||||
|
||||
export const source = pgTable(
|
||||
"source",
|
||||
{
|
||||
bias: varchar({ length: 30 }).default("neutral").notNull(),
|
||||
description: varchar({ length: 1024 }).default(sql`NULL`),
|
||||
displayName: varchar("display_name", { length: 255 }).default(sql`NULL`),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
name: varchar({ length: 255 }).notNull(),
|
||||
reliability: varchar({ length: 30 }).default("reliable").notNull(),
|
||||
transparency: varchar({ length: 30 }).default("medium").notNull(),
|
||||
updatedAt: timestamp("updated_at", { mode: "string" }).default(sql`NULL`),
|
||||
url: varchar({ length: 255 }).notNull(),
|
||||
},
|
||||
(_table) => [
|
||||
uniqueIndex("unq_source_name").using("btree", sql`lower((name)::text)`),
|
||||
uniqueIndex("unq_source_url").using("btree", sql`lower((url)::text)`),
|
||||
(table) => [
|
||||
index("idx_bookmark_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
uniqueIndex("unq_bookmark_user_name").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
sql`lower(${table.name})`,
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_bookmark_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
@@ -337,22 +263,177 @@ export const bookmarkArticle = pgTable(
|
||||
bookmarkId: uuid("bookmark_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_6fe2655d7294869c").using("btree", table.articleId.asc().nullsLast().op("uuid_ops")),
|
||||
index("idx_6fe2655d92741d25").using("btree", table.bookmarkId.asc().nullsLast().op("uuid_ops")),
|
||||
primaryKey({ columns: [table.bookmarkId, table.articleId], name: "bookmark_article_pkey" }),
|
||||
index("idx_bookmark_article_bookmark_id").using("btree", table.bookmarkId.asc().nullsLast()),
|
||||
foreignKey({
|
||||
columns: [table.bookmarkId],
|
||||
foreignColumns: [bookmark.id],
|
||||
name: "fk_6fe2655d92741d25",
|
||||
name: "fk_bookmark_article_bookmark_id",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.articleId],
|
||||
foreignColumns: [article.id],
|
||||
name: "fk_6fe2655d7294869c",
|
||||
name: "fk_bookmark_article_article_id",
|
||||
}).onDelete("cascade"),
|
||||
primaryKey({ columns: [table.bookmarkId, table.articleId], name: "bookmark_article_pkey" }),
|
||||
],
|
||||
);
|
||||
|
||||
export const loginAttempt = pgTable(
|
||||
"login_attempt",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_login_attempt_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_login_attempt_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const loginHistory = pgTable(
|
||||
"login_history",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
device: jsonb("device").$type<Device>(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
ipAddress: inet("ip_address"),
|
||||
location: jsonb("location").$type<GeoLocation>(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_login_history_user_created").using(
|
||||
"btree",
|
||||
table.userId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
index("idx_login_history_ip_address").using("btree", table.ipAddress.asc().nullsLast()),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_login_history_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const verificationToken = pgTable(
|
||||
"verification_token",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
purpose: tokenPurposeEnum("purpose").notNull(),
|
||||
token: varchar({ length: 60 }), // nullable if you support "reservations" before issue
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_verif_token_created_at").using("btree", table.createdAt.desc().nullsFirst()),
|
||||
uniqueIndex("unq_verif_user_purpose_token")
|
||||
.using("btree", table.userId, table.purpose, table.token)
|
||||
.where(sql`${table.token} IS NOT NULL`),
|
||||
uniqueIndex("unq_verif_token_token")
|
||||
.using("btree", table.token)
|
||||
.where(sql`${table.token} IS NOT NULL`),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_verification_token_user_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const followedSource = pgTable(
|
||||
"followed_source",
|
||||
{
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
followerId: uuid("follower_id").notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
sourceId: uuid("source_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_followed_source_source_id").using("btree", table.sourceId.asc().nullsLast()),
|
||||
index("idx_followed_source_follower_id").using("btree", table.followerId.asc().nullsLast()),
|
||||
index("idx_followed_source_follower_created").using(
|
||||
"btree",
|
||||
table.followerId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
uniqueIndex("unq_followed_source_user_source").using(
|
||||
"btree",
|
||||
table.followerId.asc().nullsLast(),
|
||||
table.sourceId.asc().nullsLast(),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.followerId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_followed_source_follower_id",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.sourceId],
|
||||
foreignColumns: [source.id],
|
||||
name: "fk_followed_source_source_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const comment = pgTable(
|
||||
"comment",
|
||||
{
|
||||
articleId: uuid("article_id").notNull(),
|
||||
content: varchar({ length: 512 }).notNull(),
|
||||
createdAt: timestamp("created_at").defaultNow().notNull(),
|
||||
id: uuid().primaryKey().notNull(),
|
||||
isSpam: boolean("is_spam").default(false).notNull(),
|
||||
sentiment: sentimentEnum("sentiment").notNull(),
|
||||
userId: uuid("user_id").notNull(),
|
||||
},
|
||||
(table) => [
|
||||
index("idx_comment_article_id").using("btree", table.articleId.asc().nullsLast()),
|
||||
index("idx_comment_user_id").using("btree", table.userId.asc().nullsLast()),
|
||||
index("idx_comment_article_created").using(
|
||||
"btree",
|
||||
table.articleId.asc().nullsLast(),
|
||||
table.createdAt.desc().nullsFirst(),
|
||||
),
|
||||
foreignKey({
|
||||
columns: [table.userId],
|
||||
foreignColumns: [user.id],
|
||||
name: "fk_comment_user_id",
|
||||
}).onDelete("cascade"),
|
||||
foreignKey({
|
||||
columns: [table.articleId],
|
||||
foreignColumns: [article.id],
|
||||
name: "fk_comment_article_id",
|
||||
}).onDelete("cascade"),
|
||||
],
|
||||
);
|
||||
|
||||
export const refreshToken = pgTable(
|
||||
"refresh_token",
|
||||
{
|
||||
id: uuid().primaryKey().notNull(),
|
||||
token: varchar("token", { length: 128 }).notNull(),
|
||||
username: varchar({ length: 255 }).notNull(),
|
||||
valid: timestamp().notNull(),
|
||||
},
|
||||
(table) => [
|
||||
uniqueIndex("uniq_refresh_token_token").using("btree", table.token.asc().nullsLast()),
|
||||
index("idx_refresh_token_valid").using("btree", table.valid.asc().nullsLast()),
|
||||
index("idx_refresh_token_username").using("btree", sql`lower(${table.username})`),
|
||||
],
|
||||
);
|
||||
|
||||
/* -------------------------------------------------------------------------- */
|
||||
/* Relations */
|
||||
/* -------------------------------------------------------------------------- */
|
||||
|
||||
export const bookmarkRelations = relations(bookmark, ({ one, many }) => ({
|
||||
bookmarkArticles: many(bookmarkArticle),
|
||||
user: one(user, {
|
||||
|
||||
@@ -0,0 +1,57 @@
|
||||
import { TiktokenEncoding, get_encoding } from "tiktoken";
|
||||
|
||||
import { TokenStatistics } from "@/schema";
|
||||
|
||||
/**
|
||||
* Count the number of tokens in the given text using the specified encoding.
|
||||
* @param text - The input text
|
||||
* @param encoding - The token encoding (default: "cl100k_base")
|
||||
*/
|
||||
export const computeTokenCount = (
|
||||
text: string,
|
||||
encoding: TiktokenEncoding = "cl100k_base",
|
||||
): number => {
|
||||
try {
|
||||
const encoder = get_encoding(encoding);
|
||||
const tokens = encoder.encode(text);
|
||||
encoder.free();
|
||||
return tokens.length;
|
||||
} catch {
|
||||
return text.length;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* Create token statistics for the given data.
|
||||
* @param data - The input data containing title, body, and categories
|
||||
* @returns TokenStatistics object
|
||||
*/
|
||||
export const computeTokenStatistics = (data: {
|
||||
title: string;
|
||||
body: string;
|
||||
categories: string[];
|
||||
}): TokenStatistics => {
|
||||
const title = computeTokenCount(data.title);
|
||||
const body = computeTokenCount(data.body);
|
||||
const categories = computeTokenCount(data.categories.join(","));
|
||||
const excerpt = computeTokenCount(data.body.substring(0, 200));
|
||||
|
||||
return {
|
||||
body,
|
||||
categories,
|
||||
excerpt,
|
||||
title,
|
||||
total: title + body + categories + excerpt,
|
||||
};
|
||||
};
|
||||
|
||||
/**
|
||||
* Compute the estimated reading time for the given text.
|
||||
* @param text - The input text
|
||||
* @param wordsPerMinute - The reading speed in words per minute (default: 200)
|
||||
* @returns The estimated reading time in minutes
|
||||
*/
|
||||
export const computeReadingTime = (text: string, wordsPerMinute = 200): number => {
|
||||
const words = text.trim().split(/\s+/).length;
|
||||
return Math.ceil(words / wordsPerMinute);
|
||||
};
|
||||
@@ -6,8 +6,6 @@ import {
|
||||
PAGINATION_MAX_LIMIT,
|
||||
} from "@/constants";
|
||||
|
||||
export type SortDirection = "asc" | "desc";
|
||||
|
||||
export interface PageRequest {
|
||||
page?: number;
|
||||
limit?: number;
|
||||
|
||||
Reference in New Issue
Block a user