diff --git a/AGENTS.md b/AGENTS.md new file mode 100644 index 0000000..67679af --- /dev/null +++ b/AGENTS.md @@ -0,0 +1,7 @@ +# porting to Typescript (./basango) +- when working on a porting feature (working directory is basango, refer to docs/ for more instructions) +- make sure to use `bun` 1.3 as runtime not `node` +- prefer `const fn = () => {}` instead of `function` + +# legacy (./projects) +- when working on a legacy feature (working director is projects) diff --git a/basango/apps/crawler/config/pipeline.json b/basango/apps/crawler/config/pipeline.json index 1182893..fb64dbf 100644 --- a/basango/apps/crawler/config/pipeline.json +++ b/basango/apps/crawler/config/pipeline.json @@ -57,13 +57,7 @@ "pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/", "replacement": "$3-$2-$1 $4" }, - "categories": [ - "politique", - "economie", - "culture", - "sport", - "societe" - ], + "categories": ["politique", "economie", "culture", "sport", "societe"], "source_selectors": { "articles": ".view-content > .row.views-row", "article_title": ".views-field-title a", @@ -119,31 +113,80 @@ } ], "wordpress": [ - { "source_id": "beto.cd", "source_url": "https://beto.cd", "requires_rate_limit": true }, + { + "source_id": "beto.cd", + "source_url": "https://beto.cd", + "requires_rate_limit": true + }, { "source_id": "newscd.net", "source_url": "https://newscd.net" }, - { "source_id": "africanewsrdc.net", "source_url": "https://www.africanewsrdc.net" }, - { "source_id": "angazainstitute.ac.cd", "source_url": "https://angazainstitute.ac.cd" }, + { + "source_id": "africanewsrdc.net", + "source_url": "https://www.africanewsrdc.net" + }, + { + "source_id": "angazainstitute.ac.cd", + "source_url": "https://angazainstitute.ac.cd" + }, { "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" }, { "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" }, - { "source_id": "changement7.net", "source_url": "https://changement7.net" }, + { + "source_id": "changement7.net", + "source_url": "https://changement7.net" + }, { "source_id": "congoactu.net", "source_url": "https://congoactu.net" }, - { "source_id": "congoindependant.com", "source_url": "https://www.congoindependant.com" }, - { "source_id": "congoquotidien.com", "source_url": "https://www.congoquotidien.com" }, + { + "source_id": "congoindependant.com", + "source_url": "https://www.congoindependant.com" + }, + { + "source_id": "congoquotidien.com", + "source_url": "https://www.congoquotidien.com" + }, { "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" }, - { "source_id": "environews-rdc.net", "source_url": "https://environews-rdc.net" }, - { "source_id": "freemediardc.info", "source_url": "https://www.freemediardc.info" }, - { "source_id": "geopolismagazine.org", "source_url": "https://geopolismagazine.org" }, + { + "source_id": "environews-rdc.net", + "source_url": "https://environews-rdc.net" + }, + { + "source_id": "freemediardc.info", + "source_url": "https://www.freemediardc.info" + }, + { + "source_id": "geopolismagazine.org", + "source_url": "https://geopolismagazine.org" + }, { "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" }, { "source_id": "infordc.com", "source_url": "https://infordc.com" }, - { "source_id": "kilalopress.net", "source_url": "https://kilalopress.net" }, - { "source_id": "laprosperiteonline.net", "source_url": "https://laprosperiteonline.net" }, - { "source_id": "laprunellerdc.cd", "source_url": "https://laprunellerdc.cd" }, + { + "source_id": "kilalopress.net", + "source_url": "https://kilalopress.net" + }, + { + "source_id": "laprosperiteonline.net", + "source_url": "https://laprosperiteonline.net" + }, + { + "source_id": "laprunellerdc.cd", + "source_url": "https://laprunellerdc.cd" + }, { "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" }, - { "source_id": "lesvolcansnews.net", "source_url": "https://lesvolcansnews.net" }, - { "source_id": "netic-news.net", "source_url": "https://www.netic-news.net" }, - { "source_id": "objectif-infos.cd", "source_url": "https://objectif-infos.cd" }, + { + "source_id": "lesvolcansnews.net", + "source_url": "https://lesvolcansnews.net" + }, + { + "source_id": "netic-news.net", + "source_url": "https://www.netic-news.net" + }, + { + "source_id": "objectif-infos.cd", + "source_url": "https://objectif-infos.cd" + }, { "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" }, - { "source_id": "journaldekinshasa.com", "source_url": "https://www.journaldekinshasa.com" }, + { + "source_id": "journaldekinshasa.com", + "source_url": "https://www.journaldekinshasa.com" + }, { "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" }, { "source_id": "acturdc.com", "source_url": "https://acturdc.com" }, { "source_id": "matininfos.net", "source_url": "https://matininfos.net" } diff --git a/basango/apps/crawler/config/pipeline.prod.json b/basango/apps/crawler/config/pipeline.prod.json index f7de292..83e9681 100644 --- a/basango/apps/crawler/config/pipeline.prod.json +++ b/basango/apps/crawler/config/pipeline.prod.json @@ -57,13 +57,7 @@ "pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/", "replacement": "$3-$2-$1 $4" }, - "categories": [ - "politique", - "economie", - "culture", - "sport", - "societe" - ], + "categories": ["politique", "economie", "culture", "sport", "societe"], "source_selectors": { "articles": ".view-content > .row.views-row", "article_title": ".views-field-title a", @@ -119,31 +113,80 @@ } ], "wordpress": [ - { "source_id": "beto.cd", "source_url": "https://beto.cd", "requires_rate_limit": true }, + { + "source_id": "beto.cd", + "source_url": "https://beto.cd", + "requires_rate_limit": true + }, { "source_id": "newscd.net", "source_url": "https://newscd.net" }, - { "source_id": "africanewsrdc.net", "source_url": "https://www.africanewsrdc.net" }, - { "source_id": "angazainstitute.ac.cd", "source_url": "https://angazainstitute.ac.cd" }, + { + "source_id": "africanewsrdc.net", + "source_url": "https://www.africanewsrdc.net" + }, + { + "source_id": "angazainstitute.ac.cd", + "source_url": "https://angazainstitute.ac.cd" + }, { "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" }, { "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" }, - { "source_id": "changement7.net", "source_url": "https://changement7.net" }, + { + "source_id": "changement7.net", + "source_url": "https://changement7.net" + }, { "source_id": "congoactu.net", "source_url": "https://congoactu.net" }, - { "source_id": "congoindependant.com", "source_url": "https://www.congoindependant.com" }, - { "source_id": "congoquotidien.com", "source_url": "https://www.congoquotidien.com" }, + { + "source_id": "congoindependant.com", + "source_url": "https://www.congoindependant.com" + }, + { + "source_id": "congoquotidien.com", + "source_url": "https://www.congoquotidien.com" + }, { "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" }, - { "source_id": "environews-rdc.net", "source_url": "https://environews-rdc.net" }, - { "source_id": "freemediardc.info", "source_url": "https://www.freemediardc.info" }, - { "source_id": "geopolismagazine.org", "source_url": "https://geopolismagazine.org" }, + { + "source_id": "environews-rdc.net", + "source_url": "https://environews-rdc.net" + }, + { + "source_id": "freemediardc.info", + "source_url": "https://www.freemediardc.info" + }, + { + "source_id": "geopolismagazine.org", + "source_url": "https://geopolismagazine.org" + }, { "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" }, { "source_id": "infordc.com", "source_url": "https://infordc.com" }, - { "source_id": "kilalopress.net", "source_url": "https://kilalopress.net" }, - { "source_id": "laprosperiteonline.net", "source_url": "https://laprosperiteonline.net" }, - { "source_id": "laprunellerdc.cd", "source_url": "https://laprunellerdc.cd" }, + { + "source_id": "kilalopress.net", + "source_url": "https://kilalopress.net" + }, + { + "source_id": "laprosperiteonline.net", + "source_url": "https://laprosperiteonline.net" + }, + { + "source_id": "laprunellerdc.cd", + "source_url": "https://laprunellerdc.cd" + }, { "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" }, - { "source_id": "lesvolcansnews.net", "source_url": "https://lesvolcansnews.net" }, - { "source_id": "netic-news.net", "source_url": "https://www.netic-news.net" }, - { "source_id": "objectif-infos.cd", "source_url": "https://objectif-infos.cd" }, + { + "source_id": "lesvolcansnews.net", + "source_url": "https://lesvolcansnews.net" + }, + { + "source_id": "netic-news.net", + "source_url": "https://www.netic-news.net" + }, + { + "source_id": "objectif-infos.cd", + "source_url": "https://objectif-infos.cd" + }, { "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" }, - { "source_id": "journaldekinshasa.com", "source_url": "https://www.journaldekinshasa.com" }, + { + "source_id": "journaldekinshasa.com", + "source_url": "https://www.journaldekinshasa.com" + }, { "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" }, { "source_id": "acturdc.com", "source_url": "https://acturdc.com" }, { "source_id": "matininfos.net", "source_url": "https://matininfos.net" } diff --git a/basango/apps/crawler/package.json b/basango/apps/crawler/package.json index 3b1d16c..834f2ac 100644 --- a/basango/apps/crawler/package.json +++ b/basango/apps/crawler/package.json @@ -1,22 +1,22 @@ { - "name": "@basango/crawler", - "version": "0.1.0", - "private": true, - "type": "module", - "main": "dist/index.js", - "types": "dist/index.d.ts", - "scripts": { - "build": "tsc -b", - "test": "vitest --run", - "queue": "bun run src/scripts/queue.ts", - "worker": "bun run src/scripts/worker.ts" - }, - "dependencies": { - "@basango/logger": "workspace:*", - "bullmq": "^4.17.0", - "date-fns": "^3.6.0", - "ioredis": "^5.3.2", - "tiktoken": "^1.0.14", - "zod": "^4.0.0" - } + "name": "@basango/crawler", + "version": "0.1.0", + "private": true, + "type": "module", + "main": "dist/index.js", + "types": "dist/index.d.ts", + "scripts": { + "build": "tsc -b", + "test": "vitest --run", + "queue": "bun run src/scripts/queue.ts", + "worker": "bun run src/scripts/worker.ts" + }, + "dependencies": { + "@basango/logger": "workspace:*", + "bullmq": "^4.17.0", + "date-fns": "^3.6.0", + "ioredis": "^5.3.2", + "tiktoken": "^1.0.14", + "zod": "catalog:" + } } diff --git a/basango/apps/crawler/src/__tests__/config.test.ts b/basango/apps/crawler/src/__tests__/config.test.ts index 8aa014c..14a5ea3 100644 --- a/basango/apps/crawler/src/__tests__/config.test.ts +++ b/basango/apps/crawler/src/__tests__/config.test.ts @@ -8,74 +8,74 @@ import { loadConfig } from "./config"; import { resolveConfigPath } from "./schema"; describe("loadConfig", () => { - it("parses json configuration and ensures directories", () => { - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-")); - const paths = { - root: tempDir, - data: path.join(tempDir, "data"), - logs: path.join(tempDir, "logs"), - configs: path.join(tempDir, "configs"), - }; + it("parses json configuration and ensures directories", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-")); + const paths = { + root: tempDir, + data: path.join(tempDir, "data"), + logs: path.join(tempDir, "logs"), + configs: path.join(tempDir, "configs"), + }; - const configPath = path.join(tempDir, "pipeline.json"); - fs.writeFileSync( - configPath, - JSON.stringify( - { - paths, - fetch: { - client: { timeout: 10 }, - }, - }, - null, - 2, - ), - ); + const configPath = path.join(tempDir, "pipeline.json"); + fs.writeFileSync( + configPath, + JSON.stringify( + { + paths, + fetch: { + client: { timeout: 10 }, + }, + }, + null, + 2, + ), + ); - const config = loadConfig({ configPath }); + const config = loadConfig({ configPath }); - expect(config.fetch.client.timeout).toBe(10); - expect(fs.existsSync(paths.data)).toBe(true); - expect(fs.existsSync(paths.logs)).toBe(true); - expect(fs.existsSync(paths.configs)).toBe(true); - }); + expect(config.fetch.client.timeout).toBe(10); + expect(fs.existsSync(paths.data)).toBe(true); + expect(fs.existsSync(paths.logs)).toBe(true); + expect(fs.existsSync(paths.configs)).toBe(true); + }); - it("merges environment override if available", () => { - const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-")); - const paths = { - root: tempDir, - data: path.join(tempDir, "data"), - logs: path.join(tempDir, "logs"), - configs: path.join(tempDir, "configs"), - }; + it("merges environment override if available", () => { + const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-")); + const paths = { + root: tempDir, + data: path.join(tempDir, "data"), + logs: path.join(tempDir, "logs"), + configs: path.join(tempDir, "configs"), + }; - const basePath = path.join(tempDir, "pipeline.json"); - fs.writeFileSync( - basePath, - JSON.stringify( - { - paths, - logging: { level: "INFO" }, - }, - null, - 2, - ), - ); + const basePath = path.join(tempDir, "pipeline.json"); + fs.writeFileSync( + basePath, + JSON.stringify( + { + paths, + logging: { level: "INFO" }, + }, + null, + 2, + ), + ); - const overridePath = resolveConfigPath(basePath, "production"); - fs.writeFileSync( - overridePath, - JSON.stringify( - { - logging: { level: "DEBUG" }, - }, - null, - 2, - ), - ); + const overridePath = resolveConfigPath(basePath, "production"); + fs.writeFileSync( + overridePath, + JSON.stringify( + { + logging: { level: "DEBUG" }, + }, + null, + 2, + ), + ); - const config = loadConfig({ configPath: basePath, env: "production" }); + const config = loadConfig({ configPath: basePath, env: "production" }); - expect(config.logging.level).toBe("DEBUG"); - }); + expect(config.logging.level).toBe("DEBUG"); + }); }); diff --git a/basango/apps/crawler/src/__tests__/queue.test.ts b/basango/apps/crawler/src/__tests__/queue.test.ts index fc2c5cc..2d971c4 100644 --- a/basango/apps/crawler/src/__tests__/queue.test.ts +++ b/basango/apps/crawler/src/__tests__/queue.test.ts @@ -3,56 +3,56 @@ import { describe, expect, it } from "vitest"; import { createQueueManager, createQueueSettings } from "./queue"; class InMemoryQueue { - public jobs: Array<{ name: string; data: unknown }> = []; + public jobs: Array<{ name: string; data: unknown }> = []; - async add(name: string, data: unknown) { - this.jobs.push({ name, data }); - return { id: `${name}-${this.jobs.length}` }; - } + async add(name: string, data: unknown) { + this.jobs.push({ name, data }); + return { id: `${name}-${this.jobs.length}` }; + } } describe("createQueueManager", () => { - it("prefixes queue names", () => { - const manager = createQueueManager({ - settings: createQueueSettings({ prefix: "test" }), - queueFactory: (queueName) => { - expect(queueName).toBe("listing"); - return new InMemoryQueue(); - }, - connection: { - quit: async () => undefined, - } as any, - }); + it("prefixes queue names", () => { + const manager = createQueueManager({ + settings: createQueueSettings({ prefix: "test" }), + queueFactory: (queueName) => { + expect(queueName).toBe("listing"); + return new InMemoryQueue(); + }, + connection: { + quit: async () => undefined, + } as any, + }); - expect(manager.iterQueueNames()).toEqual([ - "test:listing", - "test:articles", - "test:processed", - ]); - }); + expect(manager.iterQueueNames()).toEqual([ + "test:listing", + "test:articles", + "test:processed", + ]); + }); - it("enqueues listing job with validated payload", async () => { - const queue = new InMemoryQueue(); - const manager = createQueueManager({ - queueFactory: () => queue, - connection: { quit: async () => undefined } as any, - }); + it("enqueues listing job with validated payload", async () => { + const queue = new InMemoryQueue(); + const manager = createQueueManager({ + queueFactory: () => queue, + connection: { quit: async () => undefined } as any, + }); - const job = await manager.enqueueListing({ - source_id: "radiookapi", - env: "test", - }); + const job = await manager.enqueueListing({ + source_id: "radiookapi", + env: "test", + }); - expect(job.id).toBe("collect_listing-1"); - expect(queue.jobs[0]).toEqual({ - name: "collect_listing", - data: { - source_id: "radiookapi", - env: "test", - page_range: undefined, - date_range: undefined, - category: undefined, - }, - }); - }); + expect(job.id).toBe("collect_listing-1"); + expect(queue.jobs[0]).toEqual({ + name: "collect_listing", + data: { + source_id: "radiookapi", + env: "test", + page_range: undefined, + date_range: undefined, + category: undefined, + }, + }); + }); }); diff --git a/basango/apps/crawler/src/__tests__/queue.ts b/basango/apps/crawler/src/__tests__/queue.ts index 109e99a..aef3df3 100644 --- a/basango/apps/crawler/src/__tests__/queue.ts +++ b/basango/apps/crawler/src/__tests__/queue.ts @@ -1 +1 @@ -export * from "../services/crawler/async/queue"; +export * from "@basango/crawler/services/async/queue"; diff --git a/basango/apps/crawler/src/__tests__/schema.test.ts b/basango/apps/crawler/src/__tests__/schema.test.ts index db02185..3b86eb3 100644 --- a/basango/apps/crawler/src/__tests__/schema.test.ts +++ b/basango/apps/crawler/src/__tests__/schema.test.ts @@ -1,35 +1,35 @@ import { describe, expect, it } from "vitest"; import { - PipelineConfigSchema, - createDateRange, - formatDateRange, - isTimestampInRange, - PageRangeSpecSchema, - PageRangeSchema, - schemaToJSON, + PipelineConfigSchema, + createDateRange, + formatDateRange, + isTimestampInRange, + PageRangeSpecSchema, + PageRangeSchema, + schemaToJSON, } from "./schema"; describe("schema helpers", () => { - it("creates date range from spec", () => { - const range = createDateRange("2024-01-01:2024-01-31"); - expect(range.start).toBeLessThan(range.end); - expect(formatDateRange(range)).toBe("2024-01-01:2024-01-31"); - }); + it("creates date range from spec", () => { + const range = createDateRange("2024-01-01:2024-01-31"); + expect(range.start).toBeLessThan(range.end); + expect(formatDateRange(range)).toBe("2024-01-01:2024-01-31"); + }); - it("checks membership", () => { - const range = createDateRange("2024-01-01:2024-01-02"); - expect(isTimestampInRange(range, range.start)).toBe(true); - expect(isTimestampInRange(range, range.start - 1)).toBe(false); - }); + it("checks membership", () => { + const range = createDateRange("2024-01-01:2024-01-02"); + expect(isTimestampInRange(range, range.start)).toBe(true); + expect(isTimestampInRange(range, range.start - 1)).toBe(false); + }); - it("parses page range spec", () => { - const range = PageRangeSchema.parse(PageRangeSpecSchema.parse("1:10")); - expect(range).toEqual({ start: 1, end: 10 }); - }); + it("parses page range spec", () => { + const range = PageRangeSchema.parse(PageRangeSpecSchema.parse("1:10")); + expect(range).toEqual({ start: 1, end: 10 }); + }); - it("produces json schema", () => { - const json = schemaToJSON(PipelineConfigSchema); - expect(json.type).toBe("object"); - }); + it("produces json schema", () => { + const json = schemaToJSON(PipelineConfigSchema); + expect(json.type).toBe("object"); + }); }); diff --git a/basango/apps/crawler/src/__tests__/tasks.test.ts b/basango/apps/crawler/src/__tests__/tasks.test.ts index c779fa2..17c613b 100644 --- a/basango/apps/crawler/src/__tests__/tasks.test.ts +++ b/basango/apps/crawler/src/__tests__/tasks.test.ts @@ -1,50 +1,50 @@ import { describe, expect, it, vi } from "vitest"; import { - scheduleAsyncCrawl, - registerCrawlerTaskHandlers, - collectListing, + scheduleAsyncCrawl, + registerCrawlerTaskHandlers, + collectListing, } from "./tasks"; import { QueueManager } from "./queue"; describe("Async tasks", () => { - it("schedules crawl with provided manager", async () => { - const enqueueListing = vi.fn().mockResolvedValue({ id: "job-1" }); - const manager = { - enqueueListing, - } as unknown as QueueManager; + it("schedules crawl with provided manager", async () => { + const enqueueListing = vi.fn().mockResolvedValue({ id: "job-1" }); + const manager = { + enqueueListing, + } as unknown as QueueManager; - const jobId = await scheduleAsyncCrawl({ - sourceId: "radiookapi", - queueManager: manager, - }); + const jobId = await scheduleAsyncCrawl({ + sourceId: "radiookapi", + queueManager: manager, + }); - expect(jobId).toBe("job-1"); - expect(enqueueListing).toHaveBeenCalledWith({ - source_id: "radiookapi", - env: "development", - page_range: undefined, - date_range: undefined, - category: undefined, - }); - }); + expect(jobId).toBe("job-1"); + expect(enqueueListing).toHaveBeenCalledWith({ + source_id: "radiookapi", + env: "development", + page_range: undefined, + date_range: undefined, + category: undefined, + }); + }); - it("delegates listing collection to registered handler", async () => { - const handler = vi.fn().mockResolvedValue(5); - registerCrawlerTaskHandlers({ collectListing: handler }); + it("delegates listing collection to registered handler", async () => { + const handler = vi.fn().mockResolvedValue(5); + registerCrawlerTaskHandlers({ collectListing: handler }); - const count = await collectListing({ - source_id: "radiookapi", - env: "development", - }); + const count = await collectListing({ + source_id: "radiookapi", + env: "development", + }); - expect(count).toBe(5); - expect(handler).toHaveBeenCalledWith({ - source_id: "radiookapi", - env: "development", - page_range: undefined, - date_range: undefined, - category: undefined, - }); - }); + expect(count).toBe(5); + expect(handler).toHaveBeenCalledWith({ + source_id: "radiookapi", + env: "development", + page_range: undefined, + date_range: undefined, + category: undefined, + }); + }); }); diff --git a/basango/apps/crawler/src/__tests__/tasks.ts b/basango/apps/crawler/src/__tests__/tasks.ts index e90b719..407ea5f 100644 --- a/basango/apps/crawler/src/__tests__/tasks.ts +++ b/basango/apps/crawler/src/__tests__/tasks.ts @@ -1 +1 @@ -export * from "../services/crawler/async/tasks"; +export * from "@basango/crawler/services/async/tasks"; diff --git a/basango/apps/crawler/src/config.ts b/basango/apps/crawler/src/config.ts index 0409949..74582b0 100644 --- a/basango/apps/crawler/src/config.ts +++ b/basango/apps/crawler/src/config.ts @@ -1,186 +1,182 @@ -import fs from "node:fs"; -import path from "node:path"; +import * as fs from "node:fs"; +import * as path from "node:path"; -import { logger } from "@basango/logger"; +import {logger} from "@basango/logger"; import { - PipelineConfig, - PipelineConfigSchema, - mergePipelineConfig, - resolveConfigPath, - resolveProjectPaths, + mergePipelineConfig, + PipelineConfig, + PipelineConfigSchema, + resolveConfigPath, + resolveProjectPaths, } from "./schema"; -import { ensureDirectories } from "./utils"; +import {ensureDirectories} from "./utils"; export interface LoadConfigOptions { - configPath?: string; - env?: string; + configPath?: string; + env?: string; } const DEFAULT_CONFIG_FILES = [ - path.join(process.cwd(), "config", "pipeline.json"), - path.join(process.cwd(), "pipeline.json"), + path.join(process.cwd(), "config", "pipeline.json"), + path.join(process.cwd(), "pipeline.json"), ]; const readJsonFile = (filePath: string): unknown => { - const contents = fs.readFileSync(filePath, "utf-8"); - return contents.trim() === "" ? {} : JSON.parse(contents); + const contents = fs.readFileSync(filePath, "utf-8"); + return contents.trim() === "" ? {} : JSON.parse(contents); }; -export const locateConfigFile = (explicit?: string): string => { - if (explicit && fs.existsSync(explicit)) { - return explicit; - } +const locateConfigFile = (explicit?: string): string => { + if (explicit && fs.existsSync(explicit)) { + return explicit; + } - for (const candidate of DEFAULT_CONFIG_FILES) { - if (fs.existsSync(candidate)) { - return candidate; - } - } + for (const candidate of DEFAULT_CONFIG_FILES) { + if (fs.existsSync(candidate)) { + return candidate; + } + } - return DEFAULT_CONFIG_FILES[0]; + return DEFAULT_CONFIG_FILES[0]; }; const readPipelineConfig = (configPath: string): PipelineConfig => { - if (!fs.existsSync(configPath)) { - return PipelineConfigSchema.parse({ - paths: resolveProjectPaths(path.resolve(".")), - }); - } + if (!fs.existsSync(configPath)) { + return PipelineConfigSchema.parse({ + paths: resolveProjectPaths(path.resolve(".")), + }); + } - const raw = readJsonFile(configPath); - return PipelineConfigSchema.parse(raw); + const raw = readJsonFile(configPath); + return PipelineConfigSchema.parse(raw); }; const applyEnvironmentOverride = ( - baseConfig: PipelineConfig, - basePath: string, - env?: string, + baseConfig: PipelineConfig, + basePath: string, + env?: string, ): PipelineConfig => { - if (!env || env === "development") { - return baseConfig; - } + if (!env || env === "development") { + return baseConfig; + } - const overridePath = resolveConfigPath(basePath, env); - if (!fs.existsSync(overridePath)) { - return baseConfig; - } + const overridePath = resolveConfigPath(basePath, env); + if (!fs.existsSync(overridePath)) { + return baseConfig; + } - const overrides = PipelineConfigSchema.parse(readJsonFile(overridePath)); - return mergePipelineConfig(baseConfig, overrides); + const overrides = PipelineConfigSchema.parse(readJsonFile(overridePath)); + return mergePipelineConfig(baseConfig, overrides); }; export const loadConfig = (options: LoadConfigOptions = {}): PipelineConfig => { - const basePath = locateConfigFile(options.configPath); - const config = applyEnvironmentOverride( - readPipelineConfig(basePath), - basePath, - options.env, - ); + const basePath = locateConfigFile(options.configPath); + const config = applyEnvironmentOverride( + readPipelineConfig(basePath), + basePath, + options.env, + ); - ensureDirectories(config.paths); - return config; + ensureDirectories(config.paths); + return config; }; export const dumpConfig = ( - config: PipelineConfig, - targetPath?: string, + config: PipelineConfig, + targetPath?: string, ): void => { - const destination = targetPath ?? locateConfigFile(); - const normalized = PipelineConfigSchema.parse(config); - fs.mkdirSync(path.dirname(destination), { recursive: true }); - fs.writeFileSync(destination, JSON.stringify(normalized, null, 2)); + const destination = targetPath ?? locateConfigFile(); + const normalized = PipelineConfigSchema.parse(config); + fs.mkdirSync(path.dirname(destination), {recursive: true}); + fs.writeFileSync(destination, JSON.stringify(normalized, null, 2)); }; export interface PipelineConfigManagerOptions { - configPath?: string; - env?: string; - autoLoad?: boolean; + configPath?: string; + env?: string; + autoLoad?: boolean; } export class PipelineConfigManager { - private readonly explicitPath?: string; + private readonly explicitPath?: string; - private readonly defaultEnv: string; + private readonly defaultEnv: string; - private cache?: PipelineConfig; + private cache?: PipelineConfig; - constructor(options: PipelineConfigManagerOptions = {}) { - this.explicitPath = options.configPath; - this.defaultEnv = options.env ?? "development"; + constructor(options: PipelineConfigManagerOptions = {}) { + this.explicitPath = options.configPath; + this.defaultEnv = options.env ?? "development"; - if (options.autoLoad !== false) { - this.cache = loadConfig({ - configPath: this.explicitPath, - env: this.defaultEnv, - }); - } - } + if (options.autoLoad !== false) { + this.cache = loadConfig({ + configPath: this.explicitPath, + env: this.defaultEnv, + }); + } + } - get(env?: string): PipelineConfig { - const resolvedEnv = env ?? this.defaultEnv; + get(env?: string): PipelineConfig { + const resolvedEnv = env ?? this.defaultEnv; - if (resolvedEnv !== this.defaultEnv) { - return loadConfig({ - configPath: this.explicitPath, - env: resolvedEnv, - }); - } + if (resolvedEnv !== this.defaultEnv) { + return loadConfig({ + configPath: this.explicitPath, + env: resolvedEnv, + }); + } - if (!this.cache) { - this.cache = loadConfig({ - configPath: this.explicitPath, - env: resolvedEnv, - }); - } + if (!this.cache) { + this.cache = loadConfig({ + configPath: this.explicitPath, + env: resolvedEnv, + }); + } - return this.cache; - } + return this.cache; + } - reload(env?: string): PipelineConfig { - const resolvedEnv = env ?? this.defaultEnv; - const config = loadConfig({ - configPath: this.explicitPath, - env: resolvedEnv, - }); + reload(env?: string): PipelineConfig { + const resolvedEnv = env ?? this.defaultEnv; + const config = loadConfig({ + configPath: this.explicitPath, + env: resolvedEnv, + }); - if (resolvedEnv === this.defaultEnv) { - this.cache = config; - } + if (resolvedEnv === this.defaultEnv) { + this.cache = config; + } - return config; - } + return config; + } - ensureDirectories(config?: PipelineConfig): PipelineConfig { - const pipeline = config ?? this.get(); - ensureDirectories(pipeline.paths); - return pipeline; - } + ensureDirectories(config?: PipelineConfig): PipelineConfig { + const pipeline = config ?? this.get(); + ensureDirectories(pipeline.paths); + return pipeline; + } - setupLogging(config?: PipelineConfig): void { - const pipeline = config ?? this.get(); - this.ensureDirectories(pipeline); + setupLogging(config?: PipelineConfig): void { + const pipeline = config ?? this.get(); + this.ensureDirectories(pipeline); - const level = pipeline.logging.level.toLowerCase(); - process.env.LOG_LEVEL = level; - const normalizedLevel = level as typeof logger.level; - logger.level = normalizedLevel; + const level = pipeline.logging.level.toLowerCase(); + process.env.LOG_LEVEL = level; + logger.level = level as typeof logger.level; - if (pipeline.logging.file_logging) { - const logDir = pipeline.paths.logs; - const destination = path.join( - logDir, - pipeline.logging.log_file, - ); - fs.mkdirSync(path.dirname(destination), { recursive: true }); - if (!fs.existsSync(destination)) { - fs.writeFileSync(destination, ""); - } - } - } + if (pipeline.logging.file_logging) { + const logDir = pipeline.paths.logs; + const destination = path.join(logDir, pipeline.logging.log_file); + fs.mkdirSync(path.dirname(destination), {recursive: true}); + if (!fs.existsSync(destination)) { + fs.writeFileSync(destination, ""); + } + } + } - resolveConfigPath(env?: string): string { - const base = locateConfigFile(this.explicitPath); - return resolveConfigPath(base, env ?? this.defaultEnv); - } + resolveConfigPath(env?: string): string { + const base = locateConfigFile(this.explicitPath); + return resolveConfigPath(base, env ?? this.defaultEnv); + } } diff --git a/basango/apps/crawler/src/schema.ts b/basango/apps/crawler/src/schema.ts index fbef893..5bc08ef 100644 --- a/basango/apps/crawler/src/schema.ts +++ b/basango/apps/crawler/src/schema.ts @@ -1,7 +1,7 @@ -import path from "node:path"; +import * as path from "node:path"; -import { getUnixTime, parse, isMatch, format as formatDate } from "date-fns"; -import { z } from "zod"; +import {format as formatDate, getUnixTime, isMatch, parse} from "date-fns"; +import {z} from "zod"; export const UpdateDirectionSchema = z.enum(["forward", "backward"]); export type UpdateDirection = z.infer; @@ -10,47 +10,47 @@ export const SourceKindSchema = z.enum(["wordpress", "html"]); export type SourceKind = z.infer; export const SourceDateSchema = z.object({ - format: z.string().default("yyyy-LL-dd HH:mm"), - pattern: z.string().nullable().optional(), - replacement: z.string().nullable().optional(), + format: z.string().default("yyyy-LL-dd HH:mm"), + pattern: z.string().nullable().optional(), + replacement: z.string().nullable().optional(), }); export type SourceDate = z.infer; export const SourceSelectorsSchema = z.object({ - articles: z.string().optional().nullable(), - article_title: z.string().optional().nullable(), - article_link: z.string().optional().nullable(), - article_body: z.string().optional().nullable(), - article_date: z.string().optional().nullable(), - article_categories: z.string().optional().nullable(), - pagination: z.string().default("ul.pagination > li a"), + articles: z.string().optional().nullable(), + article_title: z.string().optional().nullable(), + article_link: z.string().optional().nullable(), + article_body: z.string().optional().nullable(), + article_date: z.string().optional().nullable(), + article_categories: z.string().optional().nullable(), + pagination: z.string().default("ul.pagination > li a"), }); export type SourceSelectors = z.infer; const BaseSourceSchema = z.object({ - source_id: z.string(), - source_url: z.string().url(), - source_date: SourceDateSchema.default(SourceDateSchema.parse({})), - source_kind: SourceKindSchema, - categories: z.array(z.string()).default([]), - supports_categories: z.boolean().default(false), - requires_details: z.boolean().default(false), - requires_rate_limit: z.boolean().default(false), + source_id: z.string(), + source_url: z.url(), + source_date: SourceDateSchema.default(SourceDateSchema.parse({})), + source_kind: SourceKindSchema, + categories: z.array(z.string()).default([]), + supports_categories: z.boolean().default(false), + requires_details: z.boolean().default(false), + requires_rate_limit: z.boolean().default(false), }); export const HtmlSourceConfigSchema = BaseSourceSchema.extend({ - source_kind: z.literal("html"), - source_selectors: SourceSelectorsSchema.default( - SourceSelectorsSchema.parse({}), - ), - pagination_template: z.string(), + source_kind: z.literal("html"), + source_selectors: SourceSelectorsSchema.default( + SourceSelectorsSchema.parse({}), + ), + pagination_template: z.string(), }); export const WordPressSourceConfigSchema = BaseSourceSchema.extend({ - source_kind: z.literal("wordpress"), - source_date: SourceDateSchema.default( - SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" }), - ), + source_kind: z.literal("wordpress"), + source_date: SourceDateSchema.default( + SourceDateSchema.parse({format: "yyyy-LL-dd'T'HH:mm:ss"}), + ), }); export type HtmlSourceConfig = z.infer; @@ -58,279 +58,274 @@ export type WordPressSourceConfig = z.infer; export type AnySourceConfig = HtmlSourceConfig | WordPressSourceConfig; export const DateRangeSchema = z - .object({ - start: z.number().int(), - end: z.number().int(), - }) - .superRefine((value, ctx) => { - if (value.start === 0 || value.end === 0) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: "Timestamp cannot be zero", - }); - } - if (value.end < value.start) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: "End timestamp must be greater than or equal to start", - }); - } - }); + .object({ + start: z.number().int(), + end: z.number().int(), + }) + .superRefine((value, ctx) => { + if (value.start === 0 || value.end === 0) { + ctx.addIssue({ + code: "custom", + message: "Timestamp cannot be zero", + }); + } + if (value.end < value.start) { + ctx.addIssue({ + code: "custom", + message: "End timestamp must be greater than or equal to start", + }); + } + }); export type DateRange = z.infer; export const PageRangeSchema = z - .object({ - start: z.number().int().min(0), - end: z.number().int().min(0), - }) - .superRefine((value, ctx) => { - if (value.end < value.start) { - ctx.addIssue({ - code: z.ZodIssueCode.custom, - message: "End page must be greater than or equal to start page", - }); - } - }); + .object({ + start: z.number().int().min(0), + end: z.number().int().min(0), + }) + .superRefine((value, ctx) => { + if (value.end < value.start) { + ctx.addIssue({ + code: "custom", + message: "End page must be greater than or equal to start page", + }); + } + }); export type PageRange = z.infer; export const PageRangeSpecSchema = z - .string() - .regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end") - .transform((spec) => { - const [startText, endText] = spec.split(":"); - return { - start: Number.parseInt(startText, 10), - end: Number.parseInt(endText, 10), - }; - }); + .string() + .regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end") + .transform((spec) => { + const [startText, endText] = spec.split(":"); + return { + start: Number.parseInt(startText, 10), + end: Number.parseInt(endText, 10), + }; + }); const defaultDateFormat = "yyyy-LL-dd"; export const DateRangeSpecSchema = z - .string() - .regex(/.+:.+/, "Expected start:end format") - .transform((spec) => { - const [startRaw, endRaw] = spec.split(":"); - return { startRaw, endRaw }; - }); + .string() + .regex(/.+:.+/, "Expected start:end format") + .transform((spec) => { + const [startRaw, endRaw] = spec.split(":"); + return {startRaw, endRaw}; + }); const parseDate = (value: string, format: string): Date => { - if (!isMatch(value, format)) { - throw new Error(`Invalid date '${value}' for format '${format}'`); - } - const parsed = parse(value, format, new Date()); - if (Number.isNaN(parsed.getTime())) { - throw new Error(`Invalid date '${value}' for format '${format}'`); - } - return parsed; + if (!isMatch(value, format)) { + throw new Error(`Invalid date '${value}' for format '${format}'`); + } + const parsed = parse(value, format, new Date()); + if (Number.isNaN(parsed.getTime())) { + throw new Error(`Invalid date '${value}' for format '${format}'`); + } + return parsed; }; export interface CreateDateRangeOptions { - format?: string; - separator?: string; + format?: string; + separator?: string; } export const createDateRange = ( - spec: string, - options: CreateDateRangeOptions = {}, + spec: string, + options: CreateDateRangeOptions = {}, ): DateRange => { - const { format = defaultDateFormat, separator = ":" } = options; - if (!separator) { - throw new Error("Separator cannot be empty"); - } + const {format = defaultDateFormat, separator = ":"} = options; + if (!separator) { + throw new Error("Separator cannot be empty"); + } - const normalized = spec.replace(separator, ":"); - const parsedSpec = DateRangeSpecSchema.parse(normalized); + const normalized = spec.replace(separator, ":"); + const parsedSpec = DateRangeSpecSchema.parse(normalized); - const startDate = parseDate(parsedSpec.startRaw, format); - const endDate = parseDate(parsedSpec.endRaw, format); + const startDate = parseDate(parsedSpec.startRaw, format); + const endDate = parseDate(parsedSpec.endRaw, format); - const range = { - start: getUnixTime(startDate), - end: getUnixTime(endDate), - }; + const range = { + start: getUnixTime(startDate), + end: getUnixTime(endDate), + }; - return DateRangeSchema.parse(range); + return DateRangeSchema.parse(range); }; export const formatDateRange = ( - range: DateRange, - fmt = defaultDateFormat, + range: DateRange, + fmt = defaultDateFormat, ): string => { - const start = formatDate(new Date(range.start * 1000), fmt); - const end = formatDate(new Date(range.end * 1000), fmt); - return `${start}:${end}`; + const start = formatDate(new Date(range.start * 1000), fmt); + const end = formatDate(new Date(range.end * 1000), fmt); + return `${start}:${end}`; }; export const isTimestampInRange = ( - range: DateRange, - timestamp: number, + range: DateRange, + timestamp: number, ): boolean => { - return range.start <= timestamp && timestamp <= range.end; + return range.start <= timestamp && timestamp <= range.end; }; export const ProjectPathsSchema = z.object({ - root: z.string(), - data: z.string(), - logs: z.string(), - configs: z.string(), + root: z.string(), + data: z.string(), + logs: z.string(), + configs: z.string(), }); export type ProjectPaths = z.infer; export const resolveProjectPaths = (rootDir: string): ProjectPaths => { - return ProjectPathsSchema.parse({ - root: rootDir, - data: path.join(rootDir, "data", "dataset"), - logs: path.join(rootDir, "data", "logs"), - configs: path.join(rootDir, "config"), - }); + return ProjectPathsSchema.parse({ + root: rootDir, + data: path.join(rootDir, "data", "dataset"), + logs: path.join(rootDir, "data", "logs"), + configs: path.join(rootDir, "config"), + }); }; export const LoggingConfigSchema = z.object({ - level: z.string().default("INFO"), - format: z - .string() - .default("%(asctime)s - %(name)s - %(levelname)s - %(message)s"), - console_logging: z.boolean().default(true), - file_logging: z.boolean().default(false), - log_file: z.string().default("crawler.log"), - max_log_size: z - .number() - .int() - .positive() - .default(10 * 1024 * 1024), - backup_count: z.number().int().nonnegative().default(5), + level: z.string().default("INFO"), + format: z + .string() + .default("%(asctime)s - %(name)s - %(levelname)s - %(message)s"), + console_logging: z.boolean().default(true), + file_logging: z.boolean().default(false), + log_file: z.string().default("crawler.log"), + max_log_size: z + .number() + .int() + .positive() + .default(10 * 1024 * 1024), + backup_count: z.number().int().nonnegative().default(5), }); export type LoggingConfig = z.infer; export const ClientConfigSchema = z.object({ - timeout: z.number().positive().default(20), - user_agent: z - .string() - .default("Basango/0.1 (+https://github.com/bernard-ng/basango)"), - follow_redirects: z.boolean().default(true), - verify_ssl: z.boolean().default(true), - rotate: z.boolean().default(true), - max_retries: z.number().int().nonnegative().default(3), - backoff_initial: z.number().nonnegative().default(1), - backoff_multiplier: z.number().positive().default(2), - backoff_max: z.number().nonnegative().default(30), - respect_retry_after: z.boolean().default(true), + timeout: z.number().positive().default(20), + user_agent: z + .string() + .default("Basango/0.1 (+https://github.com/bernard-ng/basango)"), + follow_redirects: z.boolean().default(true), + verify_ssl: z.boolean().default(true), + rotate: z.boolean().default(true), + max_retries: z.number().int().nonnegative().default(3), + backoff_initial: z.number().nonnegative().default(1), + backoff_multiplier: z.number().positive().default(2), + backoff_max: z.number().nonnegative().default(30), + respect_retry_after: z.boolean().default(true), }); export const CrawlerConfigSchema = z.object({ - source: z - .union([HtmlSourceConfigSchema, WordPressSourceConfigSchema]) - .optional(), - page_range: PageRangeSchema.optional(), - date_range: DateRangeSchema.optional(), - category: z.string().optional(), - notify: z.boolean().default(false), - is_update: z.boolean().default(false), - use_multi_threading: z.boolean().default(false), - max_workers: z.number().int().positive().default(5), - direction: UpdateDirectionSchema.default("forward"), + source: z + .union([HtmlSourceConfigSchema, WordPressSourceConfigSchema]) + .optional(), + page_range: PageRangeSchema.optional(), + date_range: DateRangeSchema.optional(), + category: z.string().optional(), + notify: z.boolean().default(false), + is_update: z.boolean().default(false), + use_multi_threading: z.boolean().default(false), + max_workers: z.number().int().positive().default(5), + direction: UpdateDirectionSchema.default("forward"), }); export type ClientConfig = z.infer; export type CrawlerConfig = z.infer & { - source?: AnySourceConfig; + source?: AnySourceConfig; }; export const FetchConfigSchema = z.object({ - client: ClientConfigSchema.default(ClientConfigSchema.parse({})), - crawler: CrawlerConfigSchema.default(CrawlerConfigSchema.parse({})), + client: ClientConfigSchema.default(ClientConfigSchema.parse({})), + crawler: CrawlerConfigSchema.default(CrawlerConfigSchema.parse({})), }); export type FetchConfig = z.infer; const SourcesConfigSchema = z.object({ - html: z.array(HtmlSourceConfigSchema).default([]), - wordpress: z.array(WordPressSourceConfigSchema).default([]), + html: z.array(HtmlSourceConfigSchema).default([]), + wordpress: z.array(WordPressSourceConfigSchema).default([]), }); export type SourcesConfig = z.infer & { - find: (sourceId: string) => AnySourceConfig | undefined; + find: (sourceId: string) => AnySourceConfig | undefined; }; export const createSourcesConfig = (input: unknown): SourcesConfig => { - const parsed = SourcesConfigSchema.parse(input); - const resolver = (sourceId: string) => - [...parsed.html, ...parsed.wordpress].find( - (source) => source.source_id === sourceId, - ); - return Object.assign({ find: resolver }, parsed); + const parsed = SourcesConfigSchema.parse(input); + const resolver = (sourceId: string) => + [...parsed.html, ...parsed.wordpress].find( + (source) => source.source_id === sourceId, + ); + return Object.assign({find: resolver}, parsed); }; export const PipelineConfigSchema = z.object({ - paths: ProjectPathsSchema.default(resolveProjectPaths(process.cwd())), - logging: LoggingConfigSchema.default(LoggingConfigSchema.parse({})), - fetch: FetchConfigSchema.default(FetchConfigSchema.parse({})), - sources: z - .union([SourcesConfigSchema, z.undefined()]) - .transform((value) => createSourcesConfig(value ?? {})), + paths: ProjectPathsSchema.default(resolveProjectPaths(process.cwd())), + logging: LoggingConfigSchema.default(LoggingConfigSchema.parse({})), + fetch: FetchConfigSchema.default(FetchConfigSchema.parse({})), + sources: z + .union([SourcesConfigSchema, z.undefined()]) + .transform((value) => createSourcesConfig(value ?? {})), }); - -export type PipelineConfig = z.infer & { - sources: SourcesConfig; -}; +export type PipelineConfig = z.infer export const mergePipelineConfig = ( - base: PipelineConfig, - overrides: Partial, + base: PipelineConfig, + overrides: Partial, ): PipelineConfig => { - const paths = overrides.paths ?? base.paths; - const logging = { ...base.logging, ...(overrides.logging ?? {}) }; - const fetch = { - client: { ...base.fetch.client, ...(overrides.fetch?.client ?? {}) }, - crawler: { ...base.fetch.crawler, ...(overrides.fetch?.crawler ?? {}) }, - }; + const paths = overrides.paths ?? base.paths; + const logging = {...base.logging, ...(overrides.logging ?? {})}; + const fetch = { + client: {...base.fetch.client, ...(overrides.fetch?.client ?? {})}, + crawler: {...base.fetch.crawler, ...(overrides.fetch?.crawler ?? {})}, + }; - const sources = createSourcesConfig({ - html: overrides.sources?.html ?? base.sources.html, - wordpress: overrides.sources?.wordpress ?? base.sources.wordpress, - }); + const sources = createSourcesConfig({ + html: overrides.sources?.html ?? base.sources.html, + wordpress: overrides.sources?.wordpress ?? base.sources.wordpress, + }); - return { - paths, - logging, - fetch, - sources, - }; + return { + paths, + logging, + fetch, + sources, + }; }; export const resolveConfigPath = (basePath: string, env?: string): string => { - if (!env || env === "development") { - return basePath; - } + if (!env || env === "development") { + return basePath; + } - const ext = path.extname(basePath); - const withoutExt = basePath.slice(0, basePath.length - ext.length); - return `${withoutExt}.${env}${ext}`; + const ext = path.extname(basePath); + const withoutExt = basePath.slice(0, basePath.length - ext.length); + return `${withoutExt}.${env}${ext}`; }; export const schemaToJSON = (schema: T): unknown => { - const candidate = schema as unknown as { toJSON?: () => unknown }; - if (typeof candidate.toJSON === "function") { - return candidate.toJSON(); - } + const toJSONSchema = (z as any).toJSONSchema as + | ((s: z.ZodTypeAny, opts?: Record) => unknown) + | undefined; - const typeName = (schema as { _def?: { typeName?: z.ZodFirstPartyTypeKind } })._def - ?.typeName; + if (typeof toJSONSchema === "function") { + try { + // target can be "draft-2020-12" | "draft-7" | "draft-4" | "openapi-3.0" + return toJSONSchema(schema, {target: "draft-2020-12", unrepresentable: "any"}); + } catch { + // fall through to minimal mapping + } + } - switch (typeName) { - case z.ZodFirstPartyTypeKind.ZodObject: - return { type: "object" }; - case z.ZodFirstPartyTypeKind.ZodArray: - return { type: "array" }; - case z.ZodFirstPartyTypeKind.ZodString: - return { type: "string" }; - case z.ZodFirstPartyTypeKind.ZodNumber: - return { type: "number" }; - case z.ZodFirstPartyTypeKind.ZodBoolean: - return { type: "boolean" }; - default: - return { type: "unknown" }; - } -}; + if (schema instanceof z.ZodObject) return {type: "object"}; + if (schema instanceof z.ZodArray) return {type: "array"}; + if (schema instanceof z.ZodString) return {type: "string"}; + if (schema instanceof z.ZodNumber) return {type: "number"}; + if (schema instanceof z.ZodBoolean) return {type: "boolean"}; + + return {type: "unknown"}; +}; \ No newline at end of file diff --git a/basango/apps/crawler/src/scripts/queue.ts b/basango/apps/crawler/src/scripts/queue.ts index 14e9767..7d163ef 100644 --- a/basango/apps/crawler/src/scripts/queue.ts +++ b/basango/apps/crawler/src/scripts/queue.ts @@ -1,88 +1,88 @@ -import { parseArgs } from "node:util"; +import {parseArgs} from "node:util"; -import { logger } from "@basango/logger"; +import {logger} from "@basango/logger"; -import { PipelineConfigManager } from "@crawler/config"; -import { scheduleAsyncCrawl } from "@crawler/services/crawler"; -import { createQueueSettings } from "@crawler/services/crawler/async/queue"; +import {PipelineConfigManager} from "@crawler/config"; +import {createQueueSettings} from "@crawler/services/async/queue"; +import {scheduleAsyncCrawl} from "@crawler/services/async/tasks"; interface QueueCliOptions { - "source-id"?: string; - env: string; - "page-range"?: string; - "date-range"?: string; - category?: string; - "redis-url"?: string; - help?: boolean; + "source-id"?: string; + env: string; + "page-range"?: string; + "date-range"?: string; + category?: string; + "redis-url"?: string; + help?: boolean; } const usage = `Usage: bun run src/scripts/queue.ts -- --source-id [options]\n\nOptions:\n --env Environment to load (default: development)\n --page-range Optional page range filter (e.g. 1:5)\n --date-range Optional date range filter (e.g. 2024-01-01:2024-01-31)\n --category Optional category to crawl\n --redis-url Override Redis connection URL\n -h, --help Show this message`; const parseCliArgs = (): QueueCliOptions => { - const { values } = parseArgs({ - options: { - "source-id": { type: "string" }, - env: { type: "string", default: "development" }, - "page-range": { type: "string" }, - "date-range": { type: "string" }, - category: { type: "string" }, - "redis-url": { type: "string" }, - help: { type: "boolean", short: "h" }, - }, - }); + const {values} = parseArgs({ + options: { + "source-id": {type: "string"}, + env: {type: "string", default: "development"}, + "page-range": {type: "string"}, + "date-range": {type: "string"}, + category: {type: "string"}, + "redis-url": {type: "string"}, + help: {type: "boolean", short: "h"}, + }, + }); - return values as QueueCliOptions; + return values as QueueCliOptions; }; const main = async (): Promise => { - const options = parseCliArgs(); + const options = parseCliArgs(); - if (options.help || !options["source-id"]) { - console.log(usage); - if (!options["source-id"]) { - process.exitCode = 1; - } - return; - } + if (options.help || !options["source-id"]) { + console.log(usage); + if (!options["source-id"]) { + process.exitCode = 1; + } + return; + } - const env = options.env ?? "development"; - const manager = new PipelineConfigManager({ env }); - const config = manager.ensureDirectories(); - manager.setupLogging(config); + const env = options.env ?? "development"; + const manager = new PipelineConfigManager({env}); + const config = manager.ensureDirectories(); + manager.setupLogging(config); - const settings = options["redis-url"] - ? createQueueSettings({ redis_url: options["redis-url"] }) - : undefined; + const settings = options["redis-url"] + ? createQueueSettings({redis_url: options["redis-url"]}) + : undefined; - try { - const jobId = await scheduleAsyncCrawl({ - sourceId: options["source-id"], - env, - pageRange: options["page-range"] ?? null, - dateRange: options["date-range"] ?? null, - category: options.category ?? null, - settings, - }); + try { + const jobId = await scheduleAsyncCrawl({ + sourceId: options["source-id"], + env, + pageRange: options["page-range"] ?? null, + dateRange: options["date-range"] ?? null, + category: options.category ?? null, + settings, + }); - logger.info( - { - jobId, - sourceId: options["source-id"], - env, - }, - "Scheduled asynchronous crawl job", - ); - console.log( - `Scheduled async crawl job ${jobId} for source '${options["source-id"]}' (env=${env})`, - ); - } catch (error) { - logger.error( - error instanceof Error ? error : { error }, - "Failed to schedule crawl job", - ); - console.error(`Failed to schedule crawl job: ${(error as Error).message}`); - process.exitCode = 1; - } + logger.info( + { + jobId, + sourceId: options["source-id"], + env, + }, + "Scheduled asynchronous crawl job", + ); + console.log( + `Scheduled async crawl job ${jobId} for source '${options["source-id"]}' (env=${env})`, + ); + } catch (error) { + logger.error( + error instanceof Error ? error : {error}, + "Failed to schedule crawl job", + ); + console.error(`Failed to schedule crawl job: ${(error as Error).message}`); + process.exitCode = 1; + } }; void main(); diff --git a/basango/apps/crawler/src/scripts/worker.ts b/basango/apps/crawler/src/scripts/worker.ts index b038021..6d5c35b 100644 --- a/basango/apps/crawler/src/scripts/worker.ts +++ b/basango/apps/crawler/src/scripts/worker.ts @@ -1,112 +1,112 @@ -import { parseArgs } from "node:util"; +import {parseArgs} from "node:util"; -import { logger } from "@basango/logger"; +import {logger} from "@basango/logger"; -import { PipelineConfigManager } from "@crawler/config"; -import { createQueueManager, createQueueSettings } from "@crawler/services/crawler/async/queue"; -import { startWorker } from "@crawler/services/crawler/async/worker"; +import {PipelineConfigManager} from "@crawler/config"; +import {createQueueManager, createQueueSettings,} from "@crawler/services/async/queue"; +import {startWorker} from "@crawler/services/async/worker"; interface WorkerCliOptions { - env: string; - queue?: string[]; - concurrency?: string; - "redis-url"?: string; - help?: boolean; + env: string; + queue?: string[]; + concurrency?: string; + "redis-url"?: string; + help?: boolean; } const usage = `Usage: bun run src/scripts/worker.ts [options]\n\nOptions:\n --env Environment to load (default: development)\n -q, --queue Queue name to listen on (repeatable)\n --concurrency Number of concurrent jobs per worker\n --redis-url Override Redis connection URL\n -h, --help Show this message`; const parseCliArgs = (): WorkerCliOptions => { - const { values } = parseArgs({ - options: { - env: { type: "string", default: "development" }, - queue: { type: "string", multiple: true, short: "q" }, - concurrency: { type: "string" }, - "redis-url": { type: "string" }, - help: { type: "boolean", short: "h" }, - }, - }); + const {values} = parseArgs({ + options: { + env: {type: "string", default: "development"}, + queue: {type: "string", multiple: true, short: "q"}, + concurrency: {type: "string"}, + "redis-url": {type: "string"}, + help: {type: "boolean", short: "h"}, + }, + }); - return values as WorkerCliOptions; + return values as WorkerCliOptions; }; const parseConcurrency = (value?: string): number | undefined => { - if (!value) { - return undefined; - } + if (!value) { + return undefined; + } - const parsed = Number.parseInt(value, 10); - if (Number.isNaN(parsed) || parsed <= 0) { - throw new Error(`Invalid concurrency value: ${value}`); - } + const parsed = Number.parseInt(value, 10); + if (Number.isNaN(parsed) || parsed <= 0) { + throw new Error(`Invalid concurrency value: ${value}`); + } - return parsed; + return parsed; }; const main = async (): Promise => { - const options = parseCliArgs(); + const options = parseCliArgs(); - if (options.help) { - console.log(usage); - return; - } + if (options.help) { + console.log(usage); + return; + } - const env = options.env ?? "development"; - const manager = new PipelineConfigManager({ env }); - const config = manager.ensureDirectories(); - manager.setupLogging(config); + const env = options.env ?? "development"; + const manager = new PipelineConfigManager({env}); + const config = manager.ensureDirectories(); + manager.setupLogging(config); - let concurrency: number | undefined; - try { - concurrency = parseConcurrency(options.concurrency); - } catch (error) { - logger.error( - error instanceof Error ? error : { error }, - "Invalid concurrency value provided", - ); - process.exitCode = 1; - return; - } - const settings = options["redis-url"] - ? createQueueSettings({ redis_url: options["redis-url"] }) - : undefined; - const queueManager = createQueueManager({ settings }); + let concurrency: number | undefined; + try { + concurrency = parseConcurrency(options.concurrency); + } catch (error) { + logger.error( + error instanceof Error ? error : {error}, + "Invalid concurrency value provided", + ); + process.exitCode = 1; + return; + } + const settings = options["redis-url"] + ? createQueueSettings({redis_url: options["redis-url"]}) + : undefined; + const queueManager = createQueueManager({settings}); - const queueNames = options.queue?.length - ? options.queue.map((name) => queueManager.queueName(name)) - : undefined; + const queueNames = options.queue?.length + ? options.queue.map((name) => queueManager.queueName(name)) + : undefined; - const handle = startWorker({ - queueManager, - queueNames, - concurrency, - }); + const handle = startWorker({ + queueManager, + queueNames, + concurrency, + }); - const shutdown = async (signal: NodeJS.Signals) => { - logger.info({ signal }, "Received shutdown signal, draining workers"); - try { - await handle.close(); - } finally { - await queueManager.close(); - process.exit(0); - } - }; + const shutdown = async (signal: NodeJS.Signals) => { + logger.info({signal}, "Received shutdown signal, draining workers"); + try { + await handle.close(); + } finally { + await queueManager.close(); + process.exit(0); + } + }; - process.once("SIGINT", (signal) => { - void shutdown(signal); - }); - process.once("SIGTERM", (signal) => { - void shutdown(signal); - }); + process.once("SIGINT", (signal) => { + void shutdown(signal); + }); + process.once("SIGTERM", (signal) => { + void shutdown(signal); + }); - logger.info( - { - env, - queueNames: queueNames ?? queueManager.iterQueueNames(), - concurrency: concurrency ?? "default", - }, - "Crawler workers started", - ); + logger.info( + { + env, + queueNames: queueNames ?? queueManager.iterQueueNames(), + concurrency: concurrency ?? "default", + }, + "Crawler workers started", + ); }; void main(); diff --git a/basango/apps/crawler/src/services/async/queue.ts b/basango/apps/crawler/src/services/async/queue.ts new file mode 100644 index 0000000..28414ac --- /dev/null +++ b/basango/apps/crawler/src/services/async/queue.ts @@ -0,0 +1,142 @@ +import { randomUUID } from "node:crypto"; + +import IORedis from "ioredis"; +import { JobsOptions, Queue, QueueOptions } from "bullmq"; +import { z } from "zod"; + +import { + ArticleTaskPayload, + ArticleTaskPayloadSchema, + ListingTaskPayload, + ListingTaskPayloadSchema, + ProcessedTaskPayload, + ProcessedTaskPayloadSchema, +} from "./schemas"; +import { parseRedisUrl } from "@crawler/utils"; + +const QueueSettingsSchema = z.object({ + redis_url: z + .string() + .default(process.env.BASANGO_REDIS_URL ?? "redis://localhost:6379/0"), + prefix: z.string().default(process.env.BASANGO_QUEUE_PREFIX ?? "crawler"), + default_timeout: z + .number() + .int() + .positive() + .default(Number(process.env.BASANGO_QUEUE_TIMEOUT ?? 600)), + result_ttl: z + .number() + .int() + .nonnegative() + .default(Number(process.env.BASANGO_QUEUE_RESULT_TTL ?? 3600)), + failure_ttl: z + .number() + .int() + .nonnegative() + .default(Number(process.env.BASANGO_QUEUE_FAILURE_TTL ?? 3600)), + listing_queue: z.string().default("listing"), + article_queue: z.string().default("articles"), + processed_queue: z.string().default("processed"), +}); + +export type QueueSettingsInput = z.input; +export type QueueSettings = z.output; + +export const createQueueSettings = ( + input?: QueueSettingsInput, +): QueueSettings => QueueSettingsSchema.parse(input ?? {}); + +export interface QueueBackend { + add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>; +} + +export type QueueFactory = ( + queueName: string, + settings: QueueSettings, + connection?: IORedis, +) => QueueBackend; + +const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => { + const redisConnection = + connection ?? + new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url)); + const options: QueueOptions = { + connection: redisConnection, + prefix: settings.prefix, + }; + + const queue = new Queue(queueName, options); + return { + add: async (name, data, opts) => { + const job = await queue.add(name, data, { + removeOnComplete: settings.result_ttl === 0 ? true : undefined, + removeOnFail: settings.failure_ttl === 0 ? true : undefined, + //timeout: settings.default_timeout * 1000, + ...opts, + }); + return { id: job.id ?? randomUUID() }; + }, + }; +}; + +export interface CreateQueueManagerOptions { + settings?: QueueSettings | QueueSettingsInput; + queueFactory?: QueueFactory; + connection?: IORedis; +} + +export interface QueueManager { + readonly settings: QueueSettings; + readonly connection: IORedis; + enqueueListing: (payload: ListingTaskPayload) => Promise<{ id: string }>; + enqueueArticle: (payload: ArticleTaskPayload) => Promise<{ id: string }>; + enqueueProcessed: (payload: ProcessedTaskPayload) => Promise<{ id: string }>; + iterQueueNames: () => string[]; + queueName: (suffix: string) => string; + close: () => Promise; +} + +export const createQueueManager = ( + options: CreateQueueManagerOptions = {}, +): QueueManager => { + const settings = createQueueSettings( + options.settings as QueueSettingsInput | undefined, + ); + + const connection = + options.connection ?? + new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url)); + const factory = options.queueFactory ?? defaultQueueFactory; + + const ensureQueue = (queueName: string) => + factory(queueName, settings, connection); + + return { + settings, + connection, + enqueueListing: (payload) => { + const data = ListingTaskPayloadSchema.parse(payload); + const queue = ensureQueue(settings.listing_queue); + return queue.add("collect_listing", data); + }, + enqueueArticle: (payload) => { + const data = ArticleTaskPayloadSchema.parse(payload); + const queue = ensureQueue(settings.article_queue); + return queue.add("collect_article", data); + }, + enqueueProcessed: (payload) => { + const data = ProcessedTaskPayloadSchema.parse(payload); + const queue = ensureQueue(settings.processed_queue); + return queue.add("forward_for_processing", data); + }, + iterQueueNames: () => [ + `${settings.prefix}:${settings.listing_queue}`, + `${settings.prefix}:${settings.article_queue}`, + `${settings.prefix}:${settings.processed_queue}`, + ], + queueName: (suffix: string) => `${settings.prefix}:${suffix}`, + close: async () => { + await connection.quit(); + }, + }; +}; diff --git a/basango/apps/crawler/src/services/async/schemas.ts b/basango/apps/crawler/src/services/async/schemas.ts new file mode 100644 index 0000000..7c0f16d --- /dev/null +++ b/basango/apps/crawler/src/services/async/schemas.ts @@ -0,0 +1,41 @@ +import { z } from "zod"; + +import { + AnySourceConfig, + DateRangeSchema, + PageRangeSchema, +} from "@crawler/schema"; + +export const ListingTaskPayloadSchema = z.object({ + source_id: z.string(), + env: z.string().default("development"), + page_range: z.string().optional().nullable(), + date_range: z.string().optional().nullable(), + category: z.string().optional().nullable(), +}); + +export type ListingTaskPayload = z.infer; + +export const ArticleTaskPayloadSchema = z.object({ + source_id: z.string(), + env: z.string().default("development"), + url: z.url(), + page: z.number().int().nonnegative().optional(), + page_range: PageRangeSchema.optional().nullable(), + date_range: DateRangeSchema.optional().nullable(), + category: z.string().optional().nullable(), +}); + +export type ArticleTaskPayload = z.infer; + +export const ProcessedTaskPayloadSchema = z.object({ + source_id: z.string(), + env: z.string().default("development"), + article: z.any(), +}); + +export type ProcessedTaskPayload = z.infer; + +export interface ListingContext { + source: AnySourceConfig; +} diff --git a/basango/apps/crawler/src/services/async/tasks.ts b/basango/apps/crawler/src/services/async/tasks.ts new file mode 100644 index 0000000..c4d7a95 --- /dev/null +++ b/basango/apps/crawler/src/services/async/tasks.ts @@ -0,0 +1,171 @@ +import { logger } from "@basango/logger"; + +import { + ArticleTaskPayload, + ArticleTaskPayloadSchema, + ListingTaskPayload, + ListingTaskPayloadSchema, + ProcessedTaskPayload, + ProcessedTaskPayloadSchema, +} from "./schemas"; +import { + createQueueManager, + QueueManager, + QueueSettings, + QueueSettingsInput, +} from "./queue"; + +export interface CrawlerTaskHandlers { + collectListing: (payload: ListingTaskPayload) => Promise | number; + collectArticle: (payload: ArticleTaskPayload) => Promise | unknown; + forwardForProcessing: ( + payload: ProcessedTaskPayload, + ) => Promise | unknown; +} + +const notImplemented = (name: keyof CrawlerTaskHandlers) => () => { + throw new Error(`Crawler task handler '${name}' is not implemented`); +}; + +let handlers: CrawlerTaskHandlers = { + collectListing: notImplemented("collectListing"), + collectArticle: notImplemented("collectArticle"), + forwardForProcessing: notImplemented("forwardForProcessing"), +}; + +export const registerCrawlerTaskHandlers = ( + overrides: Partial, +): void => { + handlers = { ...handlers, ...overrides }; +}; + +export interface ScheduleAsyncCrawlOptions { + sourceId: string; + env?: string; + pageRange?: string | null; + dateRange?: string | null; + category?: string | null; + settings?: QueueSettings | QueueSettingsInput; + queueManager?: QueueManager; +} + +export const scheduleAsyncCrawl = async ({ + sourceId, + env = "development", + pageRange, + dateRange, + category, + settings, + queueManager, +}: ScheduleAsyncCrawlOptions): Promise => { + const payload = ListingTaskPayloadSchema.parse({ + source_id: sourceId, + env, + page_range: pageRange ?? undefined, + date_range: dateRange ?? undefined, + category: category ?? undefined, + }); + + const manager = queueManager ?? createQueueManager({ settings }); + logger.debug( + { + sourceId, + env: payload.env, + pageRange: payload.page_range, + dateRange: payload.date_range, + category: payload.category, + }, + "Scheduling listing collection job", + ); + try { + const job = await manager.enqueueListing(payload); + logger.info( + { jobId: job.id, sourceId, env: payload.env }, + "Scheduled listing collection job", + ); + return job.id; + } finally { + if (!queueManager) { + await manager.close(); + } + } +}; + +export const collectListing = async (payload: unknown): Promise => { + const data = ListingTaskPayloadSchema.parse(payload); + logger.debug( + { + sourceId: data.source_id, + env: data.env, + pageRange: data.page_range, + dateRange: data.date_range, + category: data.category, + }, + "Collecting listing", + ); + + const result = await handlers.collectListing(data); + const count = typeof result === "number" ? result : 0; + + logger.info( + { + sourceId: data.source_id, + env: data.env, + queuedArticles: count, + }, + "Listing collection completed", + ); + + return count; +}; + +export const collectArticle = async (payload: unknown): Promise => { + const data = ArticleTaskPayloadSchema.parse(payload); + logger.debug( + { + sourceId: data.source_id, + env: data.env, + url: data.url, + page: data.page, + }, + "Collecting article", + ); + + const result = await handlers.collectArticle(data); + + logger.info( + { + sourceId: data.source_id, + env: data.env, + url: data.url, + }, + "Article collection completed", + ); + + return result; +}; + +export const forwardForProcessing = async ( + payload: unknown, +): Promise => { + const data = ProcessedTaskPayloadSchema.parse(payload); + logger.debug( + { + sourceId: data.source_id, + env: data.env, + }, + "Forwarding article for processing", + ); + + const result = await handlers.forwardForProcessing(data); + + logger.info( + { + sourceId: data.source_id, + env: data.env, + }, + "Article forwarded for processing", + ); + + return result; +}; diff --git a/basango/apps/crawler/src/services/async/worker.ts b/basango/apps/crawler/src/services/async/worker.ts new file mode 100644 index 0000000..4840ac7 --- /dev/null +++ b/basango/apps/crawler/src/services/async/worker.ts @@ -0,0 +1,87 @@ +import IORedis from "ioredis"; +import { Worker, QueueEvents } from "bullmq"; + +import { + createQueueManager, + QueueFactory, + QueueManager, + QueueSettings, + QueueSettingsInput, +} from "./queue"; +import { collectArticle, collectListing, forwardForProcessing } from "./tasks"; + +export interface WorkerOptions { + queueNames?: string[]; + settings?: QueueSettings | QueueSettingsInput; + connection?: IORedis; + queueFactory?: QueueFactory; + concurrency?: number; + onError?: (error: Error) => void; + queueManager?: QueueManager; +} + +export interface WorkerHandle { + readonly workers: Worker[]; + readonly events: QueueEvents[]; + close: () => Promise; +} + +export const startWorker = (options: WorkerOptions = {}): WorkerHandle => { + const manager = + options.queueManager ?? + createQueueManager({ + settings: options.settings, + connection: options.connection, + queueFactory: options.queueFactory, + }); + + const queueNames = options.queueNames ?? manager.iterQueueNames(); + const workers: Worker[] = []; + const events: QueueEvents[] = []; + + const connection = manager.connection; + + for (const queueName of queueNames) { + const worker = new Worker( + queueName, + async (job) => { + switch (job.name) { + case "collect_listing": + return collectListing(job.data); + case "collect_article": + return collectArticle(job.data); + case "forward_for_processing": + return forwardForProcessing(job.data); + default: + throw new Error(`Unknown job name: ${job.name}`); + } + }, + { + connection, + concurrency: options.concurrency ?? 5, + }, + ); + + if (options.onError) { + worker.on("failed", (_, err) => options.onError?.(err as Error)); + worker.on("error", (err) => options.onError?.(err as Error)); + } + + const queueEvents = new QueueEvents(queueName, { connection }); + + workers.push(worker); + events.push(queueEvents); + } + + return { + workers, + events, + close: async () => { + await Promise.all(workers.map((worker) => worker.close())); + await Promise.all(events.map((event) => event.close())); + if (!options.queueManager) { + await manager.close(); + } + }, + }; +}; diff --git a/basango/apps/crawler/src/services/crawler/async/queue.ts b/basango/apps/crawler/src/services/crawler/async/queue.ts deleted file mode 100644 index 262e414..0000000 --- a/basango/apps/crawler/src/services/crawler/async/queue.ts +++ /dev/null @@ -1,142 +0,0 @@ -import { randomUUID } from "node:crypto"; - -import IORedis from "ioredis"; -import { Queue, JobsOptions, QueueOptions } from "bullmq"; -import { z } from "zod"; - -import { - ListingTaskPayload, - ArticleTaskPayload, - ProcessedTaskPayload, - ListingTaskPayloadSchema, - ArticleTaskPayloadSchema, - ProcessedTaskPayloadSchema, -} from "./schemas"; -import { parseRedisUrl } from "../../../utils"; - -const QueueSettingsSchema = z.object({ - redis_url: z - .string() - .default(process.env.BASANGO_REDIS_URL ?? "redis://localhost:6379/0"), - prefix: z.string().default(process.env.BASANGO_QUEUE_PREFIX ?? "crawler"), - default_timeout: z - .number() - .int() - .positive() - .default(Number(process.env.BASANGO_QUEUE_TIMEOUT ?? 600)), - result_ttl: z - .number() - .int() - .nonnegative() - .default(Number(process.env.BASANGO_QUEUE_RESULT_TTL ?? 3600)), - failure_ttl: z - .number() - .int() - .nonnegative() - .default(Number(process.env.BASANGO_QUEUE_FAILURE_TTL ?? 3600)), - listing_queue: z.string().default("listing"), - article_queue: z.string().default("articles"), - processed_queue: z.string().default("processed"), -}); - -export type QueueSettingsInput = z.input; -export type QueueSettings = z.output; - -export const createQueueSettings = ( - input?: QueueSettingsInput, -): QueueSettings => QueueSettingsSchema.parse(input ?? {}); - -export interface QueueBackend { - add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>; -} - -export type QueueFactory = ( - queueName: string, - settings: QueueSettings, - connection?: IORedis, -) => QueueBackend; - -const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => { - const redisConnection = - connection ?? - new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url)); - const options: QueueOptions = { - connection: redisConnection, - prefix: settings.prefix, - }; - - const queue = new Queue(queueName, options); - return { - add: async (name, data, opts) => { - const job = await queue.add(name, data, { - removeOnComplete: settings.result_ttl === 0 ? true : undefined, - removeOnFail: settings.failure_ttl === 0 ? true : undefined, - timeout: settings.default_timeout * 1000, - ...opts, - }); - return { id: job.id ?? randomUUID() }; - }, - }; -}; - -export interface CreateQueueManagerOptions { - settings?: QueueSettings | QueueSettingsInput; - queueFactory?: QueueFactory; - connection?: IORedis; -} - -export interface QueueManager { - readonly settings: QueueSettings; - readonly connection: IORedis; - enqueueListing: (payload: ListingTaskPayload) => Promise<{ id: string }>; - enqueueArticle: (payload: ArticleTaskPayload) => Promise<{ id: string }>; - enqueueProcessed: (payload: ProcessedTaskPayload) => Promise<{ id: string }>; - iterQueueNames: () => string[]; - queueName: (suffix: string) => string; - close: () => Promise; -} - -export const createQueueManager = ( - options: CreateQueueManagerOptions = {}, -): QueueManager => { - const settings = createQueueSettings( - options.settings as QueueSettingsInput | undefined, - ); - - const connection = - options.connection ?? - new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url)); - const factory = options.queueFactory ?? defaultQueueFactory; - - const ensureQueue = (queueName: string) => - factory(queueName, settings, connection); - - return { - settings, - connection, - enqueueListing: (payload) => { - const data = ListingTaskPayloadSchema.parse(payload); - const queue = ensureQueue(settings.listing_queue); - return queue.add("collect_listing", data); - }, - enqueueArticle: (payload) => { - const data = ArticleTaskPayloadSchema.parse(payload); - const queue = ensureQueue(settings.article_queue); - return queue.add("collect_article", data); - }, - enqueueProcessed: (payload) => { - const data = ProcessedTaskPayloadSchema.parse(payload); - const queue = ensureQueue(settings.processed_queue); - return queue.add("forward_for_processing", data); - }, - iterQueueNames: () => [ - `${settings.prefix}:${settings.listing_queue}`, - `${settings.prefix}:${settings.article_queue}`, - `${settings.prefix}:${settings.processed_queue}`, - ], - queueName: (suffix: string) => `${settings.prefix}:${suffix}`, - close: async () => { - await connection.quit(); - }, - }; -}; diff --git a/basango/apps/crawler/src/services/crawler/async/schemas.ts b/basango/apps/crawler/src/services/crawler/async/schemas.ts deleted file mode 100644 index dac5a0a..0000000 --- a/basango/apps/crawler/src/services/crawler/async/schemas.ts +++ /dev/null @@ -1,41 +0,0 @@ -import { z } from "zod"; - -import { - AnySourceConfig, - DateRangeSchema, - PageRangeSchema, -} from "../../../schema"; - -export const ListingTaskPayloadSchema = z.object({ - source_id: z.string(), - env: z.string().default("development"), - page_range: z.string().optional().nullable(), - date_range: z.string().optional().nullable(), - category: z.string().optional().nullable(), -}); - -export type ListingTaskPayload = z.infer; - -export const ArticleTaskPayloadSchema = z.object({ - source_id: z.string(), - env: z.string().default("development"), - url: z.string().url(), - page: z.number().int().nonnegative().optional(), - page_range: PageRangeSchema.optional().nullable(), - date_range: DateRangeSchema.optional().nullable(), - category: z.string().optional().nullable(), -}); - -export type ArticleTaskPayload = z.infer; - -export const ProcessedTaskPayloadSchema = z.object({ - source_id: z.string(), - env: z.string().default("development"), - article: z.any(), -}); - -export type ProcessedTaskPayload = z.infer; - -export interface ListingContext { - source: AnySourceConfig; -} diff --git a/basango/apps/crawler/src/services/crawler/async/tasks.ts b/basango/apps/crawler/src/services/crawler/async/tasks.ts deleted file mode 100644 index cca05cd..0000000 --- a/basango/apps/crawler/src/services/crawler/async/tasks.ts +++ /dev/null @@ -1,171 +0,0 @@ -import { logger } from "@basango/logger"; - -import { - ListingTaskPayloadSchema, - ArticleTaskPayloadSchema, - ProcessedTaskPayloadSchema, - ListingTaskPayload, - ArticleTaskPayload, - ProcessedTaskPayload, -} from "./schemas"; -import { - createQueueManager, - QueueManager, - QueueSettings, - QueueSettingsInput, -} from "./queue"; - -export interface CrawlerTaskHandlers { - collectListing: (payload: ListingTaskPayload) => Promise | number; - collectArticle: (payload: ArticleTaskPayload) => Promise | unknown; - forwardForProcessing: ( - payload: ProcessedTaskPayload, - ) => Promise | unknown; -} - -const notImplemented = (name: keyof CrawlerTaskHandlers) => () => { - throw new Error(`Crawler task handler '${name}' is not implemented`); -}; - -let handlers: CrawlerTaskHandlers = { - collectListing: notImplemented("collectListing"), - collectArticle: notImplemented("collectArticle"), - forwardForProcessing: notImplemented("forwardForProcessing"), -}; - -export const registerCrawlerTaskHandlers = ( - overrides: Partial, -): void => { - handlers = { ...handlers, ...overrides }; -}; - -export interface ScheduleAsyncCrawlOptions { - sourceId: string; - env?: string; - pageRange?: string | null; - dateRange?: string | null; - category?: string | null; - settings?: QueueSettings | QueueSettingsInput; - queueManager?: QueueManager; -} - -export const scheduleAsyncCrawl = async ({ - sourceId, - env = "development", - pageRange, - dateRange, - category, - settings, - queueManager, -}: ScheduleAsyncCrawlOptions): Promise => { - const payload = ListingTaskPayloadSchema.parse({ - source_id: sourceId, - env, - page_range: pageRange ?? undefined, - date_range: dateRange ?? undefined, - category: category ?? undefined, - }); - - const manager = queueManager ?? createQueueManager({ settings }); - logger.debug( - { - sourceId, - env: payload.env, - pageRange: payload.page_range, - dateRange: payload.date_range, - category: payload.category, - }, - "Scheduling listing collection job", - ); - try { - const job = await manager.enqueueListing(payload); - logger.info( - { jobId: job.id, sourceId, env: payload.env }, - "Scheduled listing collection job", - ); - return job.id; - } finally { - if (!queueManager) { - await manager.close(); - } - } -}; - -export const collectListing = async (payload: unknown): Promise => { - const data = ListingTaskPayloadSchema.parse(payload); - logger.debug( - { - sourceId: data.source_id, - env: data.env, - pageRange: data.page_range, - dateRange: data.date_range, - category: data.category, - }, - "Collecting listing", - ); - - const result = await handlers.collectListing(data); - const count = typeof result === "number" ? result : 0; - - logger.info( - { - sourceId: data.source_id, - env: data.env, - queuedArticles: count, - }, - "Listing collection completed", - ); - - return count; -}; - -export const collectArticle = async (payload: unknown): Promise => { - const data = ArticleTaskPayloadSchema.parse(payload); - logger.debug( - { - sourceId: data.source_id, - env: data.env, - url: data.url, - page: data.page, - }, - "Collecting article", - ); - - const result = await handlers.collectArticle(data); - - logger.info( - { - sourceId: data.source_id, - env: data.env, - url: data.url, - }, - "Article collection completed", - ); - - return result; -}; - -export const forwardForProcessing = async ( - payload: unknown, -): Promise => { - const data = ProcessedTaskPayloadSchema.parse(payload); - logger.debug( - { - sourceId: data.source_id, - env: data.env, - }, - "Forwarding article for processing", - ); - - const result = await handlers.forwardForProcessing(data); - - logger.info( - { - sourceId: data.source_id, - env: data.env, - }, - "Article forwarded for processing", - ); - - return result; -}; diff --git a/basango/apps/crawler/src/services/crawler/async/worker.ts b/basango/apps/crawler/src/services/crawler/async/worker.ts deleted file mode 100644 index 4fe3ee0..0000000 --- a/basango/apps/crawler/src/services/crawler/async/worker.ts +++ /dev/null @@ -1,87 +0,0 @@ -import IORedis from "ioredis"; -import { Worker, QueueEvents } from "bullmq"; - -import { - createQueueManager, - QueueFactory, - QueueManager, - QueueSettings, - QueueSettingsInput, -} from "./queue"; -import { collectArticle, collectListing, forwardForProcessing } from "./tasks"; - -export interface WorkerOptions { - queueNames?: string[]; - settings?: QueueSettings | QueueSettingsInput; - connection?: IORedis; - queueFactory?: QueueFactory; - concurrency?: number; - onError?: (error: Error) => void; - queueManager?: QueueManager; -} - -export interface WorkerHandle { - readonly workers: Worker[]; - readonly events: QueueEvents[]; - close: () => Promise; -} - -export const startWorker = (options: WorkerOptions = {}): WorkerHandle => { - const manager = - options.queueManager ?? - createQueueManager({ - settings: options.settings, - connection: options.connection, - queueFactory: options.queueFactory, - }); - - const queueNames = options.queueNames ?? manager.iterQueueNames(); - const workers: Worker[] = []; - const events: QueueEvents[] = []; - - const connection = manager.connection; - - for (const queueName of queueNames) { - const worker = new Worker( - queueName, - async (job) => { - switch (job.name) { - case "collect_listing": - return collectListing(job.data); - case "collect_article": - return collectArticle(job.data); - case "forward_for_processing": - return forwardForProcessing(job.data); - default: - throw new Error(`Unknown job name: ${job.name}`); - } - }, - { - connection, - concurrency: options.concurrency ?? 5, - }, - ); - - if (options.onError) { - worker.on("failed", (_, err) => options.onError?.(err as Error)); - worker.on("error", (err) => options.onError?.(err as Error)); - } - - const queueEvents = new QueueEvents(queueName, { connection }); - - workers.push(worker); - events.push(queueEvents); - } - - return { - workers, - events, - close: async () => { - await Promise.all(workers.map((worker) => worker.close())); - await Promise.all(events.map((event) => event.close())); - if (!options.queueManager) { - await manager.close(); - } - }, - }; -}; diff --git a/basango/apps/crawler/src/services/crawler/index.ts b/basango/apps/crawler/src/services/crawler/index.ts deleted file mode 100644 index aa828f3..0000000 --- a/basango/apps/crawler/src/services/crawler/index.ts +++ /dev/null @@ -1,3 +0,0 @@ -export * from "./async/queue"; -export * from "./async/tasks"; -export * from "./async/worker"; diff --git a/basango/apps/crawler/src/utils.ts b/basango/apps/crawler/src/utils.ts index 6f9f544..2fc3b44 100644 --- a/basango/apps/crawler/src/utils.ts +++ b/basango/apps/crawler/src/utils.ts @@ -6,33 +6,33 @@ import { get_encoding } from "tiktoken"; import type { ProjectPaths } from "@crawler/schema"; export const ensureDirectories = (paths: ProjectPaths): void => { - for (const dir of [paths.data, paths.logs, paths.configs]) { - if (!fs.existsSync(dir)) { - fs.mkdirSync(dir, { recursive: true }); - } - } + for (const dir of [paths.data, paths.logs, paths.configs]) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + } + } }; export const parseRedisUrl = (url: string): RedisOptions => { - if (!url.startsWith("redis://")) { - return {}; - } - const parsed = new URL(url); - return { - host: parsed.hostname, - port: Number(parsed.port || 6379), - password: parsed.password || undefined, - db: Number(parsed.pathname?.replace("/", "") || 0), - }; + if (!url.startsWith("redis://")) { + return {}; + } + const parsed = new URL(url); + return { + host: parsed.hostname, + port: Number(parsed.port || 6379), + password: parsed.password || undefined, + db: Number(parsed.pathname?.replace("/", "") || 0), + }; }; export const countTokens = (text: string, encoding = "cl100k_base"): number => { - try { - const encoder = get_encoding(encoding); - const tokens = encoder.encode(text); - encoder.free(); - return tokens.length; - } catch { - return text.length; - } + try { + const encoder = get_encoding(encoding); + const tokens = encoder.encode(text); + encoder.free(); + return tokens.length; + } catch { + return text.length; + } }; diff --git a/basango/apps/crawler/tsconfig.json b/basango/apps/crawler/tsconfig.json index a11e79f..beeda9d 100644 --- a/basango/apps/crawler/tsconfig.json +++ b/basango/apps/crawler/tsconfig.json @@ -1,13 +1,13 @@ { - "extends": "@basango/tsconfig/base.json", - "compilerOptions": { - "rootDir": "src", - "outDir": "dist", - "paths": { - "@crawler": ["./src/index.ts"], - "@crawler/*": ["./src/*"] - } - }, - "include": ["src"], - "references": [] + "extends": "@basango/tsconfig/base.json", + "compilerOptions": { + "rootDir": "src", + "outDir": "dist", + "paths": { + "@crawler": ["./src/index.ts"], + "@crawler/*": ["./src/*"] + } + }, + "include": ["src"], + "references": [] } diff --git a/basango/apps/crawler/vitest.config.ts b/basango/apps/crawler/vitest.config.ts index acf96a1..869d8d7 100644 --- a/basango/apps/crawler/vitest.config.ts +++ b/basango/apps/crawler/vitest.config.ts @@ -1,9 +1,9 @@ import { defineConfig } from "vitest/config"; export default defineConfig({ - test: { - environment: "node", - globals: true, - include: ["src/**/*.test.ts"], - }, + test: { + environment: "node", + globals: true, + include: ["src/**/*.test.ts"], + }, }); diff --git a/basango/biome.json b/basango/biome.json index 39e5600..2bfef18 100644 --- a/basango/biome.json +++ b/basango/biome.json @@ -1,34 +1,34 @@ { - "$schema": "https://biomejs.dev/schemas/2.3.1/schema.json", - "vcs": { - "enabled": true, - "clientKind": "git", - "useIgnoreFile": true - }, - "files": { - "ignoreUnknown": false - }, - "formatter": { - "enabled": true, - "indentStyle": "space" - }, - "linter": { - "enabled": true, - "rules": { - "recommended": true - } - }, - "javascript": { - "formatter": { - "quoteStyle": "double" - } - }, - "assist": { - "enabled": true, - "actions": { - "source": { - "organizeImports": "on" - } - } - } + "$schema": "https://biomejs.dev/schemas/2.3.1/schema.json", + "vcs": { + "enabled": true, + "clientKind": "git", + "useIgnoreFile": true + }, + "files": { + "ignoreUnknown": false + }, + "formatter": { + "enabled": true, + "indentStyle": "space" + }, + "linter": { + "enabled": true, + "rules": { + "recommended": true + } + }, + "javascript": { + "formatter": { + "quoteStyle": "double" + } + }, + "assist": { + "enabled": true, + "actions": { + "source": { + "organizeImports": "on" + } + } + } } diff --git a/basango/bun.lock b/basango/bun.lock index 3cc9cf7..c6a3489 100644 --- a/basango/bun.lock +++ b/basango/bun.lock @@ -7,13 +7,14 @@ "@biomejs/biome": "^2.3.1", "@manypkg/cli": "^0.25.1", "turbo": "^2.5.8", - "typescript": "5.9.2", + "typescript": "catalog:", }, }, "apps/crawler": { "name": "@basango/crawler", "version": "0.1.0", "dependencies": { + "@basango/logger": "workspace:*", "bullmq": "^4.17.0", "date-fns": "^3.6.0", "ioredis": "^5.3.2", @@ -25,24 +26,28 @@ "name": "@basango/db", "version": "1.0.0", "dependencies": { + "@basango/logger": "workspace:*", "@date-fns/utc": "^2.1.1", "drizzle-orm": "^0.44.7", "pg": "^8.16.3", "snakecase-keys": "^9.0.2", }, "devDependencies": { + "@types/bun": "^1.3.1", + "@types/pg": "^8.15.6", "drizzle-kit": "^0.31.6", + "typescript": "catalog:", }, }, "packages/logger": { - "name": "@midday/logger", - "version": "0.0.0", + "name": "@basango/logger", + "version": "0.0.1", "dependencies": { "pino": "^10.1.0", "pino-pretty": "^13.1.2", }, "devDependencies": { - "typescript": "^5.9.2", + "typescript": "catalog:", }, }, "packages/tsconfig": { @@ -50,11 +55,17 @@ "version": "0.0.0", }, }, + "catalog": { + "@types/bun": "^1.3.1", + "typescript": "^5.9.3", + }, "packages": { "@basango/crawler": ["@basango/crawler@workspace:apps/crawler"], "@basango/db": ["@basango/db@workspace:packages/db"], + "@basango/logger": ["@basango/logger@workspace:packages/logger"], + "@basango/tsconfig": ["@basango/tsconfig@workspace:packages/tsconfig"], "@biomejs/biome": ["@biomejs/biome@2.3.1", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.1", "@biomejs/cli-darwin-x64": "2.3.1", "@biomejs/cli-linux-arm64": "2.3.1", "@biomejs/cli-linux-arm64-musl": "2.3.1", "@biomejs/cli-linux-x64": "2.3.1", "@biomejs/cli-linux-x64-musl": "2.3.1", "@biomejs/cli-win32-arm64": "2.3.1", "@biomejs/cli-win32-x64": "2.3.1" }, "bin": { "biome": "bin/biome" } }, "sha512-A29evf1R72V5bo4o2EPxYMm5mtyGvzp2g+biZvRFx29nWebGyyeOSsDWGx3tuNNMFRepGwxmA9ZQ15mzfabK2w=="], @@ -145,8 +156,6 @@ "@manypkg/tools": ["@manypkg/tools@2.1.0", "", { "dependencies": { "jju": "^1.4.0", "js-yaml": "^4.1.0", "tinyglobby": "^0.2.13" } }, "sha512-0FOIepYR4ugPYaHwK7hDeHDkfPOBVvayt9QpvRbi2LT/h2b0GaE/gM9Gag7fsnyYyNaTZ2IGyOuVg07IYepvYQ=="], - "@midday/logger": ["@midday/logger@workspace:packages/logger"], - "@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="], "@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="], @@ -167,6 +176,14 @@ "@pnpm/npm-conf": ["@pnpm/npm-conf@2.3.1", "", { "dependencies": { "@pnpm/config.env-replace": "^1.1.0", "@pnpm/network.ca-file": "^1.0.1", "config-chain": "^1.1.11" } }, "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw=="], + "@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="], + + "@types/node": ["@types/node@24.9.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA=="], + + "@types/pg": ["@types/pg@8.15.6", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ=="], + + "@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="], + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], @@ -179,6 +196,8 @@ "bullmq": ["bullmq@4.18.3", "", { "dependencies": { "cron-parser": "^4.6.0", "glob": "^8.0.3", "ioredis": "^5.3.2", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-H8t9vhfHEbJDaXp7aalSTe+Do+tR1nvr+lsT+jQxLhy+FFfFj/0p4aYJzADTNLdEqltuxneLVxCGVg92GkQx4w=="], + "bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="], + "change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="], "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], @@ -189,6 +208,8 @@ "cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="], + "csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="], + "date-fns": ["date-fns@3.6.0", "", {}, "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww=="], "dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="], @@ -387,7 +408,9 @@ "type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="], - "typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="], + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="], "uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="], diff --git a/basango/docs/architecture.md b/basango/docs/architecture.md new file mode 100644 index 0000000..c1f67e4 --- /dev/null +++ b/basango/docs/architecture.md @@ -0,0 +1,105 @@ +Specifying packages in a monorepo +Declaring directories for packages +First, your package manager needs to describe the locations of your packages. We recommend starting with splitting your packages into apps/ for applications and services and packages/ for everything else, like libraries and tooling. + +pnpm +yarn +npm +bun +./package.json + +{ +"workspaces": [ +"apps/*", +"packages/*" +] +} +→ +bun workspace documentation +Using this configuration, every directory with a package.json in the apps or packages directories will be considered a package. + +Turborepo does not support nested packages like apps/** or packages/** due to ambiguous behavior among package managers in the JavaScript ecosystem. Using a structure that would put a package at apps/a and another at apps/a/b will result in an error.If you'd like to group packages by directory, you can do this using globs like packages/* and packages/group/* and not creating a packages/group/package.json file. +package.json in each package +In the directory of the package, there must be a package.json to make the package discoverable to your package manager and turbo. The requirements for the package.json of a package are below. + +Root package.json +The root package.json is the base for your workspace. Below is a common example of what you would find in a root package.json: + +pnpm +yarn +npm +bun +./package.json + +{ +"private": true, +"scripts": { +"build": "turbo run build", +"dev": "turbo run dev", +"lint": "turbo run lint" +}, +"devDependencies": { +"turbo": "latest" +}, +"packageManager": "bun@1.2.0", +"workspaces": ["apps/*", "packages/*"] +} +Root turbo.json +turbo.json is used to configure the behavior of turbo. To learn more about how to configure your tasks, visit the Configuring tasks page. + +Package manager lockfile +A lockfile is key to reproducible behavior for both your package manager and turbo. Additionally, Turborepo uses the lockfile to understand the dependencies between your Internal Packages within your Workspace. + +If you do not have a lockfile present when you run turbo, you may see unpredictable behavior. +Anatomy of a package +It's often best to start thinking about designing a package as its own unit within the Workspace. At a high-level, each package is almost like its own small "project", with its own package.json, tooling configuration, and source code. There are limits to this idea—but its a good mental model to start from. + +Additionally, a package has specific entrypoints that other packages in your Workspace can use to access the package, specified by exports. + +package.json for a package +name +The name field is used to identify the package. It should be unique within your workspace. + +It's best practice to use a namespace prefix for your Internal Packages to avoid conflicts with other packages on the npm registry. For example, if your organization is named acme, you might name your packages @acme/package-name.We use @repo in our docs and examples because it is an unused, unclaimable namespace on the npm registry. You can choose to keep it or use your own prefix. +scripts +The scripts field is used to define scripts that can be run in the package's context. Turborepo will use the name of these scripts to identify what scripts to run (if any) in a package. We talk more about these scripts on the Running Tasks page. + +exports +The exports field is used to specify the entrypoints for other packages that want to use the package. When you want to use code from one package in another package, you'll import from that entrypoint. + +For example, if you had a @repo/math package, you might have the following exports field: + +./packages/math/package.json + +{ +"exports": { +".": "./src/constants.ts", +"./add": "./src/add.ts", +"./subtract": "./src/subtract.ts" +} +} +Note that this example uses the Just-in-Time Package pattern for simplicity. It exports TypeScript directly, but you might choose to use the Compiled Package pattern instead. + +The exports field in this example requires modern versions of Node.js and TypeScript. +This would allow you to import add and subtract functions from the @repo/math package like so: + +./apps/my-app/src/index.ts + +import { GRAVITATIONAL_CONSTANT, SPEED_OF_LIGHT } from '@repo/math'; +import { add } from '@repo/math/add'; +import { subtract } from '@repo/math/subtract'; +Using exports this way provides three major benefits: + +Avoiding barrel files: Barrel files are files that re-export other files in the same package, creating one entrypoint for the entire package. While they might appear convenient, they're difficult for compilers and bundlers to handle and can quickly lead to performance problems. +More powerful features: exports also has other powerful features compared to the main field like Conditional Exports. In general, we recommend using exports over main whenever possible as it is the more modern option. +IDE autocompletion: By specifying the entrypoints for your package using exports, you can ensure that your code editor can provide auto-completion for the package's exports. +imports (optional) +The imports field gives you a way to create subpaths to other modules within your package. You can think of these like "shortcuts" to write simpler import paths that are more resilient to refactors that move files. To learn how, visit the TypeScript page. + +You may be more familiar with TypeScript's compilerOptions#paths option, which accomplishes a similar goal. As of TypeScript 5.4, TypeScript can infer subpaths from imports, making it a better option since you'll be working with Node.js conventions. For more information, visit our TypeScript guide. +Source code +Of course, you'll want some source code in your package. Packages commonly use an src directory to store their source code and compile to a dist directory (that should also be located within the package), although this is not a requirement. + +Common pitfalls +If you're using TypeScript, you likely don't need a tsconfig.json in the root of your workspace. Packages should independently specify their own configurations, usually building off of a shared tsconfig.json from a separate package in the workspace. For more information, visit the TypeScript guide. +You want to avoid accessing files across package boundaries as much as possible. If you ever find yourself writing ../ to get from one package to another, you likely have an opportunity to re-think your approach by installing the package where it's needed and importing it into your code. \ No newline at end of file diff --git a/basango/package.json b/basango/package.json index fa8fa82..52c9dc1 100644 --- a/basango/package.json +++ b/basango/package.json @@ -1,28 +1,33 @@ { - "name": "basango", - "private": true, - "scripts": { - "build": "turbo run build", - "clean": "git clean -xdf node_modules", - "clean:workspaces": "turbo run clean", - "dev": "turbo run dev --parallel", - "test": "turbo run test --parallel", - "lint": "turbo run lint && manypkg check", - "format": "biome format --write .", - "typecheck": "turbo run typecheck" - }, - "devDependencies": { - "@biomejs/biome": "^2.3.1", - "@manypkg/cli": "^0.25.1", - "turbo": "^2.5.8", - "typescript": "5.9.2" - }, - "engines": { - "node": ">=22" - }, - "packageManager": "bun@1.2.8", - "workspaces": [ - "apps/*", - "packages/*" - ] + "name": "basango", + "private": true, + "scripts": { + "build": "turbo run build", + "clean": "git clean -xdf node_modules", + "clean:workspaces": "turbo run clean", + "dev": "turbo run dev --parallel", + "test": "turbo run test --parallel", + "lint": "turbo run lint && manypkg check", + "format": "biome format --write .", + "typecheck": "turbo run typecheck" + }, + "devDependencies": { + "@biomejs/biome": "^2.3.1", + "@manypkg/cli": "^0.25.1", + "turbo": "^2.5.8", + "typescript": "catalog:" + }, + "engines": { + "node": ">=22" + }, + "packageManager": "bun@1.3.1", + "workspaces": [ + "apps/*", + "packages/*" + ], + "catalog": { + "typescript": "^5.9.3", + "@types/bun": "^1.3.1", + "zod": "^4.0.0" + } } diff --git a/basango/packages/db/drizzle.config.ts b/basango/packages/db/drizzle.config.ts index d853548..dd77552 100644 --- a/basango/packages/db/drizzle.config.ts +++ b/basango/packages/db/drizzle.config.ts @@ -5,6 +5,6 @@ export default { out: "./migrations", dialect: "postgresql", dbCredentials: { - url: process.env.DATABASE_SESSION_POOLER!, + url: process.env.DATABASE_URL!, }, -} satisfies Config; \ No newline at end of file +} satisfies Config; diff --git a/basango/packages/db/package.json b/basango/packages/db/package.json index 8408715..4541af3 100644 --- a/basango/packages/db/package.json +++ b/basango/packages/db/package.json @@ -1,17 +1,23 @@ { "name": "@basango/db", - "version": "1.0.0", - "main": "index.ts", - "author": "", - "license": "ISC", - "description": "", + "private": true, + "exports": { + "./client": "./src/client.ts", + "./schema": "./src/schema.ts", + "./utils": "./src/utils/index.ts", + "./queries": "./src/queries/index.ts" + }, "dependencies": { + "@basango/logger": "workspace:*", "@date-fns/utc": "^2.1.1", "drizzle-orm": "^0.44.7", "pg": "^8.16.3", "snakecase-keys": "^9.0.2" }, "devDependencies": { - "drizzle-kit": "^0.31.6" + "@types/bun": "^1.3.1", + "@types/pg": "^8.15.6", + "drizzle-kit": "^0.31.6", + "typescript": "catalog:" } } diff --git a/basango/packages/db/src/client.ts b/basango/packages/db/src/client.ts index f554d97..3dfec2b 100644 --- a/basango/packages/db/src/client.ts +++ b/basango/packages/db/src/client.ts @@ -1,6 +1,6 @@ import { drizzle } from "drizzle-orm/node-postgres"; import { Pool } from "pg"; -import * as schema from "@basango/db/schema"; +import * as schema from "@db/schema"; const isDevelopment = process.env.NODE_ENV === "development"; diff --git a/basango/packages/db/src/constant.ts b/basango/packages/db/src/constant.ts new file mode 100644 index 0000000..53ad877 --- /dev/null +++ b/basango/packages/db/src/constant.ts @@ -0,0 +1,2 @@ +export const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/"; +export const PUBLICATION_GRAPH_DAYS = 180; diff --git a/basango/packages/db/src/queries/aggregator/articles.ts b/basango/packages/db/src/queries/aggregator/articles.ts deleted file mode 100644 index 6166ec3..0000000 --- a/basango/packages/db/src/queries/aggregator/articles.ts +++ /dev/null @@ -1,84 +0,0 @@ -import type { SQL } from "drizzle-orm"; -import { and, desc, eq, sql } from "drizzle-orm"; - -import type { Database } from "@db/client"; -import { articles, sources } from "@db/schema"; - -export interface ArticleExportRow { - articleId: string; - articleTitle: string; - articleLink: string; - articleCategories: string | null; - articleBody: string; - articleSource: string; - articleHash: string; - articlePublishedAt: string; - articleCrawledAt: string; -} - -export interface ArticleExportParams { - source?: string | null; - dateRange?: { start: number; end: number } | null; - batchSize?: number; -} - -export async function* getArticlesForExport( - db: Database, - params: ArticleExportParams = {}, -): AsyncGenerator { - const batchSize = params.batchSize && params.batchSize > 0 - ? params.batchSize - : 1000; - - const filters: SQL[] = []; - - if (params.source) { - filters.push(eq(sources.name, params.source)); - } - - if (params.dateRange) { - filters.push( - sql`${articles.publishedAt} BETWEEN to_timestamp(${params.dateRange.start}) AND to_timestamp(${params.dateRange.end})`, - ); - } - - let query = db - .select({ - articleId: articles.id, - articleTitle: articles.title, - articleLink: articles.link, - articleCategories: sql`array_to_string(${articles.categories}, ',')`, - articleBody: articles.body, - articleSource: sources.name, - articleHash: articles.hash, - articlePublishedAt: articles.publishedAt, - articleCrawledAt: articles.crawledAt, - }) - .from(articles) - .innerJoin(sources, eq(articles.sourceId, sources.id)); - - if (filters.length === 1) { - query = query.where(filters[0]); - } else if (filters.length > 1) { - query = query.where(and(...filters)); - } - - query = query.orderBy(desc(articles.publishedAt), desc(articles.id)); - - let offset = 0; - while (true) { - const rows = await query.limit(batchSize).offset(offset); - if (rows.length === 0) { - break; - } - - for (const row of rows) { - yield { - ...row, - articleCategories: row.articleCategories ?? null, - }; - } - - offset += batchSize; - } -} diff --git a/basango/packages/db/src/queries/aggregator/sources.ts b/basango/packages/db/src/queries/aggregator/sources.ts deleted file mode 100644 index 431b448..0000000 --- a/basango/packages/db/src/queries/aggregator/sources.ts +++ /dev/null @@ -1,87 +0,0 @@ -import { and, eq, sql } from "drizzle-orm"; - -import type { Database } from "@db/client"; -import { articles, sources } from "@db/schema"; - -export interface SourceStatisticsRow { - sourceId: string; - sourceName: string; - sourceCrawledAt: string | null; - articlesCount: number; - articleMetadataAvailable: number; -} - -export async function getSourceStatisticsList( - db: Database, -): Promise { - const rows = await db - .select({ - sourceId: sources.id, - sourceName: sources.name, - sourceCrawledAt: sql`max(${articles.crawledAt})`, - articlesCount: sql`count(${articles.id})`, - articleMetadataAvailable: sql`sum(CASE WHEN ${articles.metadata} IS NOT NULL THEN 1 ELSE 0 END)`, - }) - .from(sources) - .leftJoin(articles, eq(articles.sourceId, sources.id)) - .groupBy(sources.id, sources.name) - .orderBy(sources.name.asc()); - - return rows.map((row) => ({ - sourceId: row.sourceId, - sourceName: row.sourceName, - sourceCrawledAt: row.sourceCrawledAt, - articlesCount: Number(row.articlesCount ?? 0), - articleMetadataAvailable: Number(row.articleMetadataAvailable ?? 0), - })); -} - -export interface PublicationDateParams { - source: string; - category?: string | null; -} - -async function selectPublicationBoundary( - db: Database, - fn: "min" | "max", - params: PublicationDateParams, -): Promise { - const conditions = [eq(sources.name, params.source)]; - - if (params.category) { - conditions.push( - sql`${params.category} = ANY(${articles.categories})`, - ); - } - - const whereClause = conditions.length > 1 - ? and(...conditions) - : conditions[0]; - - const [result] = await db - .select({ - boundary: - fn === "min" - ? sql`min(${articles.publishedAt})` - : sql`max(${articles.publishedAt})`, - }) - .from(articles) - .innerJoin(sources, eq(articles.sourceId, sources.id)) - .where(whereClause); - - return result?.boundary ?? new Date().toISOString(); -} - -export async function getEarliestPublicationDate( - db: Database, - params: PublicationDateParams, -): Promise { - return selectPublicationBoundary(db, "min", params); -} - -export async function getLatestPublicationDate( - db: Database, - params: PublicationDateParams, -): Promise { - return selectPublicationBoundary(db, "max", params); -} diff --git a/basango/packages/db/src/queries/feed-management/articles.ts b/basango/packages/db/src/queries/articles.ts similarity index 80% rename from basango/packages/db/src/queries/feed-management/articles.ts rename to basango/packages/db/src/queries/articles.ts index 488a050..802472e 100644 --- a/basango/packages/db/src/queries/feed-management/articles.ts +++ b/basango/packages/db/src/queries/articles.ts @@ -1,18 +1,9 @@ -import type { SQL, AnyColumn } from "drizzle-orm"; -import { - and, - asc, - desc, - eq, - gt, - lt, - or, - sql, -} from "drizzle-orm"; +import type { AnyColumn, SQL } from "drizzle-orm"; +import { and, asc, desc, eq, gt, lt, or, sql } from "drizzle-orm"; import type { Database } from "@db/client"; import { - appUsers, + users, articles, bookmarkArticles, bookmarks, @@ -104,6 +95,86 @@ interface NormalizedArticleFilters { sortDirection: SortDirection; } +export interface ArticleExportRow { + articleId: string; + articleTitle: string; + articleLink: string; + articleCategories: string | null; + articleBody: string; + articleSource: string; + articleHash: string; + articlePublishedAt: string; + articleCrawledAt: string; +} + +export interface ArticleExportParams { + source?: string | null; + dateRange?: { start: number; end: number } | null; + batchSize?: number; +} + +export async function* getArticlesForExport( + db: Database, + params: ArticleExportParams = {}, +): AsyncGenerator { + const batchSize = + params.batchSize && params.batchSize > 0 ? params.batchSize : 1000; + + const filters: SQL[] = []; + + if (params.source) { + filters.push(eq(sources.name, params.source)); + } + + if (params.dateRange) { + filters.push( + sql`${articles.publishedAt} BETWEEN to_timestamp(${params.dateRange.start}) AND to_timestamp(${params.dateRange.end})`, + ); + } + + let query = db + .select({ + articleId: articles.id, + articleTitle: articles.title, + articleLink: articles.link, + articleCategories: sql< + string | null + >`array_to_string(${articles.categories}, ',')`, + articleBody: articles.body, + articleSource: sources.name, + articleHash: articles.hash, + articlePublishedAt: articles.publishedAt, + articleCrawledAt: articles.crawledAt, + }) + .from(articles) + .innerJoin(sources, eq(articles.sourceId, sources.id)); + + if (filters.length === 1) { + query = query.where(filters[0]); + } else if (filters.length > 1) { + query = query.where(and(...filters)); + } + + query = query.orderBy(desc(articles.publishedAt), desc(articles.id)); + + let offset = 0; + while (true) { + const rows = await query.limit(batchSize).offset(offset); + if (rows.length === 0) { + break; + } + + for (const row of rows) { + yield { + ...row, + articleCategories: row.articleCategories ?? null, + }; + } + + offset += batchSize; + } +} + const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/"; function normalizeArticleFilters( @@ -113,7 +184,8 @@ function normalizeArticleFilters( const trimmedCategory = filters?.category?.trim(); return { - search: trimmedSearch && trimmedSearch.length > 0 ? trimmedSearch : undefined, + search: + trimmedSearch && trimmedSearch.length > 0 ? trimmedSearch : undefined, category: trimmedCategory && trimmedCategory.length > 0 ? trimmedCategory @@ -123,9 +195,10 @@ function normalizeArticleFilters( }; } -function buildArticleFilterConditions( - filters: NormalizedArticleFilters, -): { conditions: SQL[]; searchQuery?: string } { +function buildArticleFilterConditions(filters: NormalizedArticleFilters): { + conditions: SQL[]; + searchQuery?: string; +} { const conditions: SQL[] = []; let searchQuery: string | undefined; @@ -181,7 +254,9 @@ async function fetchArticleOverview( article_id: articles.id, article_title: articles.title, article_link: articles.link, - article_categories: sql`array_to_string(${articles.categories}, ',')`, + article_categories: sql< + string | null + >`array_to_string(${articles.categories}, ',')`, article_excerpt: articles.excerpt, article_published_at: articles.publishedAt, article_image: articles.image, @@ -242,9 +317,7 @@ async function fetchArticleOverview( orderings.push(desc(articles.publishedAt), desc(articles.id)); } - const rows = await query - .orderBy(...orderings) - .limit(options.page.limit + 1); + const rows = await query.orderBy(...orderings).limit(options.page.limit + 1); return buildPaginationResult(rows, options.page, { id: "article_id", @@ -314,7 +387,9 @@ export async function getBookmarkedArticleList( article_id: articles.id, article_title: articles.title, article_link: articles.link, - article_categories: sql`array_to_string(${articles.categories}, ',')`, + article_categories: sql< + string | null + >`array_to_string(${articles.categories}, ',')`, article_excerpt: articles.excerpt, article_published_at: articles.publishedAt, article_image: articles.image, @@ -377,9 +452,7 @@ export async function getBookmarkedArticleList( orderings.push(desc(articles.publishedAt), desc(articles.id)); } - const rows = await query - .orderBy(...orderings) - .limit(page.limit + 1); + const rows = await query.orderBy(...orderings).limit(page.limit + 1); return buildPaginationResult(rows, page, { id: "article_id", @@ -398,7 +471,9 @@ export async function getArticleDetails( article_id: articles.id, article_title: articles.title, article_link: articles.link, - article_categories: sql`array_to_string(${articles.categories}, ',')`, + article_categories: sql< + string | null + >`array_to_string(${articles.categories}, ',')`, article_body: articles.body, article_hash: articles.hash, article_published_at: articles.publishedAt, @@ -442,10 +517,7 @@ export async function getArticleCommentList( whereConditions.push( or( lt(comments.createdAt, cursor.date), - and( - eq(comments.createdAt, cursor.date), - lt(comments.id, cursor.id), - ), + and(eq(comments.createdAt, cursor.date), lt(comments.id, cursor.id)), ), ); } @@ -456,11 +528,11 @@ export async function getArticleCommentList( comment_content: comments.content, comment_sentiment: comments.sentiment, comment_created_at: comments.createdAt, - user_id: appUsers.id, - user_name: appUsers.name, + user_id: users.id, + user_name: users.name, }) .from(comments) - .innerJoin(appUsers, eq(comments.userId, appUsers.id)); + .innerJoin(users, eq(comments.userId, users.id)); if (whereConditions.length === 1) { query = query.where(whereConditions[0]); diff --git a/basango/packages/db/src/queries/feed-management/bookmarks.ts b/basango/packages/db/src/queries/bookmarks.ts similarity index 100% rename from basango/packages/db/src/queries/feed-management/bookmarks.ts rename to basango/packages/db/src/queries/bookmarks.ts diff --git a/basango/packages/db/src/queries/index.ts b/basango/packages/db/src/queries/index.ts new file mode 100644 index 0000000..3b24f06 --- /dev/null +++ b/basango/packages/db/src/queries/index.ts @@ -0,0 +1,4 @@ +export * from "./articles"; +export * from "./bookmarks"; +export * from "./sources"; +export * from "./users"; diff --git a/basango/packages/db/src/queries/feed-management/sources.ts b/basango/packages/db/src/queries/sources.ts similarity index 63% rename from basango/packages/db/src/queries/feed-management/sources.ts rename to basango/packages/db/src/queries/sources.ts index b15f95e..5db5e91 100644 --- a/basango/packages/db/src/queries/feed-management/sources.ts +++ b/basango/packages/db/src/queries/sources.ts @@ -9,11 +9,8 @@ import { decodeCursor, type PageRequest, type PaginationMeta, - type PageState, } from "@db/utils/pagination"; - -const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/"; -const PUBLICATION_GRAPH_DAYS = 180; +import { PUBLICATION_GRAPH_DAYS, SOURCE_IMAGE_BASE } from "@db/constant"; export interface SourceOverviewRow { source_id: string; @@ -62,12 +59,97 @@ export interface SourceDetailsResult { categoryShares: CategoryShare[]; } +export interface SourceStatisticsRow { + sourceId: string; + sourceName: string; + sourceCrawledAt: string | null; + articlesCount: number; + articleMetadataAvailable: number; +} + +export async function getSourceStatisticsList( + db: Database, +): Promise { + const rows = await db + .select({ + sourceId: sources.id, + sourceName: sources.name, + sourceCrawledAt: sql`max + (${articles.crawledAt})`, + articlesCount: sql`count + (${articles.id})`, + articleMetadataAvailable: sql`sum + (CASE WHEN ${articles.metadata} IS NOT NULL THEN 1 ELSE 0 END)`, + }) + .from(sources) + .leftJoin(articles, eq(articles.sourceId, sources.id)) + .groupBy(sources.id, sources.name) + .orderBy(sources.name.asc()); + + return rows.map((row) => ({ + sourceId: row.sourceId, + sourceName: row.sourceName, + sourceCrawledAt: row.sourceCrawledAt, + articlesCount: Number(row.articlesCount ?? 0), + articleMetadataAvailable: Number(row.articleMetadataAvailable ?? 0), + })); +} + +export interface PublicationDateParams { + source: string; + category?: string | null; +} + +async function selectPublicationBoundary( + db: Database, + fn: "min" | "max", + params: PublicationDateParams, +): Promise { + const conditions: SQL[] = [eq(sources.name, params.source)]; + + if (params.category) { + conditions.push(sql`${params.category} = ANY(${articles.categories})`); + } + + const whereClause = + conditions.length > 1 ? and(...conditions) : conditions[0]; + + const [result] = await db + .select({ + boundary: + fn === "min" + ? sql`min + (${articles.publishedAt})` + : sql`max + (${articles.publishedAt})`, + }) + .from(articles) + .innerJoin(sources, eq(articles.sourceId, sources.id)) + .where(whereClause); + + return result?.boundary ?? new Date().toISOString(); +} + +export async function getEarliestPublicationDate( + db: Database, + params: PublicationDateParams, +): Promise { + return selectPublicationBoundary(db, "min", params); +} + +export async function getLatestPublicationDate( + db: Database, + params: PublicationDateParams, +): Promise { + return selectPublicationBoundary(db, "max", params); +} + function buildFollowExistsExpression(userId: string): SQL { - return sql`EXISTS ( - SELECT 1 - FROM ${followedSources} f - WHERE f.source_id = ${sources.id} AND f.follower_id = ${userId} - )`; + return sql`EXISTS + (SELECT 1 + FROM ${followedSources} f + WHERE f.source_id = ${sources.id} + AND f.follower_id = ${userId})`; } export async function getSourceOverviewList( @@ -126,16 +208,27 @@ async function fetchPublicationGraph( const rows = await db .select({ - day: sql`date(${articles.publishedAt})`, - count: sql`count(${articles.id})`, + day: sql`date + (${articles.publishedAt})`, + count: sql`count + (${articles.id})`, }) .from(articles) .where(eq(articles.sourceId, sourceId)) .where( - sql`${articles.publishedAt} BETWEEN to_timestamp(${range.start}) AND to_timestamp(${range.end})`, + sql`${articles.publishedAt} BETWEEN to_timestamp( + ${range.start} + ) + AND + to_timestamp + ( + ${range.end} + )`, ) - .groupBy(sql`date(${articles.publishedAt})`) - .orderBy(sql`date(${articles.publishedAt})`); + .groupBy(sql`date + (${articles.publishedAt})`) + .orderBy(sql`date + (${articles.publishedAt})`); const counts = new Map(); for (const row of rows) { @@ -164,7 +257,8 @@ async function fetchCategoryShares( ): Promise { const rows = await db .select({ - categories: sql`array_to_string(${articles.categories}, ',')`, + categories: sql`array_to_string + (${articles.categories}, ',')`, }) .from(articles) .where(eq(articles.sourceId, sourceId)); @@ -179,7 +273,10 @@ async function fetchCategoryShares( } } - const total = Array.from(counts.values()).reduce((acc, value) => acc + value, 0); + const total = Array.from(counts.values()).reduce( + (acc, value) => acc + value, + 0, + ); const shares: CategoryShare[] = Array.from(counts.entries()).map( ([category, count]) => ({ @@ -211,9 +308,18 @@ export async function getSourceDetails( source_reliability: sources.reliability, source_transparency: sources.transparency, source_image: sql`('${SOURCE_IMAGE_BASE}' || ${sources.name} || '.png')`, - articles_count: sql`count(${articles.id})`, - source_crawled_at: sql`max(${articles.crawledAt})`, - articles_metadata_available: sql`count(*) FILTER (WHERE ${articles.metadata} IS NOT NULL)`, + articles_count: sql`count + (${articles.id})`, + source_crawled_at: sql`max + (${articles.crawledAt})`, + articles_metadata_available: sql`count + (*) + FILTER (WHERE + ${articles.metadata} + IS + NOT + NULL + )`, source_is_followed: followExpression, }) .from(sources) diff --git a/basango/packages/db/src/queries/identity/users.ts b/basango/packages/db/src/queries/users.ts similarity index 61% rename from basango/packages/db/src/queries/identity/users.ts rename to basango/packages/db/src/queries/users.ts index c8ba004..6b0d08a 100644 --- a/basango/packages/db/src/queries/identity/users.ts +++ b/basango/packages/db/src/queries/users.ts @@ -1,7 +1,7 @@ import { eq } from "drizzle-orm"; import type { Database } from "@db/client"; -import { appUsers } from "@db/schema"; +import { users } from "@db/schema"; export interface UserProfileRow { user_id: string; @@ -17,14 +17,14 @@ export async function getUserProfile( ): Promise { const [row] = await db .select({ - user_id: appUsers.id, - user_name: appUsers.name, - user_email: appUsers.email, - user_created_at: appUsers.createdAt, - user_updated_at: appUsers.updatedAt, + user_id: users.id, + user_name: users.name, + user_email: users.email, + user_created_at: users.createdAt, + user_updated_at: users.updatedAt, }) - .from(appUsers) - .where(eq(appUsers.id, params.userId)) + .from(users) + .where(eq(users.id, params.userId)) .limit(1); return row ?? null; diff --git a/basango/packages/db/src/schema.ts b/basango/packages/db/src/schema.ts index d4d5904..2b71668 100644 --- a/basango/packages/db/src/schema.ts +++ b/basango/packages/db/src/schema.ts @@ -1,29 +1,22 @@ -import { type SQL, relations, sql } from "drizzle-orm"; +import { relations, sql } from "drizzle-orm"; import { - bigint, boolean, customType, - date, doublePrecision, foreignKey, index, integer, - json, jsonb, - numeric, + inet, pgEnum, - pgMaterializedView, - pgPolicy, pgTable, primaryKey, - smallint, text, timestamp, unique, uniqueIndex, uuid, varchar, - vector, } from "drizzle-orm/pg-core"; export const tsvector = customType<{ @@ -34,14 +27,6 @@ export const tsvector = customType<{ }, }); -export const inet = customType<{ - data: string; -}>({ - dataType() { - return "inet"; - }, -}); - type NumericConfig = { precision?: number; scale?: number; @@ -62,71 +47,6 @@ export const numericCasted = customType<{ toDriver: (value: number) => value.toString(), }); -export const accountTypeEnum = pgEnum("account_type", [ - "depository", - "credit", - "other_asset", - "loan", - "other_liability", -]); - -export const bankProvidersEnum = pgEnum("bank_providers", [ - "gocardless", - "plaid", - "teller", - "enablebanking", -]); - -export const connectionStatusEnum = pgEnum("connection_status", [ - "disconnected", - "connected", - "unknown", -]); - -export const documentProcessingStatusEnum = pgEnum( - "document_processing_status", - ["pending", "processing", "completed", "failed"], -); - -export const inboxAccountProvidersEnum = pgEnum("inbox_account_providers", [ - "gmail", - "outlook", -]); - -export const inboxAccountStatusEnum = pgEnum("inbox_account_status", [ - "connected", - "disconnected", -]); - -export const inboxStatusEnum = pgEnum("inbox_status", [ - "processing", - "pending", - "archived", - "new", - "analyzing", - "suggested_match", - "no_match", - "done", - "deleted", -]); - -export const inboxTypeEnum = pgEnum("inbox_type", ["invoice", "expense"]); -export const invoiceDeliveryTypeEnum = pgEnum("invoice_delivery_type", [ - "create", - "create_and_send", - "scheduled", -]); - -export const invoiceSizeEnum = pgEnum("invoice_size", ["a4", "letter"]); -export const invoiceStatusEnum = pgEnum("invoice_status", [ - "draft", - "overdue", - "paid", - "unpaid", - "canceled", - "scheduled", -]); - export const articleSentimentEnum = pgEnum("article_sentiment", [ "positive", "neutral", @@ -156,2371 +76,7 @@ export const transparencyEnum = pgEnum("transparency", [ export const verificationTokenPurposeEnum = pgEnum( "verification_token_purpose", - [ - "confirm_account", - "password_reset", - "unlock_account", - "delete_account", - ], -); - -export const plansEnum = pgEnum("plans", ["trial", "starter", "pro"]); -export const subscriptionStatusEnum = pgEnum("subscription_status", [ - "active", - "canceled", - "past_due", - "unpaid", - "trialing", - "incomplete", - "incomplete_expired", -]); -export const reportTypesEnum = pgEnum("reportTypes", [ - "profit", - "revenue", - "burn_rate", - "expense", -]); - -export const teamRolesEnum = pgEnum("teamRoles", ["owner", "member"]); -export const trackerStatusEnum = pgEnum("trackerStatus", [ - "in_progress", - "completed", -]); - -export const transactionMethodsEnum = pgEnum("transactionMethods", [ - "payment", - "card_purchase", - "card_atm", - "transfer", - "other", - "unknown", - "ach", - "interest", - "deposit", - "wire", - "fee", -]); - -export const transactionStatusEnum = pgEnum("transactionStatus", [ - "posted", - "pending", - "excluded", - "completed", - "archived", -]); - -export const transactionFrequencyEnum = pgEnum("transaction_frequency", [ - "weekly", - "biweekly", - "monthly", - "semi_monthly", - "annually", - "irregular", - "unknown", -]); - -export const activityTypeEnum = pgEnum("activity_type", [ - // System-generated activities - "transactions_enriched", - "transactions_created", - "invoice_paid", - "inbox_new", - "inbox_auto_matched", - "inbox_needs_review", - "inbox_cross_currency_matched", - "invoice_overdue", - "invoice_sent", - "inbox_match_confirmed", - - // User actions - "document_uploaded", - "document_processed", - "invoice_duplicated", - "invoice_scheduled", - "invoice_reminder_sent", - "invoice_cancelled", - "invoice_created", - "draft_invoice_created", - "tracker_entry_created", - "tracker_project_created", - "transactions_categorized", - "transactions_assigned", - "transaction_attachment_created", - "transaction_category_created", - "transactions_exported", - "customer_created", -]); - -export const activitySourceEnum = pgEnum("activity_source", [ - "system", // Automated system processes - "user", // Direct user actions -]); - -export const activityStatusEnum = pgEnum("activity_status", [ - "unread", - "read", - "archived", -]); - -export const documentTagEmbeddings = pgTable( - "document_tag_embeddings", - { - slug: text().primaryKey().notNull(), - embedding: vector({ dimensions: 768 }), - name: text().notNull(), - model: text().notNull().default("gemini-embedding-001"), - }, - (table) => [ - index("document_tag_embeddings_idx") - .using("hnsw", table.embedding.asc().nullsLast().op("vector_cosine_ops")) - .with({ m: "16", ef_construction: "64" }), - pgPolicy("Enable insert for authenticated users only", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`true`, - }), - ], -); - -export const transactionCategoryEmbeddings = pgTable( - "transaction_category_embeddings", - { - name: text().primaryKey().notNull(), // Unique by name - same embedding for all teams - embedding: vector({ dimensions: 768 }), - model: text().notNull().default("gemini-embedding-001"), - system: boolean().default(false).notNull(), // Whether this comes from system categories - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - }, - (table) => [ - // Vector similarity index for fast cosine similarity search - index("transaction_category_embeddings_vector_idx") - .using("hnsw", table.embedding.asc().nullsLast().op("vector_cosine_ops")) - .with({ m: "16", ef_construction: "64" }), - // System categories index for filtering - index("transaction_category_embeddings_system_idx").using( - "btree", - table.system.asc().nullsLast().op("bool_ops"), - ), - pgPolicy("Enable read access for authenticated users", { - as: "permissive", - for: "select", - to: ["authenticated"], - using: sql`true`, - }), - pgPolicy("Enable insert for authenticated users only", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`true`, - }), - pgPolicy("Enable update for authenticated users only", { - as: "permissive", - for: "update", - to: ["authenticated"], - using: sql`true`, - }), - ], -); - -export const transactions = pgTable( - "transactions", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - date: date().notNull(), - name: text().notNull(), - method: transactionMethodsEnum().notNull(), - amount: numericCasted({ precision: 10, scale: 2 }).notNull(), - currency: text().notNull(), - teamId: uuid("team_id").notNull(), - assignedId: uuid("assigned_id"), - note: varchar(), - bankAccountId: uuid("bank_account_id"), - internalId: text("internal_id").notNull(), - status: transactionStatusEnum().default("posted"), - balance: numericCasted({ precision: 10, scale: 2 }), - manual: boolean().default(false), - notified: boolean().default(false), - internal: boolean().default(false), - description: text(), - categorySlug: text("category_slug"), - baseAmount: numericCasted({ precision: 10, scale: 2 }), - counterpartyName: text("counterparty_name"), - baseCurrency: text("base_currency"), - taxAmount: numericCasted("tax_amount", { precision: 10, scale: 2 }), - taxRate: numericCasted("tax_rate", { precision: 10, scale: 2 }), - taxType: text("tax_type"), - recurring: boolean(), - frequency: transactionFrequencyEnum(), - merchantName: text("merchant_name"), - enrichmentCompleted: boolean("enrichment_completed").default(false), - ftsVector: tsvector("fts_vector") - .notNull() - .generatedAlwaysAs( - (): SQL => sql` - to_tsvector( - 'english', - ( - (COALESCE(name, ''::text) || ' '::text) || COALESCE(description, ''::text) - ) - ) - `, - ), - }, - (table) => [ - index("idx_transactions_date").using( - "btree", - table.date.asc().nullsLast().op("date_ops"), - ), - index("idx_transactions_fts").using( - "gin", - table.ftsVector.asc().nullsLast().op("tsvector_ops"), - ), - index("idx_transactions_fts_vector").using( - "gin", - table.ftsVector.asc().nullsLast().op("tsvector_ops"), - ), - index("idx_transactions_id").using( - "btree", - table.id.asc().nullsLast().op("uuid_ops"), - ), - index("idx_transactions_name").using( - "btree", - table.name.asc().nullsLast().op("text_ops"), - ), - index("idx_transactions_name_trigram").using( - "gin", - table.name.asc().nullsLast().op("gin_trgm_ops"), - ), - index("idx_transactions_team_id_date_name").using( - "btree", - table.teamId.asc().nullsLast().op("date_ops"), - table.date.asc().nullsLast().op("date_ops"), - table.name.asc().nullsLast().op("uuid_ops"), - ), - index("idx_transactions_team_id_name").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - table.name.asc().nullsLast().op("uuid_ops"), - ), - index("idx_trgm_name").using( - "gist", - table.name.asc().nullsLast().op("gist_trgm_ops"), - ), - index("transactions_assigned_id_idx").using( - "btree", - table.assignedId.asc().nullsLast().op("uuid_ops"), - ), - index("transactions_bank_account_id_idx").using( - "btree", - table.bankAccountId.asc().nullsLast().op("uuid_ops"), - ), - index("transactions_category_slug_idx").using( - "btree", - table.categorySlug.asc().nullsLast().op("text_ops"), - ), - index( - "transactions_team_id_date_currency_bank_account_id_category_idx", - ).using( - "btree", - table.teamId.asc().nullsLast().op("enum_ops"), - table.date.asc().nullsLast().op("date_ops"), - table.currency.asc().nullsLast().op("text_ops"), - table.bankAccountId.asc().nullsLast().op("date_ops"), - ), - index("transactions_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.assignedId], - foreignColumns: [users.id], - name: "public_transactions_assigned_id_fkey", - }).onDelete("set null"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "public_transactions_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.bankAccountId], - foreignColumns: [bankAccounts.id], - name: "transactions_bank_account_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId, table.categorySlug], - foreignColumns: [ - transactionCategories.teamId, - transactionCategories.slug, - ], - name: "transactions_category_slug_team_id_fkey", - }), - unique("transactions_internal_id_key").on(table.internalId), - pgPolicy("Transactions can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Transactions can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("Transactions can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Transactions can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const trackerEntries = pgTable( - "tracker_entries", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - // You can use { mode: "bigint" } if numbers are exceeding js number limitations - duration: bigint({ mode: "number" }), - projectId: uuid("project_id"), - start: timestamp({ withTimezone: true, mode: "string" }), - stop: timestamp({ withTimezone: true, mode: "string" }), - assignedId: uuid("assigned_id"), - teamId: uuid("team_id"), - description: text(), - rate: numericCasted({ precision: 10, scale: 2 }), - currency: text(), - billed: boolean().default(false), - date: date().defaultNow(), - }, - (table) => [ - index("tracker_entries_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.assignedId], - foreignColumns: [users.id], - name: "tracker_entries_assigned_id_fkey", - }).onDelete("set null"), - foreignKey({ - columns: [table.projectId], - foreignColumns: [trackerProjects.id], - name: "tracker_entries_project_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "tracker_entries_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Entries can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Entries can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["authenticated"], - }), - pgPolicy("Entries can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["authenticated"], - }), - pgPolicy("Entries can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["authenticated"], - }), - ], -); - -export const customerTags = pgTable( - "customer_tags", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - customerId: uuid("customer_id").notNull(), - teamId: uuid("team_id").notNull(), - tagId: uuid("tag_id").notNull(), - }, - (table) => [ - foreignKey({ - columns: [table.customerId], - foreignColumns: [customers.id], - name: "customer_tags_customer_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.tagId], - foreignColumns: [tags.id], - name: "customer_tags_tag_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "customer_tags_team_id_fkey", - }).onDelete("cascade"), - unique("unique_customer_tag").on(table.customerId, table.tagId), - pgPolicy("Tags can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const inboxAccounts = pgTable( - "inbox_accounts", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - email: text().notNull(), - accessToken: text("access_token").notNull(), - refreshToken: text("refresh_token").notNull(), - teamId: uuid("team_id").notNull(), - lastAccessed: timestamp("last_accessed", { - withTimezone: true, - mode: "string", - }).notNull(), - provider: inboxAccountProvidersEnum().notNull(), - externalId: text("external_id").notNull(), - expiryDate: timestamp("expiry_date", { - withTimezone: true, - mode: "string", - }).notNull(), - scheduleId: text("schedule_id"), - status: inboxAccountStatusEnum().default("connected").notNull(), - errorMessage: text("error_message"), - }, - (table) => [ - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "inbox_accounts_team_id_fkey", - }).onDelete("cascade"), - unique("inbox_accounts_email_key").on(table.email), - unique("inbox_accounts_external_id_key").on(table.externalId), - pgPolicy("Inbox accounts can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Inbox accounts can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Inbox accounts can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const bankAccounts = pgTable( - "bank_accounts", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - createdBy: uuid("created_by").notNull(), - teamId: uuid("team_id").notNull(), - name: text(), - currency: text(), - bankConnectionId: uuid("bank_connection_id"), - enabled: boolean().default(true).notNull(), - accountId: text("account_id").notNull(), - balance: numericCasted({ precision: 10, scale: 2 }).default(0), - manual: boolean().default(false), - type: accountTypeEnum(), - baseCurrency: text("base_currency"), - baseBalance: numericCasted({ precision: 10, scale: 2 }), - errorDetails: text("error_details"), - errorRetries: smallint("error_retries"), - accountReference: text("account_reference"), - }, - (table) => [ - index("bank_accounts_bank_connection_id_idx").using( - "btree", - table.bankConnectionId.asc().nullsLast().op("uuid_ops"), - ), - index("bank_accounts_created_by_idx").using( - "btree", - table.createdBy.asc().nullsLast().op("uuid_ops"), - ), - index("bank_accounts_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.bankConnectionId], - foreignColumns: [bankConnections.id], - name: "bank_accounts_bank_connection_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.createdBy], - foreignColumns: [users.id], - name: "bank_accounts_created_by_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "public_bank_accounts_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Bank Accounts can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Bank Accounts can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("Bank Accounts can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Bank Accounts can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const invoices = pgTable( - "invoices", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - updatedAt: timestamp("updated_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - dueDate: timestamp("due_date", { withTimezone: true, mode: "string" }), - invoiceNumber: text("invoice_number"), - customerId: uuid("customer_id"), - amount: numericCasted({ precision: 10, scale: 2 }), - currency: text(), - lineItems: jsonb("line_items"), - paymentDetails: jsonb("payment_details"), - customerDetails: jsonb("customer_details"), - companyDatails: jsonb("company_datails"), - note: text(), - internalNote: text("internal_note"), - teamId: uuid("team_id").notNull(), - paidAt: timestamp("paid_at", { withTimezone: true, mode: "string" }), - fts: tsvector("fts") - .notNull() - .generatedAlwaysAs( - (): SQL => sql` - to_tsvector( - 'english', - ( - (COALESCE((amount)::text, ''::text) || ' '::text) || COALESCE(invoice_number, ''::text) - ) - ) - `, - ), - vat: numericCasted({ precision: 10, scale: 2 }), - tax: numericCasted({ precision: 10, scale: 2 }), - url: text(), - filePath: text("file_path").array(), - status: invoiceStatusEnum().default("draft").notNull(), - viewedAt: timestamp("viewed_at", { withTimezone: true, mode: "string" }), - fromDetails: jsonb("from_details"), - issueDate: timestamp("issue_date", { withTimezone: true, mode: "string" }), - template: jsonb(), - noteDetails: jsonb("note_details"), - customerName: text("customer_name"), - token: text().default("").notNull(), - sentTo: text("sent_to"), - reminderSentAt: timestamp("reminder_sent_at", { - withTimezone: true, - mode: "string", - }), - discount: numericCasted({ precision: 10, scale: 2 }), - fileSize: bigint("file_size", { mode: "number" }), - userId: uuid("user_id"), - subtotal: numericCasted({ precision: 10, scale: 2 }), - topBlock: jsonb("top_block"), - bottomBlock: jsonb("bottom_block"), - sentAt: timestamp("sent_at", { withTimezone: true, mode: "string" }), - scheduledAt: timestamp("scheduled_at", { - withTimezone: true, - mode: "string", - }), - scheduledJobId: text("scheduled_job_id"), - }, - (table) => [ - index("invoices_created_at_idx").using( - "btree", - table.createdAt.asc().nullsLast().op("timestamptz_ops"), - ), - index("invoices_fts").using( - "gin", - table.fts.asc().nullsLast().op("tsvector_ops"), - ), - index("invoices_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "invoices_created_by_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.customerId], - foreignColumns: [customers.id], - name: "invoices_customer_id_fkey", - }).onDelete("set null"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "invoices_team_id_fkey", - }).onDelete("cascade"), - unique("invoices_scheduled_job_id_key").on(table.scheduledJobId), - pgPolicy("Invoices can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const customers = pgTable( - "customers", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - name: text().notNull(), - email: text().notNull(), - billingEmail: text(), - country: text(), - addressLine1: text("address_line_1"), - addressLine2: text("address_line_2"), - city: text(), - state: text(), - zip: text(), - note: text(), - teamId: uuid("team_id").defaultRandom().notNull(), - website: text(), - phone: text(), - vatNumber: text("vat_number"), - countryCode: text("country_code"), - token: text().default("").notNull(), - contact: text(), - fts: tsvector("fts") - .notNull() - .generatedAlwaysAs( - (): SQL => sql` - to_tsvector( - 'english'::regconfig, - COALESCE(name, ''::text) || ' ' || - COALESCE(contact, ''::text) || ' ' || - COALESCE(phone, ''::text) || ' ' || - COALESCE(email, ''::text) || ' ' || - COALESCE(address_line_1, ''::text) || ' ' || - COALESCE(address_line_2, ''::text) || ' ' || - COALESCE(city, ''::text) || ' ' || - COALESCE(state, ''::text) || ' ' || - COALESCE(zip, ''::text) || ' ' || - COALESCE(country, ''::text) - ) - `, - ), - }, - (table) => [ - index("customers_fts").using( - "gin", - table.fts.asc().nullsLast().op("tsvector_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "customers_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Customers can be handled by members of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const exchangeRates = pgTable( - "exchange_rates", - { - id: uuid().defaultRandom().primaryKey().notNull(), - base: text(), - rate: numericCasted({ precision: 10, scale: 2 }), - target: text(), - updatedAt: timestamp("updated_at", { withTimezone: true, mode: "string" }), - }, - (table) => [ - index("exchange_rates_base_target_idx").using( - "btree", - table.base.asc().nullsLast().op("text_ops"), - table.target.asc().nullsLast().op("text_ops"), - ), - unique("unique_rate").on(table.base, table.target), - pgPolicy("Enable read access for authenticated users", { - as: "permissive", - for: "select", - to: ["public"], - using: sql`true`, - }), - ], -); - -export const tags = pgTable( - "tags", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - teamId: uuid("team_id").notNull(), - name: text().notNull(), - }, - (table) => [ - index("tags_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "tags_team_id_fkey", - }).onDelete("cascade"), - unique("unique_tag_name").on(table.teamId, table.name), - pgPolicy("Tags can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const trackerReports = pgTable( - "tracker_reports", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - linkId: text("link_id"), - shortLink: text("short_link"), - teamId: uuid("team_id").defaultRandom(), - projectId: uuid("project_id").defaultRandom(), - createdBy: uuid("created_by"), - }, - (table) => [ - index("tracker_reports_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.createdBy], - foreignColumns: [users.id], - name: "public_tracker_reports_created_by_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.projectId], - foreignColumns: [trackerProjects.id], - name: "public_tracker_reports_project_id_fkey", - }) - .onUpdate("cascade") - .onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "tracker_reports_team_id_fkey", - }) - .onUpdate("cascade") - .onDelete("cascade"), - pgPolicy("Reports can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const invoiceComments = pgTable("invoice_comments", { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), -}); - -export const trackerProjectTags = pgTable( - "tracker_project_tags", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - trackerProjectId: uuid("tracker_project_id").notNull(), - tagId: uuid("tag_id").notNull(), - teamId: uuid("team_id").notNull(), - }, - (table) => [ - index("tracker_project_tags_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("tracker_project_tags_tracker_project_id_tag_id_team_id_idx").using( - "btree", - table.trackerProjectId.asc().nullsLast().op("uuid_ops"), - table.tagId.asc().nullsLast().op("uuid_ops"), - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.tagId], - foreignColumns: [tags.id], - name: "project_tags_tag_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.trackerProjectId], - foreignColumns: [trackerProjects.id], - name: "project_tags_tracker_project_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "tracker_project_tags_team_id_fkey", - }).onDelete("cascade"), - unique("unique_project_tag").on(table.trackerProjectId, table.tagId), - pgPolicy("Tags can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const reports = pgTable( - "reports", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - linkId: text("link_id"), - teamId: uuid("team_id"), - shortLink: text("short_link"), - from: timestamp({ withTimezone: true, mode: "string" }), - to: timestamp({ withTimezone: true, mode: "string" }), - type: reportTypesEnum(), - expireAt: timestamp("expire_at", { withTimezone: true, mode: "string" }), - currency: text(), - createdBy: uuid("created_by"), - }, - (table) => [ - index("reports_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.createdBy], - foreignColumns: [users.id], - name: "public_reports_created_by_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "reports_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Reports can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Reports can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("Reports can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Reports can be updated by member of team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const bankConnections = pgTable( - "bank_connections", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - institutionId: text("institution_id").notNull(), - expiresAt: timestamp("expires_at", { withTimezone: true, mode: "string" }), - teamId: uuid("team_id").notNull(), - name: text().notNull(), - logoUrl: text("logo_url"), - accessToken: text("access_token"), - enrollmentId: text("enrollment_id"), - provider: bankProvidersEnum().notNull(), - lastAccessed: timestamp("last_accessed", { - withTimezone: true, - mode: "string", - }), - referenceId: text("reference_id"), - status: connectionStatusEnum().default("connected"), - errorDetails: text("error_details"), - errorRetries: smallint("error_retries").default(sql`'0'`), - }, - (table) => [ - index("bank_connections_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "bank_connections_team_id_fkey", - }).onDelete("cascade"), - unique("unique_bank_connections").on(table.institutionId, table.teamId), - pgPolicy("Bank Connections can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Bank Connections can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("Bank Connections can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Bank Connections can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const userInvites = pgTable( - "user_invites", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - teamId: uuid("team_id"), - email: text(), - role: teamRolesEnum(), - code: text().default("nanoid(24)"), - invitedBy: uuid("invited_by"), - }, - (table) => [ - index("user_invites_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "public_user_invites_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.invitedBy], - foreignColumns: [users.id], - name: "user_invites_invited_by_fkey", - }).onDelete("cascade"), - unique("unique_team_invite").on(table.teamId, table.email), - unique("user_invites_code_key").on(table.code), - pgPolicy("Enable select for users based on email", { - as: "permissive", - for: "select", - to: ["public"], - using: sql`((auth.jwt() ->> 'email'::text) = email)`, - }), - pgPolicy("User Invites can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - }), - pgPolicy("User Invites can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("User Invites can be deleted by invited email", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("User Invites can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("User Invites can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const documentTags = pgTable( - "document_tags", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - name: text().notNull(), - slug: text().notNull(), - teamId: uuid("team_id").notNull(), - }, - (table) => [ - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "document_tags_team_id_fkey", - }).onDelete("cascade"), - unique("unique_slug_per_team").on(table.slug, table.teamId), - pgPolicy("Tags can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const transactionTags = pgTable( - "transaction_tags", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - teamId: uuid("team_id").notNull(), - tagId: uuid("tag_id").notNull(), - transactionId: uuid("transaction_id").notNull(), - }, - (table) => [ - index("transaction_tags_tag_id_idx").using( - "btree", - table.tagId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_tags_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_tags_transaction_id_tag_id_team_id_idx").using( - "btree", - table.transactionId.asc().nullsLast().op("uuid_ops"), - table.tagId.asc().nullsLast().op("uuid_ops"), - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.tagId], - foreignColumns: [tags.id], - name: "transaction_tags_tag_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "transaction_tags_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.transactionId], - foreignColumns: [transactions.id], - name: "transaction_tags_transaction_id_fkey", - }).onDelete("cascade"), - unique("unique_tag").on(table.tagId, table.transactionId), - pgPolicy("Transaction Tags can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const transactionAttachments = pgTable( - "transaction_attachments", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - type: text(), - transactionId: uuid("transaction_id"), - teamId: uuid("team_id"), - // You can use { mode: "bigint" } if numbers are exceeding js number limitations - size: bigint({ mode: "number" }), - name: text(), - path: text().array(), - }, - (table) => [ - index("transaction_attachments_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_attachments_transaction_id_idx").using( - "btree", - table.transactionId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "public_transaction_attachments_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.transactionId], - foreignColumns: [transactions.id], - name: "public_transaction_attachments_transaction_id_fkey", - }).onDelete("set null"), - pgPolicy("Transaction Attachments can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Transaction Attachments can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy( - "Transaction Attachments can be selected by a member of the team", - { as: "permissive", for: "select", to: ["public"] }, - ), - pgPolicy("Transaction Attachments can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const teams = pgTable( - "teams", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - name: text(), - logoUrl: text("logo_url"), - inboxId: text("inbox_id").default("generate_inbox(10)"), - email: text(), - inboxEmail: text("inbox_email"), - inboxForwarding: boolean("inbox_forwarding").default(true), - baseCurrency: text("base_currency"), - countryCode: text("country_code"), - documentClassification: boolean("document_classification").default(false), - flags: text().array(), - canceledAt: timestamp("canceled_at", { - withTimezone: true, - mode: "string", - }), - plan: plansEnum().default("trial").notNull(), - // subscriptionStatus: subscriptionStatusEnum("subscription_status"), - exportSettings: jsonb("export_settings"), - }, - (table) => [ - unique("teams_inbox_id_key").on(table.inboxId), - pgPolicy("Enable insert for authenticated users only", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`true`, - }), - pgPolicy("Invited users can select team if they are invited.", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Teams can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - pgPolicy("Teams can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Teams can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const documents = pgTable( - "documents", - { - id: uuid().defaultRandom().primaryKey().notNull(), - name: text(), - createdAt: timestamp("created_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - metadata: jsonb(), - pathTokens: text("path_tokens").array(), - teamId: uuid("team_id"), - parentId: text("parent_id"), - objectId: uuid("object_id"), - ownerId: uuid("owner_id"), - tag: text(), - title: text(), - body: text(), - fts: tsvector("fts") - .notNull() - .generatedAlwaysAs( - (): SQL => - sql`to_tsvector('english'::regconfig, ((title || ' '::text) || body))`, - ), - summary: text(), - content: text(), - date: date(), - language: text(), - processingStatus: - documentProcessingStatusEnum("processing_status").default("pending"), - ftsSimple: tsvector("fts_simple"), - ftsEnglish: tsvector("fts_english"), - ftsLanguage: tsvector("fts_language"), - }, - (table) => [ - index("documents_name_idx").using( - "btree", - table.name.asc().nullsLast().op("text_ops"), - ), - index("documents_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("documents_team_id_parent_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("text_ops"), - table.parentId.asc().nullsLast().op("text_ops"), - ), - index("idx_documents_fts_english").using( - "gin", - table.ftsEnglish.asc().nullsLast().op("tsvector_ops"), - ), - index("idx_documents_fts_language").using( - "gin", - table.ftsLanguage.asc().nullsLast().op("tsvector_ops"), - ), - index("idx_documents_fts_simple").using( - "gin", - table.ftsSimple.asc().nullsLast().op("tsvector_ops"), - ), - index("idx_gin_documents_title").using( - "gin", - table.title.asc().nullsLast().op("gin_trgm_ops"), - ), - index("idx_gin_documents_name").using( - "gin", - table.name.asc().nullsLast().op("gin_trgm_ops"), - ), - foreignKey({ - columns: [table.ownerId], - foreignColumns: [users.id], - name: "documents_created_by_fkey", - }).onDelete("set null"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "storage_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Documents can be deleted by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Documents can be selected by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - }), - pgPolicy("Documents can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - pgPolicy("Enable insert for authenticated users only", { - as: "permissive", - for: "insert", - to: ["authenticated"], - }), - ], -); - -export const apps = pgTable( - "apps", - { - id: uuid().defaultRandom().primaryKey().notNull(), - teamId: uuid("team_id").defaultRandom(), - config: jsonb(), - createdAt: timestamp("created_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - appId: text("app_id").notNull(), - createdBy: uuid("created_by").defaultRandom(), - settings: jsonb(), - }, - (table) => [ - foreignKey({ - columns: [table.createdBy], - foreignColumns: [users.id], - name: "apps_created_by_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "integrations_team_id_fkey", - }).onDelete("cascade"), - unique("unique_app_id_team_id").on(table.teamId, table.appId), - pgPolicy("Apps can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Apps can be inserted by a member of the team", { - as: "permissive", - for: "insert", - to: ["public"], - }), - pgPolicy("Apps can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Apps can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const invoiceTemplates = pgTable( - "invoice_templates", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - teamId: uuid("team_id").notNull(), - customerLabel: text("customer_label"), - fromLabel: text("from_label"), - invoiceNoLabel: text("invoice_no_label"), - issueDateLabel: text("issue_date_label"), - dueDateLabel: text("due_date_label"), - descriptionLabel: text("description_label"), - priceLabel: text("price_label"), - quantityLabel: text("quantity_label"), - totalLabel: text("total_label"), - vatLabel: text("vat_label"), - taxLabel: text("tax_label"), - paymentLabel: text("payment_label"), - noteLabel: text("note_label"), - logoUrl: text("logo_url"), - currency: text(), - paymentDetails: jsonb("payment_details"), - fromDetails: jsonb("from_details"), - noteDetails: jsonb("note_details"), - size: invoiceSizeEnum().default("a4"), - dateFormat: text("date_format"), - includeVat: boolean("include_vat"), - includeTax: boolean("include_tax"), - taxRate: numericCasted("tax_rate", { precision: 10, scale: 2 }), - deliveryType: invoiceDeliveryTypeEnum("delivery_type") - .default("create") - .notNull(), - discountLabel: text("discount_label"), - includeDiscount: boolean("include_discount"), - includeDecimals: boolean("include_decimals"), - includeQr: boolean("include_qr"), - totalSummaryLabel: text("total_summary_label"), - title: text(), - vatRate: numericCasted("vat_rate", { precision: 10, scale: 2 }), - includeUnits: boolean("include_units"), - subtotalLabel: text("subtotal_label"), - includePdf: boolean("include_pdf"), - sendCopy: boolean("send_copy"), - }, - (table) => [ - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "invoice_settings_team_id_fkey", - }).onDelete("cascade"), - unique("invoice_templates_team_id_key").on(table.teamId), - pgPolicy("Invoice templates can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const invoiceProducts = pgTable( - "invoice_products", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - updatedAt: timestamp("updated_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - teamId: uuid("team_id").notNull(), - createdBy: uuid("created_by"), - name: text().notNull(), - description: text(), - price: numericCasted({ precision: 10, scale: 2 }), - currency: text(), - unit: text(), - isActive: boolean().default(true).notNull(), - usageCount: integer("usage_count").default(0).notNull(), - lastUsedAt: timestamp("last_used_at", { - withTimezone: true, - mode: "string", - }), - // Full-text search for product names and descriptions - fts: tsvector("fts") - .notNull() - .generatedAlwaysAs( - (): SQL => sql` - to_tsvector( - 'english', - ( - (COALESCE(name, ''::text) || ' '::text) || COALESCE(description, ''::text) - ) - ) - `, - ), - }, - (table) => [ - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "invoice_products_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.createdBy], - foreignColumns: [users.id], - name: "invoice_products_created_by_fkey", - }).onDelete("set null"), - index("invoice_products_team_id_idx").on(table.teamId), - index("invoice_products_created_by_idx").on(table.createdBy), - index("invoice_products_fts_idx").using("gin", table.fts), - index("invoice_products_name_idx").on(table.name), - index("invoice_products_usage_count_idx").on(table.usageCount), - index("invoice_products_last_used_at_idx").on(table.lastUsedAt), - // Composite index for team + active status for fast filtering - index("invoice_products_team_active_idx").on(table.teamId, table.isActive), - // Unique constraint for upsert operations (team + name + currency + price combination) - unique("invoice_products_team_name_currency_price_unique").on( - table.teamId, - table.name, - table.currency, - table.price, - ), - pgPolicy("Enable read access for team members", { - as: "permissive", - for: "select", - to: ["public"], - using: sql`team_id = (select auth.jwt() ->> 'team_id')::uuid`, - }), - pgPolicy("Enable insert access for team members", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`team_id = (select auth.jwt() ->> 'team_id')::uuid`, - }), - pgPolicy("Enable update access for team members", { - as: "permissive", - for: "update", - to: ["public"], - using: sql`team_id = (select auth.jwt() ->> 'team_id')::uuid`, - }), - pgPolicy("Enable delete access for team members", { - as: "permissive", - for: "delete", - to: ["public"], - using: sql`team_id = (select auth.jwt() ->> 'team_id')::uuid`, - }), - ], -); - -export const transactionEnrichments = pgTable( - "transaction_enrichments", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - name: text(), - teamId: uuid("team_id"), - categorySlug: text("category_slug"), - system: boolean().default(false), - }, - (table) => [ - index("transaction_enrichments_category_slug_team_id_idx").using( - "btree", - table.categorySlug.asc().nullsLast().op("text_ops"), - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId, table.categorySlug], - foreignColumns: [ - transactionCategories.teamId, - transactionCategories.slug, - ], - name: "transaction_enrichments_category_slug_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "transaction_enrichments_team_id_fkey", - }).onDelete("cascade"), - unique("unique_team_name").on(table.name, table.teamId), - pgPolicy("Enable insert for authenticated users only", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`true`, - }), - pgPolicy("Enable update for authenticated users only", { - as: "permissive", - for: "update", - to: ["authenticated"], - }), - ], -); - -export const users = pgTable( - "users", - { - id: uuid().primaryKey().notNull(), - fullName: text("full_name"), - avatarUrl: text("avatar_url"), - email: text(), - teamId: uuid("team_id"), - createdAt: timestamp("created_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - locale: text().default("en"), - weekStartsOnMonday: boolean("week_starts_on_monday").default(false), - timezone: text(), - timezoneAutoSync: boolean("timezone_auto_sync").default(true), - timeFormat: numericCasted("time_format").default(24), - dateFormat: text("date_format"), - }, - (table) => [ - index("users_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.id], - foreignColumns: [table.id], - name: "users_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "users_team_id_fkey", - }).onDelete("set null"), - pgPolicy("Users can insert their own profile.", { - as: "permissive", - for: "insert", - to: ["public"], - withCheck: sql`(auth.uid() = id)`, - }), - pgPolicy("Users can select their own profile.", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Users can select users if they are in the same team", { - as: "permissive", - for: "select", - to: ["authenticated"], - }), - pgPolicy("Users can update own profile.", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const trackerProjects = pgTable( - "tracker_projects", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - teamId: uuid("team_id"), - rate: numericCasted({ precision: 10, scale: 2 }), - currency: text(), - status: trackerStatusEnum().default("in_progress").notNull(), - description: text(), - name: text().notNull(), - billable: boolean().default(false), - // You can use { mode: "bigint" } if numbers are exceeding js number limitations - estimate: bigint({ mode: "number" }), - customerId: uuid("customer_id"), - fts: tsvector("fts") - .notNull() - .generatedAlwaysAs( - (): SQL => sql` - to_tsvector( - 'english'::regconfig, - ( - (COALESCE(name, ''::text) || ' '::text) || COALESCE(description, ''::text) - ) - ) - `, - ), - }, - (table) => [ - index("tracker_projects_fts").using( - "gin", - table.fts.asc().nullsLast().op("tsvector_ops"), - ), - index("tracker_projects_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.customerId], - foreignColumns: [customers.id], - name: "tracker_projects_customer_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "tracker_projects_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Projects can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Projects can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["authenticated"], - }), - pgPolicy("Projects can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["authenticated"], - }), - pgPolicy("Projects can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["authenticated"], - }), - ], -); - -export const inbox = pgTable( - "inbox", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - teamId: uuid("team_id"), - filePath: text("file_path").array(), - fileName: text("file_name"), - transactionId: uuid("transaction_id"), - amount: numericCasted("amount", { precision: 10, scale: 2 }), - currency: text(), - contentType: text("content_type"), - // You can use { mode: "bigint" } if numbers are exceeding js number limitations - size: bigint({ mode: "number" }), - attachmentId: uuid("attachment_id"), - date: date(), - forwardedTo: text("forwarded_to"), - referenceId: text("reference_id"), - meta: json(), - status: inboxStatusEnum().default("new"), - website: text(), - displayName: text("display_name"), - fts: tsvector("fts") - .notNull() - .generatedAlwaysAs( - (): SQL => - sql`generate_inbox_fts(display_name, extract_product_names((meta -> 'products'::text)))`, - ), - type: inboxTypeEnum(), - description: text(), - baseAmount: numericCasted("base_amount", { precision: 10, scale: 2 }), - baseCurrency: text("base_currency"), - taxAmount: numericCasted("tax_amount", { precision: 10, scale: 2 }), - taxRate: numericCasted("tax_rate", { precision: 10, scale: 2 }), - taxType: text("tax_type"), - inboxAccountId: uuid("inbox_account_id"), - }, - (table) => [ - index("inbox_attachment_id_idx").using( - "btree", - table.attachmentId.asc().nullsLast().op("uuid_ops"), - ), - index("inbox_created_at_idx").using( - "btree", - table.createdAt.asc().nullsLast().op("timestamptz_ops"), - ), - index("inbox_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("inbox_transaction_id_idx").using( - "btree", - table.transactionId.asc().nullsLast().op("uuid_ops"), - ), - index("inbox_inbox_account_id_idx").using( - "btree", - table.inboxAccountId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.attachmentId], - foreignColumns: [transactionAttachments.id], - name: "inbox_attachment_id_fkey", - }).onDelete("set null"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "public_inbox_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.transactionId], - foreignColumns: [transactions.id], - name: "public_inbox_transaction_id_fkey", - }).onDelete("set null"), - foreignKey({ - columns: [table.inboxAccountId], - foreignColumns: [inboxAccounts.id], - name: "inbox_inbox_account_id_fkey", - }).onDelete("set null"), - unique("inbox_reference_id_key").on(table.referenceId), - pgPolicy("Inbox can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Inbox can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["public"], - }), - pgPolicy("Inbox can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["public"], - }), - ], -); - -export const transactionEmbeddings = pgTable( - "transaction_embeddings", - { - id: uuid().defaultRandom().primaryKey().notNull(), - transactionId: uuid("transaction_id").notNull(), - teamId: uuid("team_id").notNull(), - embedding: vector("embedding", { dimensions: 768 }), - sourceText: text("source_text").notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - model: text("model").notNull().default("gemini-embedding-001"), - }, - (table) => [ - index("transaction_embeddings_transaction_id_idx").using( - "btree", - table.transactionId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_embeddings_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - // Vector similarity index for fast cosine similarity searches - index("transaction_embeddings_vector_idx").using( - "hnsw", - table.embedding.op("vector_cosine_ops"), - ), - foreignKey({ - columns: [table.transactionId], - foreignColumns: [transactions.id], - name: "transaction_embeddings_transaction_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "transaction_embeddings_team_id_fkey", - }).onDelete("cascade"), - unique("transaction_embeddings_unique").on(table.transactionId), - ], -); - -export const inboxEmbeddings = pgTable( - "inbox_embeddings", - { - id: uuid().defaultRandom().primaryKey().notNull(), - inboxId: uuid("inbox_id").notNull(), - teamId: uuid("team_id").notNull(), - embedding: vector("embedding", { dimensions: 768 }), - sourceText: text("source_text").notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - model: text("model").notNull().default("gemini-embedding-001"), - }, - (table) => [ - index("inbox_embeddings_inbox_id_idx").using( - "btree", - table.inboxId.asc().nullsLast().op("uuid_ops"), - ), - index("inbox_embeddings_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - // Vector similarity index for fast cosine similarity searches - index("inbox_embeddings_vector_idx").using( - "hnsw", - table.embedding.op("vector_cosine_ops"), - ), - foreignKey({ - columns: [table.inboxId], - foreignColumns: [inbox.id], - name: "inbox_embeddings_inbox_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "inbox_embeddings_team_id_fkey", - }).onDelete("cascade"), - unique("inbox_embeddings_unique").on(table.inboxId), - ], -); - -export const transactionMatchSuggestions = pgTable( - "transaction_match_suggestions", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - - // Core relationship - teamId: uuid("team_id").notNull(), - inboxId: uuid("inbox_id").notNull(), - transactionId: uuid("transaction_id").notNull(), - - // Match scores for transparency - confidenceScore: numericCasted("confidence_score", { - precision: 4, - scale: 3, - }).notNull(), - amountScore: numericCasted("amount_score", { precision: 4, scale: 3 }), - currencyScore: numericCasted("currency_score", { precision: 4, scale: 3 }), - dateScore: numericCasted("date_score", { precision: 4, scale: 3 }), - embeddingScore: numericCasted("embedding_score", { - precision: 4, - scale: 3, - }), - nameScore: numericCasted("name_score", { precision: 4, scale: 3 }), - - // Match context - matchType: text("match_type").notNull(), // 'auto_matched', 'high_confidence', 'suggested' - matchDetails: jsonb("match_details"), - - // User interaction tracking - status: text("status").default("pending").notNull(), // 'pending', 'confirmed', 'declined', 'expired', 'unmatched' - userActionAt: timestamp("user_action_at", { - withTimezone: true, - mode: "string", - }), - userId: uuid("user_id"), - }, - (table) => [ - index("transaction_match_suggestions_inbox_id_idx").using( - "btree", - table.inboxId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_match_suggestions_transaction_id_idx").using( - "btree", - table.transactionId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_match_suggestions_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_match_suggestions_status_idx").using( - "btree", - table.status.asc().nullsLast().op("text_ops"), - ), - index("transaction_match_suggestions_confidence_idx").using( - "btree", - table.confidenceScore.desc().nullsLast(), - ), - index("transaction_match_suggestions_lookup_idx").using( - "btree", - table.transactionId.asc().nullsLast().op("uuid_ops"), - table.teamId.asc().nullsLast().op("uuid_ops"), - table.status.asc().nullsLast().op("text_ops"), - ), - foreignKey({ - columns: [table.inboxId], - foreignColumns: [inbox.id], - name: "transaction_match_suggestions_inbox_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.transactionId], - foreignColumns: [transactions.id], - name: "transaction_match_suggestions_transaction_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "transaction_match_suggestions_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "transaction_match_suggestions_user_id_fkey", - }).onDelete("set null"), - unique("transaction_match_suggestions_unique").on( - table.inboxId, - table.transactionId, - ), - ], -); - -export const documentTagAssignments = pgTable( - "document_tag_assignments", - { - documentId: uuid("document_id").notNull(), - tagId: uuid("tag_id").notNull(), - teamId: uuid("team_id").notNull(), - }, - (table) => [ - index("idx_document_tag_assignments_document_id").using( - "btree", - table.documentId.asc().nullsLast().op("uuid_ops"), - ), - index("idx_document_tag_assignments_tag_id").using( - "btree", - table.tagId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.documentId], - foreignColumns: [documents.id], - name: "document_tag_assignments_document_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.tagId], - foreignColumns: [documentTags.id], - name: "document_tag_assignments_tag_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "document_tag_assignments_team_id_fkey", - }).onDelete("cascade"), - primaryKey({ - columns: [table.documentId, table.tagId], - name: "document_tag_assignments_pkey", - }), - unique("document_tag_assignments_unique").on(table.documentId, table.tagId), - pgPolicy("Tags can be handled by a member of the team", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const usersOnTeam = pgTable( - "users_on_team", - { - userId: uuid("user_id").notNull(), - teamId: uuid("team_id").notNull(), - id: uuid().defaultRandom().notNull(), - role: teamRolesEnum(), - createdAt: timestamp("created_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - }, - (table) => [ - index("users_on_team_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("users_on_team_user_id_idx").using( - "btree", - table.userId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "users_on_team_team_id_fkey", - }) - .onUpdate("cascade") - .onDelete("cascade"), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "users_on_team_user_id_fkey", - }).onDelete("cascade"), - primaryKey({ - columns: [table.userId, table.teamId, table.id], - name: "members_pkey", - }), - pgPolicy("Enable insert for authenticated users only", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`true`, - }), - pgPolicy("Enable updates for users on team", { - as: "permissive", - for: "update", - to: ["authenticated"], - }), - pgPolicy("Select for current user teams", { - as: "permissive", - for: "select", - to: ["authenticated"], - }), - pgPolicy("Users on team can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["public"], - }), - ], -); - -export const transactionCategories = pgTable( - "transaction_categories", - { - id: uuid().defaultRandom().notNull(), - name: text().notNull(), - teamId: uuid("team_id").notNull(), - color: text(), - createdAt: timestamp("created_at", { - withTimezone: true, - mode: "string", - }).defaultNow(), - system: boolean().default(false), - slug: text(), // Generated in database - taxRate: numericCasted("tax_rate", { precision: 10, scale: 2 }), - taxType: text("tax_type"), - taxReportingCode: text("tax_reporting_code"), - excluded: boolean("excluded").default(false), - description: text(), - parentId: uuid("parent_id"), - }, - (table) => [ - index("transaction_categories_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("transaction_categories_parent_id_idx").using( - "btree", - table.parentId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "transaction_categories_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.parentId], - foreignColumns: [table.id], - name: "transaction_categories_parent_id_fkey", - }).onDelete("set null"), - primaryKey({ - columns: [table.teamId, table.slug], - name: "transaction_categories_pkey", - }), - unique("unique_team_slug").on(table.teamId, table.slug), - pgPolicy("Users on team can manage categories", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const usersInAuth = pgTable( - "auth.users", - { - instanceId: uuid("instance_id"), - id: uuid("id").notNull(), - aud: varchar("aud", { length: 255 }), - role: varchar("role", { length: 255 }), - email: varchar("email", { length: 255 }), - encryptedPassword: varchar("encrypted_password", { length: 255 }), - emailConfirmedAt: timestamp("email_confirmed_at", { withTimezone: true }), - invitedAt: timestamp("invited_at", { withTimezone: true }), - confirmationToken: varchar("confirmation_token", { length: 255 }), - confirmationSentAt: timestamp("confirmation_sent_at", { - withTimezone: true, - }), - recoveryToken: varchar("recovery_token", { length: 255 }), - recoverySentAt: timestamp("recovery_sent_at", { withTimezone: true }), - emailChangeTokenNew: varchar("email_change_token_new", { length: 255 }), - emailChange: varchar("email_change", { length: 255 }), - emailChangeSentAt: timestamp("email_change_sent_at", { - withTimezone: true, - }), - lastSignInAt: timestamp("last_sign_in_at", { withTimezone: true }), - rawAppMetaData: jsonb("raw_app_meta_data"), - rawUserMetaData: jsonb("raw_user_meta_data"), - isSuperAdmin: boolean("is_super_admin"), - createdAt: timestamp("created_at", { withTimezone: true }), - updatedAt: timestamp("updated_at", { withTimezone: true }), - phone: text("phone").default(sql`null::character varying`), - phoneConfirmedAt: timestamp("phone_confirmed_at", { withTimezone: true }), - phoneChange: text("phone_change").default(sql`''::character varying`), - phoneChangeToken: varchar("phone_change_token", { length: 255 }).default( - sql`''::character varying`, - ), - phoneChangeSentAt: timestamp("phone_change_sent_at", { - withTimezone: true, - }), - // Drizzle ORM does not support .stored() for generated columns, so we omit it - confirmedAt: timestamp("confirmed_at", { - withTimezone: true, - mode: "string", - }).generatedAlwaysAs(sql`LEAST(email_confirmed_at, phone_confirmed_at)`), - emailChangeTokenCurrent: varchar("email_change_token_current", { - length: 255, - }).default(sql`''::character varying`), - emailChangeConfirmStatus: smallint("email_change_confirm_status").default( - 0, - ), - bannedUntil: timestamp("banned_until", { withTimezone: true }), - reauthenticationToken: varchar("reauthentication_token", { - length: 255, - }).default(sql`''::character varying`), - reauthenticationSentAt: timestamp("reauthentication_sent_at", { - withTimezone: true, - }), - isSsoUser: boolean("is_sso_user").notNull().default(false), - deletedAt: timestamp("deleted_at", { withTimezone: true }), - isAnonymous: boolean("is_anonymous").notNull().default(false), - }, - (table) => [ - primaryKey({ columns: [table.id], name: "users_pkey" }), - unique("users_phone_key").on(table.phone), - unique("confirmation_token_idx").on(table.confirmationToken), - unique("email_change_token_current_idx").on(table.emailChangeTokenCurrent), - unique("email_change_token_new_idx").on(table.emailChangeTokenNew), - unique("reauthentication_token_idx").on(table.reauthenticationToken), - unique("recovery_token_idx").on(table.recoveryToken), - unique("users_email_partial_key").on(table.email), - index("users_instance_id_email_idx").on( - table.instanceId, - sql`lower((email)::text)`, - ), - index("users_instance_id_idx").on(table.instanceId), - index("users_is_anonymous_idx").on(table.isAnonymous), - // Check constraint for email_change_confirm_status - { - kind: "check", - name: "users_email_change_confirm_status_check", - expression: sql`((email_change_confirm_status >= 0) AND (email_change_confirm_status <= 2))`, - }, - ], -); - -export const shortLinks = pgTable( - "short_links", - { - id: uuid().defaultRandom().primaryKey().notNull(), - shortId: text("short_id").notNull(), - url: text().notNull(), - type: text("type"), - size: numericCasted("size", { precision: 10, scale: 2 }), - mimeType: text("mime_type"), - fileName: text("file_name"), - teamId: uuid("team_id").notNull(), - userId: uuid("user_id").notNull(), - expiresAt: timestamp("expires_at", { withTimezone: true, mode: "string" }), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - }, - (table) => [ - index("short_links_short_id_idx").using( - "btree", - table.shortId.asc().nullsLast().op("text_ops"), - ), - index("short_links_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("short_links_user_id_idx").using( - "btree", - table.userId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "short_links_user_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "short_links_team_id_fkey", - }).onDelete("cascade"), - unique("short_links_short_id_unique").on(table.shortId), - pgPolicy("Short links can be created by a member of the team", { - as: "permissive", - for: "insert", - to: ["authenticated"], - withCheck: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Short links can be selected by a member of the team", { - as: "permissive", - for: "select", - to: ["authenticated"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Short links can be updated by a member of the team", { - as: "permissive", - for: "update", - to: ["authenticated"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - pgPolicy("Short links can be deleted by a member of the team", { - as: "permissive", - for: "delete", - to: ["authenticated"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -export const apiKeys = pgTable( - "api_keys", - { - id: uuid("id").notNull().defaultRandom().primaryKey(), - keyEncrypted: text("key_encrypted").notNull(), - name: text("name").notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .notNull() - .defaultNow(), - userId: uuid("user_id").notNull(), - teamId: uuid("team_id").notNull(), - keyHash: text("key_hash"), - scopes: text("scopes").array().notNull().default(sql`'{}'::text[]`), - lastUsedAt: timestamp("last_used_at", { - withTimezone: true, - mode: "string", - }), - }, - (table) => [ - index("api_keys_key_idx").using( - "btree", - table.keyHash.asc().nullsLast().op("text_ops"), - ), - index("api_keys_user_id_idx").using( - "btree", - table.userId.asc().nullsLast().op("uuid_ops"), - ), - index("api_keys_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "api_keys_user_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "api_keys_team_id_fkey", - }).onDelete("cascade"), - unique("api_keys_key_unique").on(table.keyHash), - ], + ["confirm_account", "password_reset", "unlock_account", "delete_account"], ); export const sources = pgTable( @@ -2531,7 +87,9 @@ export const sources = pgTable( name: varchar("name", { length: 255 }).notNull(), displayName: varchar("display_name", { length: 255 }), description: varchar("description", { length: 1024 }), - createdAt: timestamp("created_at", { mode: "string" }).defaultNow().notNull(), + createdAt: timestamp("created_at", { mode: "string" }) + .defaultNow() + .notNull(), updatedAt: timestamp("updated_at", { mode: "string" }), bias: biasEnum("bias").notNull().default("neutral"), reliability: reliabilityEnum("reliability").notNull().default("reliable"), @@ -2540,11 +98,13 @@ export const sources = pgTable( (table) => [ uniqueIndex("unq_source_name").using( "btree", - sql`lower(${table.name})`, + sql`lower + (${table.name})`, ), uniqueIndex("unq_source_url").using( "btree", - sql`lower(${table.url})`, + sql`lower + (${table.url})`, ), ], ); @@ -2562,12 +122,10 @@ export const articles = pgTable( metadata: jsonb("metadata"), tokenStatistics: jsonb("token_statistics"), image: varchar("image", { length: 1024 }).generatedAlwaysAs( - sql`(metadata->>'image')`, - { stored: true }, + () => sql`(metadata->>'image')`, ), excerpt: varchar("excerpt", { length: 255 }).generatedAlwaysAs( - sql`((left(body, 200) || '...'))`, - { stored: true }, + () => sql`((left(body, 200) || '...'))`, ), publishedAt: timestamp("published_at", { mode: "string" }).notNull(), crawledAt: timestamp("crawled_at", { mode: "string" }).notNull(), @@ -2578,22 +136,23 @@ export const articles = pgTable( transparency: transparencyEnum("transparency").notNull().default("medium"), readingTime: integer("reading_time").default(1), tsv: tsvector("tsv").generatedAlwaysAs( - sql`(setweight(to_tsvector('french', coalesce(title, '')), 'A') || setweight(to_tsvector('french', coalesce(body, '')), 'B'))`, - { stored: true }, + () => sql`( + setweight(to_tsvector('french', coalesce(title, '')), 'A') + || setweight(to_tsvector('french', coalesce(body, '')), 'B') + )`, ), }, (table) => [ index("article_source_id_idx").on(table.sourceId), - index("idx_article_published_at") - .using("btree", table.publishedAt.desc()), - index("idx_article_published_id") - .using("btree", table.publishedAt.desc(), table.id.desc()), + index("idx_article_published_at").using("btree", table.publishedAt.desc()), + index("idx_article_published_id").using( + "btree", + table.publishedAt.desc(), + table.id.desc(), + ), unique("unq_article_hash").on(table.hash), index("gin_article_tsv").using("gin", table.tsv), - index("gin_article_link_trgm").using( - "gin", - table.link.op("gin_trgm_ops"), - ), + index("gin_article_link_trgm").using("gin", table.link.op("gin_trgm_ops")), index("gin_article_title_trgm").using( "gin", table.title.op("gin_trgm_ops"), @@ -2617,7 +176,7 @@ export const articles = pgTable( ], ); -export const appUsers = pgTable( +export const users = pgTable( "user", { id: uuid("id").notNull().defaultRandom().primaryKey(), @@ -2631,10 +190,7 @@ export const appUsers = pgTable( roles: jsonb("roles").notNull(), }, (table) => [ - uniqueIndex("unq_user_email").using( - "btree", - sql`lower(${table.email})`, - ), + uniqueIndex("unq_user_email").using("btree", sql`lower (${table.email})`), { kind: "check", name: "chk_user_roles_array", @@ -2663,7 +219,7 @@ export const bookmarks = pgTable( ), foreignKey({ columns: [table.userId], - foreignColumns: [appUsers.id], + foreignColumns: [users.id], name: "bookmark_user_id_fkey", }).onDelete("cascade"), ], @@ -2716,7 +272,7 @@ export const comments = pgTable( ), foreignKey({ columns: [table.userId], - foreignColumns: [appUsers.id], + foreignColumns: [users.id], name: "comment_user_id_fkey", }).onDelete("cascade"), foreignKey({ @@ -2745,7 +301,7 @@ export const followedSources = pgTable( ), foreignKey({ columns: [table.followerId], - foreignColumns: [appUsers.id], + foreignColumns: [users.id], name: "followed_source_follower_id_fkey", }).onDelete("cascade"), foreignKey({ @@ -2771,7 +327,7 @@ export const loginAttempts = pgTable( ), foreignKey({ columns: [table.userId], - foreignColumns: [appUsers.id], + foreignColumns: [users.id], name: "login_attempt_user_id_fkey", }).onDelete("cascade"), ], @@ -2803,7 +359,7 @@ export const loginHistories = pgTable( index("login_history_ip_address_idx").on(table.ipAddress), foreignKey({ columns: [table.userId], - foreignColumns: [appUsers.id], + foreignColumns: [users.id], name: "login_history_user_id_fkey", }).onDelete("cascade"), ], @@ -2842,7 +398,7 @@ export const verificationTokens = pgTable( .where(sql`token IS NOT NULL`), foreignKey({ columns: [table.userId], - foreignColumns: [appUsers.id], + foreignColumns: [users.id], name: "verification_token_user_id_fkey", }).onDelete("cascade"), ], @@ -2864,7 +420,7 @@ export const articlesRelations = relations(articles, ({ one, many }) => ({ comments: many(comments), })); -export const appUsersRelations = relations(appUsers, ({ many }) => ({ +export const appUsersRelations = relations(users, ({ many }) => ({ bookmarks: many(bookmarks), comments: many(comments), loginAttempts: many(loginAttempts), @@ -2874,9 +430,9 @@ export const appUsersRelations = relations(appUsers, ({ many }) => ({ })); export const bookmarksRelations = relations(bookmarks, ({ one, many }) => ({ - user: one(appUsers, { + user: one(users, { fields: [bookmarks.userId], - references: [appUsers.id], + references: [users.id], }), articles: many(bookmarkArticles), })); @@ -2900,18 +456,18 @@ export const commentsRelations = relations(comments, ({ one }) => ({ fields: [comments.articleId], references: [articles.id], }), - user: one(appUsers, { + user: one(users, { fields: [comments.userId], - references: [appUsers.id], + references: [users.id], }), })); export const followedSourcesRelations = relations( followedSources, ({ one }) => ({ - follower: one(appUsers, { + follower: one(users, { fields: [followedSources.followerId], - references: [appUsers.id], + references: [users.id], }), source: one(sources, { fields: [followedSources.sourceId], @@ -2920,821 +476,26 @@ export const followedSourcesRelations = relations( }), ); -export const loginAttemptsRelations = relations( - loginAttempts, - ({ one }) => ({ - user: one(appUsers, { - fields: [loginAttempts.userId], - references: [appUsers.id], - }), +export const loginAttemptsRelations = relations(loginAttempts, ({ one }) => ({ + user: one(users, { + fields: [loginAttempts.userId], + references: [users.id], }), -); +})); -export const loginHistoriesRelations = relations( - loginHistories, - ({ one }) => ({ - user: one(appUsers, { - fields: [loginHistories.userId], - references: [appUsers.id], - }), +export const loginHistoriesRelations = relations(loginHistories, ({ one }) => ({ + user: one(users, { + fields: [loginHistories.userId], + references: [users.id], }), -); +})); export const verificationTokensRelations = relations( verificationTokens, ({ one }) => ({ - user: one(appUsers, { + user: one(users, { fields: [verificationTokens.userId], - references: [appUsers.id], - }), - }), -); -// OAuth Applications -export const oauthApplications = pgTable( - "oauth_applications", - { - id: uuid("id").notNull().defaultRandom().primaryKey(), - name: text("name").notNull(), - slug: text("slug").notNull().unique(), - description: text("description"), - overview: text("overview"), - developerName: text("developer_name"), - logoUrl: text("logo_url"), - website: text("website"), - installUrl: text("install_url"), - screenshots: text("screenshots").array().default(sql`'{}'::text[]`), - redirectUris: text("redirect_uris").array().notNull(), - clientId: text("client_id").notNull().unique(), - clientSecret: text("client_secret").notNull(), - scopes: text("scopes").array().notNull().default(sql`'{}'::text[]`), - teamId: uuid("team_id").notNull(), - createdBy: uuid("created_by").notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .notNull() - .defaultNow(), - updatedAt: timestamp("updated_at", { withTimezone: true, mode: "string" }) - .notNull() - .defaultNow(), - isPublic: boolean("is_public").default(false), - active: boolean("active").default(true), - status: text("status", { - enum: ["draft", "pending", "approved", "rejected"], - }).default("draft"), - }, - (table) => [ - index("oauth_applications_team_id_idx").using( - "btree", - table.teamId.asc().nullsLast().op("uuid_ops"), - ), - index("oauth_applications_client_id_idx").using( - "btree", - table.clientId.asc().nullsLast().op("text_ops"), - ), - index("oauth_applications_slug_idx").using( - "btree", - table.slug.asc().nullsLast().op("text_ops"), - ), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "oauth_applications_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.createdBy], - foreignColumns: [users.id], - name: "oauth_applications_created_by_fkey", - }).onDelete("cascade"), - pgPolicy("OAuth applications can be managed by team members", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(team_id IN ( SELECT private.get_teams_for_authenticated_user() AS get_teams_for_authenticated_user))`, - }), - ], -); - -// OAuth Authorization Codes -export const oauthAuthorizationCodes = pgTable( - "oauth_authorization_codes", - { - id: uuid("id").notNull().defaultRandom().primaryKey(), - code: text("code").notNull().unique(), - applicationId: uuid("application_id").notNull(), - userId: uuid("user_id").notNull(), - teamId: uuid("team_id").notNull(), - scopes: text("scopes").array().notNull(), - redirectUri: text("redirect_uri").notNull(), - expiresAt: timestamp("expires_at", { - withTimezone: true, - mode: "string", - }).notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .notNull() - .defaultNow(), - used: boolean("used").default(false), - codeChallenge: text("code_challenge"), - codeChallengeMethod: text("code_challenge_method"), - }, - (table) => [ - index("oauth_authorization_codes_code_idx").using( - "btree", - table.code.asc().nullsLast().op("text_ops"), - ), - index("oauth_authorization_codes_application_id_idx").using( - "btree", - table.applicationId.asc().nullsLast().op("uuid_ops"), - ), - index("oauth_authorization_codes_user_id_idx").using( - "btree", - table.userId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.applicationId], - foreignColumns: [oauthApplications.id], - name: "oauth_authorization_codes_application_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "oauth_authorization_codes_user_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "oauth_authorization_codes_team_id_fkey", - }).onDelete("cascade"), - ], -); - -// OAuth Access Tokens -export const oauthAccessTokens = pgTable( - "oauth_access_tokens", - { - id: uuid("id").notNull().defaultRandom().primaryKey(), - token: text("token").notNull().unique(), - refreshToken: text("refresh_token").unique(), - applicationId: uuid("application_id").notNull(), - userId: uuid("user_id").notNull(), - teamId: uuid("team_id").notNull(), - scopes: text("scopes").array().notNull(), - expiresAt: timestamp("expires_at", { - withTimezone: true, - mode: "string", - }).notNull(), - refreshTokenExpiresAt: timestamp("refresh_token_expires_at", { - withTimezone: true, - mode: "string", - }), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .notNull() - .defaultNow(), - lastUsedAt: timestamp("last_used_at", { - withTimezone: true, - mode: "string", - }), - revoked: boolean("revoked").default(false), - revokedAt: timestamp("revoked_at", { withTimezone: true, mode: "string" }), - }, - (table) => [ - index("oauth_access_tokens_token_idx").using( - "btree", - table.token.asc().nullsLast().op("text_ops"), - ), - index("oauth_access_tokens_refresh_token_idx").using( - "btree", - table.refreshToken.asc().nullsLast().op("text_ops"), - ), - index("oauth_access_tokens_application_id_idx").using( - "btree", - table.applicationId.asc().nullsLast().op("uuid_ops"), - ), - index("oauth_access_tokens_user_id_idx").using( - "btree", - table.userId.asc().nullsLast().op("uuid_ops"), - ), - foreignKey({ - columns: [table.applicationId], - foreignColumns: [oauthApplications.id], - name: "oauth_access_tokens_application_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "oauth_access_tokens_user_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "oauth_access_tokens_team_id_fkey", - }).onDelete("cascade"), - ], -); - -export const transactionsRelations = relations( - transactions, - ({ one, many }) => ({ - user: one(users, { - fields: [transactions.assignedId], references: [users.id], }), - team: one(teams, { - fields: [transactions.teamId], - references: [teams.id], - }), - bankAccount: one(bankAccounts, { - fields: [transactions.bankAccountId], - references: [bankAccounts.id], - }), - transactionCategory: one(transactionCategories, { - fields: [transactions.teamId], - references: [transactionCategories.teamId], - }), - transactionTags: many(transactionTags), - transactionAttachments: many(transactionAttachments), - inboxes: many(inbox), }), ); - -export const usersRelations = relations(users, ({ one, many }) => ({ - transactions: many(transactions), - trackerEntries: many(trackerEntries), - bankAccounts: many(bankAccounts), - invoices: many(invoices), - trackerReports: many(trackerReports), - reports: many(reports), - userInvites: many(userInvites), - documents: many(documents), - apps: many(apps), - apiKeys: many(apiKeys), - shortLinks: many(shortLinks), - oauthApplications: many(oauthApplications), - oauthAuthorizationCodes: many(oauthAuthorizationCodes), - oauthAccessTokens: many(oauthAccessTokens), - usersInAuth: one(usersInAuth, { - fields: [users.id], - references: [usersInAuth.id], - }), - team: one(teams, { - fields: [users.teamId], - references: [teams.id], - }), - usersOnTeams: many(usersOnTeam), -})); - -export const shortLinksRelations = relations(shortLinks, ({ one }) => ({ - user: one(users, { - fields: [shortLinks.userId], - references: [users.id], - }), - team: one(teams, { - fields: [shortLinks.teamId], - references: [teams.id], - }), -})); - -export const apiKeysRelations = relations(apiKeys, ({ one }) => ({ - user: one(users, { - fields: [apiKeys.userId], - references: [users.id], - }), - team: one(teams, { - fields: [apiKeys.teamId], - references: [teams.id], - }), -})); - -export const teamsRelations = relations(teams, ({ many }) => ({ - transactions: many(transactions), - trackerEntries: many(trackerEntries), - customerTags: many(customerTags), - inboxAccounts: many(inboxAccounts), - bankAccounts: many(bankAccounts), - invoices: many(invoices), - customers: many(customers), - tags: many(tags), - trackerReports: many(trackerReports), - trackerProjectTags: many(trackerProjectTags), - reports: many(reports), - bankConnections: many(bankConnections), - userInvites: many(userInvites), - documentTags: many(documentTags), - transactionTags: many(transactionTags), - transactionAttachments: many(transactionAttachments), - documents: many(documents), - apps: many(apps), - apiKeys: many(apiKeys), - shortLinks: many(shortLinks), - invoiceTemplates: many(invoiceTemplates), - transactionEnrichments: many(transactionEnrichments), - users: many(users), - trackerProjects: many(trackerProjects), - inboxes: many(inbox), - documentTagAssignments: many(documentTagAssignments), - usersOnTeams: many(usersOnTeam), - transactionCategories: many(transactionCategories), -})); - -export const bankAccountsRelations = relations( - bankAccounts, - ({ one, many }) => ({ - transactions: many(transactions), - bankConnection: one(bankConnections, { - fields: [bankAccounts.bankConnectionId], - references: [bankConnections.id], - }), - user: one(users, { - fields: [bankAccounts.createdBy], - references: [users.id], - }), - team: one(teams, { - fields: [bankAccounts.teamId], - references: [teams.id], - }), - }), -); - -export const transactionCategoriesRelations = relations( - transactionCategories, - ({ one, many }) => ({ - transactions: many(transactions), - transactionEnrichments: many(transactionEnrichments), - team: one(teams, { - fields: [transactionCategories.teamId], - references: [teams.id], - }), - parent: one(transactionCategories, { - fields: [transactionCategories.parentId], - references: [transactionCategories.id], - relationName: "parent_child", - }), - children: many(transactionCategories, { - relationName: "parent_child", - }), - }), -); - -export const trackerEntriesRelations = relations(trackerEntries, ({ one }) => ({ - user: one(users, { - fields: [trackerEntries.assignedId], - references: [users.id], - }), - trackerProject: one(trackerProjects, { - fields: [trackerEntries.projectId], - references: [trackerProjects.id], - }), - team: one(teams, { - fields: [trackerEntries.teamId], - references: [teams.id], - }), -})); - -export const trackerProjectsRelations = relations( - trackerProjects, - ({ one, many }) => ({ - trackerEntries: many(trackerEntries), - trackerReports: many(trackerReports), - trackerProjectTags: many(trackerProjectTags), - customer: one(customers, { - fields: [trackerProjects.customerId], - references: [customers.id], - }), - team: one(teams, { - fields: [trackerProjects.teamId], - references: [teams.id], - }), - }), -); - -export const customerTagsRelations = relations(customerTags, ({ one }) => ({ - customer: one(customers, { - fields: [customerTags.customerId], - references: [customers.id], - }), - tag: one(tags, { - fields: [customerTags.tagId], - references: [tags.id], - }), - team: one(teams, { - fields: [customerTags.teamId], - references: [teams.id], - }), -})); - -export const customersRelations = relations(customers, ({ one, many }) => ({ - customerTags: many(customerTags), - invoices: many(invoices), - team: one(teams, { - fields: [customers.teamId], - references: [teams.id], - }), - trackerProjects: many(trackerProjects), -})); - -export const tagsRelations = relations(tags, ({ one, many }) => ({ - customerTags: many(customerTags), - team: one(teams, { - fields: [tags.teamId], - references: [teams.id], - }), - trackerProjectTags: many(trackerProjectTags), - transactionTags: many(transactionTags), -})); - -export const inboxAccountsRelations = relations(inboxAccounts, ({ one }) => ({ - team: one(teams, { - fields: [inboxAccounts.teamId], - references: [teams.id], - }), -})); - -export const bankConnectionsRelations = relations( - bankConnections, - ({ one, many }) => ({ - bankAccounts: many(bankAccounts), - team: one(teams, { - fields: [bankConnections.teamId], - references: [teams.id], - }), - }), -); - -export const invoicesRelations = relations(invoices, ({ one }) => ({ - user: one(users, { - fields: [invoices.userId], - references: [users.id], - }), - customer: one(customers, { - fields: [invoices.customerId], - references: [customers.id], - }), - team: one(teams, { - fields: [invoices.teamId], - references: [teams.id], - }), -})); - -export const trackerReportsRelations = relations(trackerReports, ({ one }) => ({ - user: one(users, { - fields: [trackerReports.createdBy], - references: [users.id], - }), - trackerProject: one(trackerProjects, { - fields: [trackerReports.projectId], - references: [trackerProjects.id], - }), - team: one(teams, { - fields: [trackerReports.teamId], - references: [teams.id], - }), -})); - -export const trackerProjectTagsRelations = relations( - trackerProjectTags, - ({ one }) => ({ - tag: one(tags, { - fields: [trackerProjectTags.tagId], - references: [tags.id], - }), - trackerProject: one(trackerProjects, { - fields: [trackerProjectTags.trackerProjectId], - references: [trackerProjects.id], - }), - team: one(teams, { - fields: [trackerProjectTags.teamId], - references: [teams.id], - }), - }), -); - -export const reportsRelations = relations(reports, ({ one }) => ({ - user: one(users, { - fields: [reports.createdBy], - references: [users.id], - }), - team: one(teams, { - fields: [reports.teamId], - references: [teams.id], - }), -})); - -export const userInvitesRelations = relations(userInvites, ({ one }) => ({ - team: one(teams, { - fields: [userInvites.teamId], - references: [teams.id], - }), - user: one(users, { - fields: [userInvites.invitedBy], - references: [users.id], - }), -})); - -export const documentTagsRelations = relations( - documentTags, - ({ one, many }) => ({ - team: one(teams, { - fields: [documentTags.teamId], - references: [teams.id], - }), - documentTagAssignments: many(documentTagAssignments), - }), -); - -export const transactionTagsRelations = relations( - transactionTags, - ({ one }) => ({ - tag: one(tags, { - fields: [transactionTags.tagId], - references: [tags.id], - }), - team: one(teams, { - fields: [transactionTags.teamId], - references: [teams.id], - }), - transaction: one(transactions, { - fields: [transactionTags.transactionId], - references: [transactions.id], - }), - }), -); - -export const transactionAttachmentsRelations = relations( - transactionAttachments, - ({ one, many }) => ({ - team: one(teams, { - fields: [transactionAttachments.teamId], - references: [teams.id], - }), - transaction: one(transactions, { - fields: [transactionAttachments.transactionId], - references: [transactions.id], - }), - inboxes: many(inbox), - }), -); - -export const documentsRelations = relations(documents, ({ one, many }) => ({ - user: one(users, { - fields: [documents.ownerId], - references: [users.id], - }), - team: one(teams, { - fields: [documents.teamId], - references: [teams.id], - }), - documentTagAssignments: many(documentTagAssignments), -})); - -export const appsRelations = relations(apps, ({ one }) => ({ - user: one(users, { - fields: [apps.createdBy], - references: [users.id], - }), - team: one(teams, { - fields: [apps.teamId], - references: [teams.id], - }), -})); - -export const invoiceTemplatesRelations = relations( - invoiceTemplates, - ({ one }) => ({ - team: one(teams, { - fields: [invoiceTemplates.teamId], - references: [teams.id], - }), - }), -); - -export const transactionEnrichmentsRelations = relations( - transactionEnrichments, - ({ one }) => ({ - transactionCategory: one(transactionCategories, { - fields: [transactionEnrichments.teamId], - references: [transactionCategories.teamId], - }), - team: one(teams, { - fields: [transactionEnrichments.teamId], - references: [teams.id], - }), - }), -); - -export const usersInAuthRelations = relations(usersInAuth, ({ many }) => ({ - users: many(users), -})); - -export const inboxRelations = relations(inbox, ({ one }) => ({ - transactionAttachment: one(transactionAttachments, { - fields: [inbox.attachmentId], - references: [transactionAttachments.id], - }), - team: one(teams, { - fields: [inbox.teamId], - references: [teams.id], - }), - transaction: one(transactions, { - fields: [inbox.transactionId], - references: [transactions.id], - }), -})); - -export const documentTagAssignmentsRelations = relations( - documentTagAssignments, - ({ one }) => ({ - document: one(documents, { - fields: [documentTagAssignments.documentId], - references: [documents.id], - }), - documentTag: one(documentTags, { - fields: [documentTagAssignments.tagId], - references: [documentTags.id], - }), - team: one(teams, { - fields: [documentTagAssignments.teamId], - references: [teams.id], - }), - }), -); - -export const usersOnTeamRelations = relations(usersOnTeam, ({ one }) => ({ - team: one(teams, { - fields: [usersOnTeam.teamId], - references: [teams.id], - }), - user: one(users, { - fields: [usersOnTeam.userId], - references: [users.id], - }), -})); - -// OAuth Relations -export const oauthApplicationsRelations = relations( - oauthApplications, - ({ one, many }) => ({ - team: one(teams, { - fields: [oauthApplications.teamId], - references: [teams.id], - }), - createdBy: one(users, { - fields: [oauthApplications.createdBy], - references: [users.id], - }), - authorizationCodes: many(oauthAuthorizationCodes), - accessTokens: many(oauthAccessTokens), - }), -); - -export const oauthAuthorizationCodesRelations = relations( - oauthAuthorizationCodes, - ({ one }) => ({ - application: one(oauthApplications, { - fields: [oauthAuthorizationCodes.applicationId], - references: [oauthApplications.id], - }), - user: one(users, { - fields: [oauthAuthorizationCodes.userId], - references: [users.id], - }), - team: one(teams, { - fields: [oauthAuthorizationCodes.teamId], - references: [teams.id], - }), - }), -); - -export const oauthAccessTokensRelations = relations( - oauthAccessTokens, - ({ one }) => ({ - application: one(oauthApplications, { - fields: [oauthAccessTokens.applicationId], - references: [oauthApplications.id], - }), - user: one(users, { - fields: [oauthAccessTokens.userId], - references: [users.id], - }), - team: one(teams, { - fields: [oauthAccessTokens.teamId], - references: [teams.id], - }), - }), -); - -export const activities = pgTable( - "activities", - { - id: uuid().defaultRandom().primaryKey().notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - - // Core fields - teamId: uuid("team_id").notNull(), - userId: uuid("user_id"), - type: activityTypeEnum().notNull(), - priority: smallint().default(5), // 1-3 = notifications, 4-10 = insights only - - // Group related activities together (e.g., same business event across multiple users) - groupId: uuid("group_id"), - - // Source of the activity - source: activitySourceEnum().notNull(), - - // All the data - metadata: jsonb().notNull(), - - // Simple lifecycle (only for notifications) - status: activityStatusEnum().default("unread").notNull(), - - // Timestamp of last system use (e.g. insight generation, digest inclusion) - lastUsedAt: timestamp("last_used_at", { - withTimezone: true, - mode: "string", - }), - }, - (table) => [ - // Optimized indexes - index("activities_notifications_idx").using( - "btree", - table.teamId, - table.priority, - table.status, - table.createdAt.desc(), - ), - index("activities_insights_idx").using( - "btree", - table.teamId, - table.type, - table.source, - table.createdAt.desc(), - ), - index("activities_metadata_gin_idx").using("gin", table.metadata), - index("activities_group_id_idx").on(table.groupId), - index("activities_insights_group_idx").using( - "btree", - table.teamId, - table.groupId, - table.type, - table.createdAt.desc(), - ), - - // Foreign keys - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "activities_team_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "activities_user_id_fkey", - }).onDelete("set null"), - ], -); - -export const notificationSettings = pgTable( - "notification_settings", - { - id: uuid().defaultRandom().primaryKey().notNull(), - userId: uuid("user_id").notNull(), - teamId: uuid("team_id").notNull(), - notificationType: text("notification_type").notNull(), - channel: text("channel").notNull(), // 'in_app', 'email', 'push' - enabled: boolean().default(true).notNull(), - createdAt: timestamp("created_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - updatedAt: timestamp("updated_at", { withTimezone: true, mode: "string" }) - .defaultNow() - .notNull(), - }, - (table) => [ - unique("notification_settings_user_team_type_channel_key").on( - table.userId, - table.teamId, - table.notificationType, - table.channel, - ), - index("notification_settings_user_team_idx").on(table.userId, table.teamId), - index("notification_settings_type_channel_idx").on( - table.notificationType, - table.channel, - ), - foreignKey({ - columns: [table.userId], - foreignColumns: [users.id], - name: "notification_settings_user_id_fkey", - }).onDelete("cascade"), - foreignKey({ - columns: [table.teamId], - foreignColumns: [teams.id], - name: "notification_settings_team_id_fkey", - }).onDelete("cascade"), - pgPolicy("Users can manage their own notification settings", { - as: "permissive", - for: "all", - to: ["public"], - using: sql`(user_id = auth.uid())`, - }), - ], -); diff --git a/basango/packages/db/src/utils/api-keys.ts b/basango/packages/db/src/utils/api-keys.ts index 7504f0c..d20a6b1 100644 --- a/basango/packages/db/src/utils/api-keys.ts +++ b/basango/packages/db/src/utils/api-keys.ts @@ -7,14 +7,14 @@ import { randomBytes } from "node:crypto"; export function generateApiKey(): string { // Generate 32 random bytes and convert to hex const randomString = randomBytes(32).toString("hex"); - return `mid_${randomString}`; + return `basango_${randomString}`; } /** * Validates if a string is a valid API key format * @param key The key to validate - * @returns True if the key starts with 'mid-' and has the correct length + * @returns True if the key starts with 'basango_' and has the correct length */ export function isValidApiKeyFormat(key: string): boolean { - return key.startsWith("mid_") && key.length === 68; // mid_ (4) + 64 hex chars + return key.startsWith("basango_") && key.length === 68; // basango_ (8) + 64 hex chars } diff --git a/basango/packages/db/src/utils/health.ts b/basango/packages/db/src/utils/health.ts index e69de29..40da39b 100644 --- a/basango/packages/db/src/utils/health.ts +++ b/basango/packages/db/src/utils/health.ts @@ -0,0 +1,6 @@ +import { sql } from "drizzle-orm"; +import { db } from "@db/client"; + +export async function checkHealth() { + await db.execute(sql`SELECT 1`); +} diff --git a/basango/packages/db/src/utils/index.ts b/basango/packages/db/src/utils/index.ts new file mode 100644 index 0000000..f6feccc --- /dev/null +++ b/basango/packages/db/src/utils/index.ts @@ -0,0 +1,4 @@ +export * from "./api-keys"; +export * from "./health"; +export * from "./pagination"; +export * from "./search-query"; diff --git a/basango/packages/db/src/utils/pagination.ts b/basango/packages/db/src/utils/pagination.ts index 134c3fc..a8ff83e 100644 --- a/basango/packages/db/src/utils/pagination.ts +++ b/basango/packages/db/src/utils/pagination.ts @@ -32,13 +32,15 @@ const DEFAULT_LIMIT = 5; const MAX_LIMIT = 100; export function createPageState(request: PageRequest = {}): PageState { - const page = Number.isFinite(request.page) && (request.page ?? 0) > 0 - ? Math.trunc(request.page!) - : DEFAULT_PAGE; + const page = + Number.isFinite(request.page) && (request.page ?? 0) > 0 + ? Math.trunc(request.page!) + : DEFAULT_PAGE; - let limit = Number.isFinite(request.limit) && (request.limit ?? 0) > 0 - ? Math.trunc(request.limit!) - : DEFAULT_LIMIT; + let limit = + Number.isFinite(request.limit) && (request.limit ?? 0) > 0 + ? Math.trunc(request.limit!) + : DEFAULT_LIMIT; if (limit < DEFAULT_LIMIT) { limit = DEFAULT_LIMIT; diff --git a/basango/packages/db/tsconfig.json b/basango/packages/db/tsconfig.json index d100dfe..da20e94 100644 --- a/basango/packages/db/tsconfig.json +++ b/basango/packages/db/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@midday/tsconfig/base.json", + "extends": "@basango/tsconfig/base.json", "include": ["src"], "exclude": ["node_modules"], "compilerOptions": { "baseUrl": ".", "paths": { - "@/db": ["./src/*"] + "@db/*": ["./src/*"] } } -} \ No newline at end of file +} diff --git a/basango/packages/logger/package.json b/basango/packages/logger/package.json index 7d27b2c..ef59b0b 100644 --- a/basango/packages/logger/package.json +++ b/basango/packages/logger/package.json @@ -1,18 +1,18 @@ { - "name": "@basango/logger", - "version": "0.0.1", - "private": true, - "type": "module", - "main": "src/index.ts", - "types": "src/index.ts", - "scripts": { - "typecheck": "tsc --noEmit" - }, - "devDependencies": { - "typescript": "^5.9.2" - }, - "dependencies": { - "pino": "^10.1.0", - "pino-pretty": "^13.1.2" - } + "name": "@basango/logger", + "version": "0.0.1", + "private": true, + "type": "module", + "main": "src/index.ts", + "types": "src/index.ts", + "scripts": { + "typecheck": "tsc --noEmit" + }, + "devDependencies": { + "typescript": "catalog:" + }, + "dependencies": { + "pino": "^10.1.0", + "pino-pretty": "^13.1.2" + } } diff --git a/basango/packages/logger/src/index.ts b/basango/packages/logger/src/index.ts index 967c42e..85cee15 100644 --- a/basango/packages/logger/src/index.ts +++ b/basango/packages/logger/src/index.ts @@ -1,20 +1,20 @@ import pino from "pino"; export const logger = pino({ - level: process.env.LOG_LEVEL || "info", - // Use pretty printing in development, structured JSON in production - ...(process.env.NODE_ENV === "development" && { - transport: { - target: "pino-pretty", - options: { - colorize: true, - translateTime: "HH:MM:ss", - ignore: "pid,hostname", - messageFormat: true, - hideObject: false, - }, - }, - }), + level: process.env.LOG_LEVEL || "info", + // Use pretty printing in development, structured JSON in production + ...(process.env.NODE_ENV === "development" && { + transport: { + target: "pino-pretty", + options: { + colorize: true, + translateTime: "HH:MM:ss", + ignore: "pid,hostname", + messageFormat: true, + hideObject: false, + }, + }, + }), }); export default logger; diff --git a/basango/packages/logger/tsconfig.json b/basango/packages/logger/tsconfig.json index 85bb54e..e82cb5c 100644 --- a/basango/packages/logger/tsconfig.json +++ b/basango/packages/logger/tsconfig.json @@ -1,5 +1,5 @@ { - "extends": "@basango/tsconfig/base.json", - "include": ["src/**/*"], - "exclude": ["node_modules"], + "extends": "@basango/tsconfig/base.json", + "include": ["src/**/*"], + "exclude": ["node_modules"] } diff --git a/basango/packages/tsconfig/base.json b/basango/packages/tsconfig/base.json index 0756a8c..5117f2a 100644 --- a/basango/packages/tsconfig/base.json +++ b/basango/packages/tsconfig/base.json @@ -1,19 +1,19 @@ { - "$schema": "https://json.schemastore.org/tsconfig", - "compilerOptions": { - "declaration": true, - "declarationMap": true, - "esModuleInterop": true, - "incremental": false, - "isolatedModules": true, - "lib": ["es2022", "DOM", "DOM.Iterable"], - "module": "NodeNext", - "moduleDetection": "force", - "moduleResolution": "NodeNext", - "noUncheckedIndexedAccess": true, - "resolveJsonModule": true, - "skipLibCheck": true, - "strict": true, - "target": "ES2022" - } + "$schema": "https://json.schemastore.org/tsconfig", + "compilerOptions": { + "declaration": true, + "declarationMap": true, + "esModuleInterop": true, + "incremental": false, + "isolatedModules": true, + "lib": ["es2022", "DOM", "DOM.Iterable"], + "module": "NodeNext", + "moduleDetection": "force", + "moduleResolution": "NodeNext", + "noUncheckedIndexedAccess": true, + "resolveJsonModule": true, + "skipLibCheck": true, + "strict": true, + "target": "ES2022" + } } diff --git a/basango/packages/tsconfig/nextjs.json b/basango/packages/tsconfig/nextjs.json index 20317a2..e6defa4 100644 --- a/basango/packages/tsconfig/nextjs.json +++ b/basango/packages/tsconfig/nextjs.json @@ -1,12 +1,12 @@ { - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "./base.json", - "compilerOptions": { - "plugins": [{ "name": "next" }], - "module": "ESNext", - "moduleResolution": "Bundler", - "allowJs": true, - "jsx": "preserve", - "noEmit": true - } + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "./base.json", + "compilerOptions": { + "plugins": [{ "name": "next" }], + "module": "ESNext", + "moduleResolution": "Bundler", + "allowJs": true, + "jsx": "preserve", + "noEmit": true + } } diff --git a/basango/packages/tsconfig/package.json b/basango/packages/tsconfig/package.json index a91647e..881ea8a 100644 --- a/basango/packages/tsconfig/package.json +++ b/basango/packages/tsconfig/package.json @@ -1,12 +1,12 @@ { - "name": "@basango/tsconfig", - "version": "0.0.0", - "private": true, - "license": "MIT", - "publishConfig": { - "access": "public" - }, - "files": [ - "base.json" - ] + "name": "@basango/tsconfig", + "version": "0.0.0", + "private": true, + "license": "MIT", + "publishConfig": { + "access": "public" + }, + "files": [ + "base.json" + ] } diff --git a/basango/packages/tsconfig/react-library.json b/basango/packages/tsconfig/react-library.json index 44957d6..c3a1b26 100644 --- a/basango/packages/tsconfig/react-library.json +++ b/basango/packages/tsconfig/react-library.json @@ -1,7 +1,7 @@ { - "$schema": "https://json.schemastore.org/tsconfig", - "extends": "./base.json", - "compilerOptions": { - "jsx": "react-jsx" - } + "$schema": "https://json.schemastore.org/tsconfig", + "extends": "./base.json", + "compilerOptions": { + "jsx": "react-jsx" + } } diff --git a/basango/tsconfig.json b/basango/tsconfig.json index 4d296f6..482c9c3 100644 --- a/basango/tsconfig.json +++ b/basango/tsconfig.json @@ -1,3 +1,3 @@ { - "extends": "@basango/tsconfig/base.json" + "extends": "@basango/tsconfig/base.json" } diff --git a/basango/turbo.json b/basango/turbo.json index 45f4415..cc73732 100644 --- a/basango/turbo.json +++ b/basango/turbo.json @@ -1,43 +1,43 @@ { - "$schema": "https://turborepo.com/schema.json", - "globalDependencies": ["**/.env"], - "ui": "tui", - "tasks": { - "topo": { - "dependsOn": ["^topo"] - }, - "build": { - "dependsOn": ["^build"], - "inputs": ["$TURBO_DEFAULT$", ".env*"], - "outputs": [ - ".next/**", - "!.next/cache/**", - "next-env.d.ts", - ".expo/**", - "dist/**", - "build/**", - "lib/**" - ], - "passThroughEnv": [] - }, - "start": { - "cache": false - }, - "test": { - "cache": false - }, - "dev": { - "inputs": ["$TURBO_DEFAULT$", ".env"], - "cache": false, - "persistent": true - }, - "format": {}, - "lint": { - "dependsOn": ["^topo"] - }, - "typecheck": { - "dependsOn": ["^topo"], - "outputs": [] - } - } + "$schema": "https://turborepo.com/schema.json", + "globalDependencies": ["**/.env"], + "ui": "tui", + "tasks": { + "topo": { + "dependsOn": ["^topo"] + }, + "build": { + "dependsOn": ["^build"], + "inputs": ["$TURBO_DEFAULT$", ".env*"], + "outputs": [ + ".next/**", + "!.next/cache/**", + "next-env.d.ts", + ".expo/**", + "dist/**", + "build/**", + "lib/**" + ], + "passThroughEnv": [] + }, + "start": { + "cache": false + }, + "test": { + "cache": false + }, + "dev": { + "inputs": ["$TURBO_DEFAULT$", ".env"], + "cache": false, + "persistent": true + }, + "format": {}, + "lint": { + "dependsOn": ["^topo"] + }, + "typecheck": { + "dependsOn": ["^topo"], + "outputs": [] + } + } } diff --git a/bun.lock b/bun.lock new file mode 100644 index 0000000..418644f --- /dev/null +++ b/bun.lock @@ -0,0 +1,258 @@ +{ + "lockfileVersion": 1, + "workspaces": { + "": { + "name": "basango-monorepo", + "devDependencies": { + "@types/node": "^20.11.30", + "typescript": "^5.4.0", + "vitest": "^1.6.0", + }, + }, + }, + "packages": { + "@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="], + + "@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="], + + "@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="], + + "@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="], + + "@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="], + + "@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="], + + "@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="], + + "@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="], + + "@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="], + + "@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="], + + "@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="], + + "@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="], + + "@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="], + + "@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="], + + "@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="], + + "@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="], + + "@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="], + + "@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="], + + "@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="], + + "@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="], + + "@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="], + + "@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="], + + "@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="], + + "@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="], + + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + + "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.52.5", "", { "os": "android", "cpu": "arm" }, "sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ=="], + + "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.52.5", "", { "os": "android", "cpu": "arm64" }, "sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA=="], + + "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.52.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA=="], + + "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.52.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA=="], + + "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.52.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA=="], + + "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.52.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ=="], + + "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ=="], + + "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ=="], + + "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg=="], + + "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q=="], + + "@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA=="], + + "@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.52.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw=="], + + "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw=="], + + "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg=="], + + "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.52.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ=="], + + "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q=="], + + "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg=="], + + "@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.52.5", "", { "os": "none", "cpu": "arm64" }, "sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw=="], + + "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.52.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w=="], + + "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.52.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg=="], + + "@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ=="], + + "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg=="], + + "@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="], + + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + + "@types/node": ["@types/node@20.19.24", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA=="], + + "@vitest/expect": ["@vitest/expect@1.6.1", "", { "dependencies": { "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "chai": "^4.3.10" } }, "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog=="], + + "@vitest/runner": ["@vitest/runner@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "p-limit": "^5.0.0", "pathe": "^1.1.1" } }, "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA=="], + + "@vitest/snapshot": ["@vitest/snapshot@1.6.1", "", { "dependencies": { "magic-string": "^0.30.5", "pathe": "^1.1.1", "pretty-format": "^29.7.0" } }, "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ=="], + + "@vitest/spy": ["@vitest/spy@1.6.1", "", { "dependencies": { "tinyspy": "^2.2.0" } }, "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw=="], + + "@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="], + + "acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="], + + "acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="], + + "ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="], + + "assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="], + + "cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="], + + "chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="], + + "check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="], + + "confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="], + + "cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="], + + "debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="], + + "deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="], + + "diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="], + + "esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="], + + "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + + "execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="], + + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + + "get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="], + + "get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="], + + "human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="], + + "is-stream": ["is-stream@3.0.0", "", {}, "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA=="], + + "isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="], + + "js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="], + + "local-pkg": ["local-pkg@0.5.1", "", { "dependencies": { "mlly": "^1.7.3", "pkg-types": "^1.2.1" } }, "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ=="], + + "loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="], + + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + + "merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="], + + "mimic-fn": ["mimic-fn@4.0.0", "", {}, "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw=="], + + "mlly": ["mlly@1.8.0", "", { "dependencies": { "acorn": "^8.15.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.1" } }, "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g=="], + + "ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="], + + "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + + "npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="], + + "onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="], + + "p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="], + + "path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="], + + "pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="], + + "pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="], + + "picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="], + + "pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="], + + "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], + + "pretty-format": ["pretty-format@29.7.0", "", { "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } }, "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ=="], + + "react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="], + + "rollup": ["rollup@4.52.5", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.52.5", "@rollup/rollup-android-arm64": "4.52.5", "@rollup/rollup-darwin-arm64": "4.52.5", "@rollup/rollup-darwin-x64": "4.52.5", "@rollup/rollup-freebsd-arm64": "4.52.5", "@rollup/rollup-freebsd-x64": "4.52.5", "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", "@rollup/rollup-linux-arm-musleabihf": "4.52.5", "@rollup/rollup-linux-arm64-gnu": "4.52.5", "@rollup/rollup-linux-arm64-musl": "4.52.5", "@rollup/rollup-linux-loong64-gnu": "4.52.5", "@rollup/rollup-linux-ppc64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-musl": "4.52.5", "@rollup/rollup-linux-s390x-gnu": "4.52.5", "@rollup/rollup-linux-x64-gnu": "4.52.5", "@rollup/rollup-linux-x64-musl": "4.52.5", "@rollup/rollup-openharmony-arm64": "4.52.5", "@rollup/rollup-win32-arm64-msvc": "4.52.5", "@rollup/rollup-win32-ia32-msvc": "4.52.5", "@rollup/rollup-win32-x64-gnu": "4.52.5", "@rollup/rollup-win32-x64-msvc": "4.52.5", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw=="], + + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], + + "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], + + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], + + "signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="], + + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + + "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + + "std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="], + + "strip-final-newline": ["strip-final-newline@3.0.0", "", {}, "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw=="], + + "strip-literal": ["strip-literal@2.1.1", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q=="], + + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], + + "tinypool": ["tinypool@0.8.4", "", {}, "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ=="], + + "tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="], + + "type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="], + + "typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="], + + "ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="], + + "undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="], + + "vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="], + + "vite-node": ["vite-node@1.6.1", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.3.4", "pathe": "^1.1.1", "picocolors": "^1.0.0", "vite": "^5.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA=="], + + "vitest": ["vitest@1.6.1", "", { "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", "@vitest/snapshot": "1.6.1", "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "acorn-walk": "^8.3.2", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", "local-pkg": "^0.5.0", "magic-string": "^0.30.5", "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", "strip-literal": "^2.0.0", "tinybench": "^2.5.1", "tinypool": "^0.8.3", "vite": "^5.0.0", "vite-node": "1.6.1", "why-is-node-running": "^2.2.2" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", "@vitest/browser": "1.6.1", "@vitest/ui": "1.6.1", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag=="], + + "which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="], + + "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + + "yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="], + + "mlly/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + + "npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="], + + "pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + } +}