From 1c478ae443d52ff0ec55bb4e785d23b74952f3d3 Mon Sep 17 00:00:00 2001 From: bernard-ng Date: Fri, 7 Nov 2025 13:28:46 +0200 Subject: [PATCH] [crawler] full migration to typescript --- basango/apps/crawler/config/pipeline.json | 63 +++++---- basango/apps/crawler/config/sources.json | 122 +++++++++------- basango/apps/crawler/package.json | 42 +++--- basango/apps/crawler/src/config.ts | 76 +++++----- basango/apps/crawler/src/http/http-client.ts | 11 +- basango/apps/crawler/src/http/open-graph.ts | 7 +- .../crawler/src/process/async/handlers.ts | 27 ++-- .../apps/crawler/src/process/async/queue.ts | 36 +++-- .../apps/crawler/src/process/async/schemas.ts | 16 +-- .../apps/crawler/src/process/async/tasks.ts | 11 +- .../apps/crawler/src/process/async/worker.ts | 8 +- basango/apps/crawler/src/process/crawler.ts | 6 +- .../apps/crawler/src/process/parsers/base.ts | 13 +- .../apps/crawler/src/process/parsers/html.ts | 37 +++-- .../crawler/src/process/parsers/wordpress.ts | 23 ++- .../apps/crawler/src/process/persistence.ts | 37 +++-- .../apps/crawler/src/process/sync/tasks.ts | 8 +- basango/apps/crawler/src/schema.ts | 42 +++--- basango/apps/crawler/src/scripts/crawl.ts | 2 +- basango/apps/crawler/src/scripts/queue.ts | 2 +- basango/apps/crawler/src/scripts/utils.ts | 12 +- basango/apps/crawler/src/utils.ts | 21 +-- basango/apps/crawler/tsconfig.json | 2 +- basango/apps/crawler/vitest.config.ts | 10 +- basango/biome.json | 3 +- basango/bun.lock | 131 ++++++++++++++++-- basango/packages/logger/src/index.ts | 2 +- basango/packages/tsconfig/base.json | 2 +- 28 files changed, 468 insertions(+), 304 deletions(-) diff --git a/basango/apps/crawler/config/pipeline.json b/basango/apps/crawler/config/pipeline.json index ab271eb..bf2753b 100644 --- a/basango/apps/crawler/config/pipeline.json +++ b/basango/apps/crawler/config/pipeline.json @@ -1,42 +1,41 @@ { - "paths": { - "root": "%env(BASANGO_CRAWLER_ROOT_PATH)%", - "data": "%env(BASANGO_CRAWLER_DATA_PATH)%", - "logs": "%env(BASANGO_CRAWLER_LOGS_PATH)%", - "config": "%env(BASANGO_CRAWLER_CONFIG_PATH)%" - }, "fetch": { - "client": { - "timeout": 20, - "userAgent": "%env(BASANGO_CRAWLER_FETCH_USER_AGENT)%", - "followRedirects": true, - "verifySsl": true, - "rotate": true, - "maxRetries": "%env(BASANGO_CRAWLER_FETCH_MAX_RETRIES)%", - "backoffInitial": 1, - "backoffMultiplier": 2, - "backoffMax": 30, - "respectRetryAfter": "%env(BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER)%" - }, - "crawler": { - "notify": false, - "useMultiThreading": false, - "maxWorkers": 5, - "direction": "%env(BASANGO_CRAWLER_UPDATE_DIRECTION)%" - }, "async": { - "redisUrl": "%env(BASANGO_CRAWLER_ASYNC_REDIS_URL)%", "prefix": "basango:crawler", + "queues": { + "details": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS)%", + "listing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_LISTING)%", + "processing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING)%" + }, + "redisUrl": "%env(BASANGO_CRAWLER_ASYNC_REDIS_URL)%", "ttl": { "default": 600, - "result": "%env(BASANGO_CRAWLER_ASYNC_TTL_RESULT)%", - "failure": "%env(BASANGO_CRAWLER_ASYNC_TTL_FAILURE)%" - }, - "queues": { - "listing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_LISTING)%", - "details": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS)%", - "processing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING)%" + "failure": "%env(number:BASANGO_CRAWLER_ASYNC_TTL_FAILURE)%", + "result": "%env(number:BASANGO_CRAWLER_ASYNC_TTL_RESULT)%" } + }, + "client": { + "backoffInitial": 1, + "backoffMax": 30, + "backoffMultiplier": 2, + "followRedirects": true, + "maxRetries": "%env(number:BASANGO_CRAWLER_FETCH_MAX_RETRIES)%", + "respectRetryAfter": "%env(boolean:BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER)%", + "rotate": true, + "timeout": 20, + "userAgent": "%env(BASANGO_CRAWLER_FETCH_USER_AGENT)%", + "verifySsl": true + }, + "crawler": { + "direction": "%env(BASANGO_CRAWLER_UPDATE_DIRECTION)%", + "maxWorkers": 5, + "notify": false, + "useMultiThreading": false } + }, + "paths": { + "config": "%env(BASANGO_CRAWLER_CONFIG_PATH)%", + "data": "%env(BASANGO_CRAWLER_DATA_PATH)%", + "root": "%env(BASANGO_CRAWLER_ROOT_PATH)%" } } diff --git a/basango/apps/crawler/config/sources.json b/basango/apps/crawler/config/sources.json index a09b7da..37193a7 100644 --- a/basango/apps/crawler/config/sources.json +++ b/basango/apps/crawler/config/sources.json @@ -2,188 +2,216 @@ "sources": { "html": [ { - "sourceId": "radiookapi.net", - "sourceUrl": "https://www.radiookapi.net", + "paginationTemplate": "actualite", + "requiresDetails": true, + "requiresRateLimit": false, "sourceDate": { "pattern": "/(\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/", "replacement": "$3-$2-$1 $4" }, + "sourceId": "radiookapi.net", + "sourceKind": "html", "sourceSelectors": { + "articleBody": ".field-name-body", + "articleCategories": ".views-field-field-cat-gorie a", + "articleDate": ".views-field-created", + "articleLink": ".views-field-title a", "articles": ".view-content > .views-row.content-row", "articleTitle": "h1.page-header", - "articleLink": ".views-field-title a", - "articleBody": ".field-name-body", - "articleDate": ".views-field-created", - "articleCategories": ".views-field-field-cat-gorie a", "pagination": "ul.pagination > li.pager-last > a" }, - "paginationTemplate": "actualite", - "supportsCategories": false, - "requiresDetails": true, - "requiresRateLimit": false + "sourceUrl": "https://www.radiookapi.net", + "supportsCategories": false }, { - "sourceId": "7sur7.cd", - "sourceUrl": "https://7sur7.cd", + "categories": ["politique", "economie", "culture", "sport", "societe"], + "paginationTemplate": "index.php/category/{category}", + "requiresDetails": false, + "requiresRateLimit": false, "sourceDate": { "pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/", "replacement": "$3-$2-$1 $4" }, - "categories": ["politique", "economie", "culture", "sport", "societe"], + "sourceId": "7sur7.cd", + "sourceKind": "html", "sourceSelectors": { - "articles": ".view-content > .row.views-row", - "articleTitle": ".views-field-title a", - "articleLink": ".views-field-title a", "articleBody": ".field.field--name-body", "articleDate": ".views-field-created", + "articleLink": ".views-field-title a", + "articles": ".view-content > .row.views-row", + "articleTitle": ".views-field-title a", "pagination": "ul.pagination > li.pager__item.pager__item--last > a" }, - "paginationTemplate": "index.php/category/{category}", - "supportsCategories": true, - "requiresDetails": false, - "requiresRateLimit": false + "sourceUrl": "https://7sur7.cd", + "supportsCategories": true }, { - "sourceId": "mediacongo.net", - "sourceUrl": "https://www.mediacongo.net", + "paginationTemplate": "articles.html", + "requiresDetails": true, + "requiresRateLimit": false, "sourceDate": { "format": "%d.%m.%Y %H:%M" }, + "sourceId": "mediacongo.net", + "sourceKind": "html", "sourceSelectors": { - "articles": ".for_aitems > .article_other_item", - "articleTitle": "img", - "articleLink": "a:first-child", - "articleCategories": "a.color_link", "articleBody": ".article_ttext", + "articleCategories": "a.color_link", "articleDate": ".article_other_about", + "articleLink": "a:first-child", + "articles": ".for_aitems > .article_other_item", + "articleTitle": "h1", "pagination": "div.pagination > div > a:last-child" }, - "paginationTemplate": "articles.html", - "supportsCategories": false, - "requiresDetails": true, - "requiresRateLimit": false + "sourceUrl": "https://www.mediacongo.net", + "supportsCategories": false }, { - "sourceId": "actualite.cd", - "sourceUrl": "https://actualite.cd", + "paginationTemplate": "actualite", + "requiresDetails": true, + "requiresRateLimit": false, "sourceDate": { "pattern": "/(\\d{1}) (\\d{1,2}) (\\d{2}) (\\d{4}) - (\\d{2}:\\d{2})/", "replacement": "$4-$3-$2 $5" }, + "sourceId": "actualite.cd", + "sourceKind": "html", "sourceSelectors": { - "articles": "#views-bootstrap-taxonomy-term-page-2 > div > div", - "articleTitle": "#actu-titre a", - "articleLink": "#actu-titre a", - "articleCategories": "#actu-cat a", "articleBody": ".views-field.views-field-body", - "articleDate": "#p-date" + "articleCategories": "#actu-cat", + "articleDate": "#p-date", + "articleLink": "#actu-titre a", + "articles": "#views-bootstrap-taxonomy-term-page-2 > div > div", + "articleTitle": "h1.page-title" }, - "paginationTemplate": "actualite", - "supportsCategories": false, - "requiresDetails": true, - "requiresRateLimit": false + "sourceUrl": "https://actualite.cd", + "supportsCategories": false } ], "wordpress": [ { + "requiresRateLimit": true, "sourceId": "beto.cd", - "sourceUrl": "https://beto.cd", - "requiresRateLimit": true + "sourceKind": "wordpress", + "sourceUrl": "https://beto.cd" }, - { "sourceId": "newscd.net", "sourceUrl": "https://newscd.net" }, + { "sourceId": "newscd.net", "sourceKind": "wordpress", "sourceUrl": "https://newscd.net" }, { "sourceId": "africanewsrdc.net", + "sourceKind": "wordpress", "sourceUrl": "https://www.africanewsrdc.net" }, { "sourceId": "angazainstitute.ac.cd", + "sourceKind": "wordpress", "sourceUrl": "https://angazainstitute.ac.cd" }, - { "sourceId": "b-onetv.cd", "sourceUrl": "https://b-onetv.cd" }, + { "sourceId": "b-onetv.cd", "sourceKind": "wordpress", "sourceUrl": "https://b-onetv.cd" }, { "sourceId": "bukavufm.com", + "sourceKind": "wordpress", "sourceUrl": "https://bukavufm.com" }, { "sourceId": "changement7.net", + "sourceKind": "wordpress", "sourceUrl": "https://changement7.net" }, { "sourceId": "congoactu.net", + "sourceKind": "wordpress", "sourceUrl": "https://congoactu.net" }, { "sourceId": "congoindependant.com", + "sourceKind": "wordpress", "sourceUrl": "https://www.congoindependant.com" }, { "sourceId": "congoquotidien.com", + "sourceKind": "wordpress", "sourceUrl": "https://www.congoquotidien.com" }, { "sourceId": "cumulard.cd", + "sourceKind": "wordpress", "sourceUrl": "https://www.cumulard.cd" }, { "sourceId": "environews-rdc.net", + "sourceKind": "wordpress", "sourceUrl": "https://environews-rdc.net" }, { "sourceId": "freemediardc.info", + "sourceKind": "wordpress", "sourceUrl": "https://www.freemediardc.info" }, { "sourceId": "geopolismagazine.org", + "sourceKind": "wordpress", "sourceUrl": "https://geopolismagazine.org" }, { "sourceId": "habarirdc.net", + "sourceKind": "wordpress", "sourceUrl": "https://habarirdc.net" }, - { "sourceId": "infordc.com", "sourceUrl": "https://infordc.com" }, + { "sourceId": "infordc.com", "sourceKind": "wordpress", "sourceUrl": "https://infordc.com" }, { "sourceId": "kilalopress.net", + "sourceKind": "wordpress", "sourceUrl": "https://kilalopress.net" }, { "sourceId": "laprosperiteonline.net", + "sourceKind": "wordpress", "sourceUrl": "https://laprosperiteonline.net" }, { "sourceId": "laprunellerdc.cd", + "sourceKind": "wordpress", "sourceUrl": "https://laprunellerdc.cd" }, { "sourceId": "lesmedias.net", + "sourceKind": "wordpress", "sourceUrl": "https://lesmedias.net" }, { "sourceId": "lesvolcansnews.net", + "sourceKind": "wordpress", "sourceUrl": "https://lesvolcansnews.net" }, { "sourceId": "netic-news.net", + "sourceKind": "wordpress", "sourceUrl": "https://www.netic-news.net" }, { "sourceId": "objectif-infos.cd", + "sourceKind": "wordpress", "sourceUrl": "https://objectif-infos.cd" }, { "sourceId": "scooprdc.net", + "sourceKind": "wordpress", "sourceUrl": "https://scooprdc.net" }, { "sourceId": "journaldekinshasa.com", + "sourceKind": "wordpress", "sourceUrl": "https://www.journaldekinshasa.com" }, { "sourceId": "lepotentiel.cd", + "sourceKind": "wordpress", "sourceUrl": "https://lepotentiel.cd" }, - { "sourceId": "acturdc.com", "sourceUrl": "https://acturdc.com" }, + { "sourceId": "acturdc.com", "sourceKind": "wordpress", "sourceUrl": "https://acturdc.com" }, { "sourceId": "matininfos.net", + "sourceKind": "wordpress", "sourceUrl": "https://matininfos.net" } ] diff --git a/basango/apps/crawler/package.json b/basango/apps/crawler/package.json index 2f2d77f..ad2dd01 100644 --- a/basango/apps/crawler/package.json +++ b/basango/apps/crawler/package.json @@ -1,28 +1,30 @@ { "name": "@basango/crawler", "private": true, - "type": "module", - "scripts": { - "=========== CODE STYLE ============": "", - "test": "vitest --run", - "lint": "biome check .", - "lint:fix": "biome check --write .", - "format": "biome format --write .", - "============= CLI =============": "", - "crawl:sync": "bun run src/scripts/crawl.ts", - "crawl:async": "bun run src/scripts/queue.ts", - "crawl:worker": "bun run src/scripts/worker.ts" - }, "dependencies": { "@basango/logger": "workspace:*", - "@devscast/config": "^1.0.2", - "bullmq": "^4.17.0", - "date-fns": "catalog:", - "ioredis": "^5.3.2", + "@devscast/config": "^1.0.3", + "bullmq": "^4.18.3", + "date-fns": "^3.6.0", + "ioredis": "^5.8.2", "node-html-parser": "^7.0.1", - "tiktoken": "^1.0.14", + "tiktoken": "^1.0.22", "turndown": "^7.2.2", - "yaml": "^2.8.1", - "zod": "catalog:" - } + "zod": "^4.1.12" + }, + "devDependencies": { + "@types/turndown": "^5.0.6", + "vitest": "^4.0.7" + }, + "scripts": { + "crawler:async": "bun run src/scripts/queue.ts", + "crawler:sync": "bun run src/scripts/crawl.ts", + "crawler:worker": "bun run src/scripts/worker.ts", + "format": "biome format --write .", + "lint": "biome check .", + "lint:fix": "biome check --write .", + "test": "vitest --run", + "typecheck": "tsc --noEmit" + }, + "type": "module" } diff --git a/basango/apps/crawler/src/config.ts b/basango/apps/crawler/src/config.ts index e02e582..d929fc4 100644 --- a/basango/apps/crawler/src/config.ts +++ b/basango/apps/crawler/src/config.ts @@ -1,6 +1,6 @@ import path from "node:path"; -import { loadConfig } from "@devscast/config"; +import { loadConfig as defineConfig } from "@devscast/config"; import { z } from "zod"; import { DateRangeSchema, @@ -13,49 +13,49 @@ import { export const PROJECT_DIR = path.resolve(__dirname, "../"); export const PipelineConfigSchema = z.object({ - paths: z.object({ - root: z.string().default(PROJECT_DIR), - data: z.string().default(path.join(PROJECT_DIR, "data", "dataset")), - config: z.string().default(path.join(PROJECT_DIR, "config")), - }), fetch: z.object({ - client: z.object({ - timeout: z.number().positive().default(20), - userAgent: z.string().default("Basango/0.1 (+https://github.com/bernard-ng/basango)"), - followRedirects: z.boolean().default(true), - verifySsl: z.boolean().default(true), - rotate: z.boolean().default(true), - maxRetries: z.number().int().nonnegative().default(3), - backoffInitial: z.number().nonnegative().default(1), - backoffMultiplier: z.number().positive().default(2), - backoffMax: z.number().nonnegative().default(30), - respectRetryAfter: z.boolean().default(true), - }), - crawler: z.object({ - source: z.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema]).optional(), - pageRange: PageRangeSchema.optional(), - dateRange: DateRangeSchema.optional(), - category: z.string().optional(), - notify: z.boolean().default(false), - isUpdate: z.boolean().default(false), - useMultiThreading: z.boolean().default(false), - maxWorkers: z.number().int().positive().default(5), - direction: UpdateDirectionSchema.default("forward"), - }), async: z.object({ - redisUrl: z.string().default("redis://localhost:6379/0"), prefix: z.string().default("basango:crawler:queue"), - ttl: z.object({ - default: z.number().int().positive().default(600), - result: z.number().int().nonnegative().default(3600), - failure: z.number().int().nonnegative().default(3600), - }), queues: z.object({ - listing: z.string().default("listing"), details: z.string().default("details"), + listing: z.string().default("listing"), processing: z.string().default("processing"), }), + redisUrl: z.string().default("redis://localhost:6379/0"), + ttl: z.object({ + default: z.number().int().positive().default(600), + failure: z.number().int().nonnegative().default(3600), + result: z.number().int().nonnegative().default(3600), + }), }), + client: z.object({ + backoffInitial: z.number().nonnegative().default(1), + backoffMax: z.number().nonnegative().default(30), + backoffMultiplier: z.number().positive().default(2), + followRedirects: z.boolean().default(true), + maxRetries: z.number().int().nonnegative().default(3), + respectRetryAfter: z.boolean().default(true), + rotate: z.boolean().default(true), + timeout: z.number().positive().default(20), + userAgent: z.string().default("Basango/0.1 (+https://github.com/bernard-ng/basango)"), + verifySsl: z.boolean().default(true), + }), + crawler: z.object({ + category: z.string().optional(), + dateRange: DateRangeSchema.optional(), + direction: UpdateDirectionSchema.default("forward"), + isUpdate: z.boolean().default(false), + maxWorkers: z.number().int().positive().default(5), + notify: z.boolean().default(false), + pageRange: PageRangeSchema.optional(), + source: z.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema]).optional(), + useMultiThreading: z.boolean().default(false), + }), + }), + paths: z.object({ + config: z.string().default(path.join(PROJECT_DIR, "config")), + data: z.string().default(path.join(PROJECT_DIR, "data", "datasets")), + root: z.string().default(PROJECT_DIR), }), sources: z.object({ html: z.array(HtmlSourceConfigSchema).default([]), @@ -63,12 +63,12 @@ export const PipelineConfigSchema = z.object({ }), }); -export const { config, env } = loadConfig({ - schema: PipelineConfigSchema, +export const { config, env } = defineConfig({ cwd: process.cwd(), env: { path: path.join(PROJECT_DIR, ".env"), }, + schema: PipelineConfigSchema, sources: [ path.join(PROJECT_DIR, "config", "pipeline.json"), path.join(PROJECT_DIR, "config", "sources.json"), diff --git a/basango/apps/crawler/src/http/http-client.ts b/basango/apps/crawler/src/http/http-client.ts index 34c73b2..1bae380 100644 --- a/basango/apps/crawler/src/http/http-client.ts +++ b/basango/apps/crawler/src/http/http-client.ts @@ -1,12 +1,11 @@ import { setTimeout as delay } from "node:timers/promises"; - +import { FetchClientConfig } from "@/config"; import { DEFAULT_RETRY_AFTER_HEADER, DEFAULT_USER_AGENT, TRANSIENT_HTTP_STATUSES, } from "@/constants"; import { UserAgents } from "@/http/user-agent"; -import { FetchClientConfig } from "@/config"; export type HttpHeaders = Record; export type HttpParams = Record; @@ -72,7 +71,7 @@ const buildUrl = (url: string, params?: HttpParams): string => { */ const computeBackoff = (config: FetchClientConfig, attempt: number): number => { const base = Math.min( - config.backoffInitial * Math.pow(config.backoffMultiplier, attempt), + config.backoffInitial * config.backoffMultiplier ** attempt, config.backoffMax, ); const jitter = Math.random() * base * 0.25; @@ -172,11 +171,11 @@ export class SyncHttpClient extends BaseHttpClient { const headers = this.buildHeaders(options.headers); const init: RequestInit = { - method, - headers, body: options.data as BodyInit | undefined, - signal: controller.signal, + headers, + method, redirect: this.config.followRedirects ? "follow" : "manual", + signal: controller.signal, }; if (options.json !== undefined) { diff --git a/basango/apps/crawler/src/http/open-graph.ts b/basango/apps/crawler/src/http/open-graph.ts index 5024d6e..03b632c 100644 --- a/basango/apps/crawler/src/http/open-graph.ts +++ b/basango/apps/crawler/src/http/open-graph.ts @@ -1,9 +1,8 @@ import { parse } from "node-html-parser"; - +import { config } from "@/config"; import { OPEN_GRAPH_USER_AGENT } from "@/constants"; import { SyncHttpClient } from "@/http/http-client"; import { UserAgents } from "@/http/user-agent"; -import { config } from "@/config"; import { ArticleMetadata } from "@/schema"; /** @@ -47,8 +46,8 @@ export class OpenGraph { const provider = new UserAgents(true, OPEN_GRAPH_USER_AGENT); this.client = new SyncHttpClient(settings, { - userAgentProvider: provider, defaultHeaders: { "User-Agent": provider.og() }, + userAgentProvider: provider, }); } @@ -94,9 +93,9 @@ export class OpenGraph { } return { - title, description, image, + title, url: canonical, }; } diff --git a/basango/apps/crawler/src/process/async/handlers.ts b/basango/apps/crawler/src/process/async/handlers.ts index fe871cb..b99dd9c 100644 --- a/basango/apps/crawler/src/process/async/handlers.ts +++ b/basango/apps/crawler/src/process/async/handlers.ts @@ -1,20 +1,19 @@ import { logger } from "@basango/logger"; import { config, env } from "@/config"; -import { Article, HtmlSourceConfig, SourceKindSchema, WordPressSourceConfig } from "@/schema"; -import { createDateRange, formatDateRange, formatPageRange, resolveSourceConfig } from "@/utils"; +import { SyncHttpClient } from "@/http/http-client"; +import { createQueueManager, QueueManager } from "@/process/async/queue"; import { DetailsTaskPayload, ListingTaskPayload, ProcessingTaskPayload, } from "@/process/async/schemas"; -import { createQueueManager, QueueManager } from "@/process/async/queue"; +import { resolveCrawlerConfig } from "@/process/crawler"; import { HtmlCrawler } from "@/process/parsers/html"; import { WordPressCrawler } from "@/process/parsers/wordpress"; import { JsonlPersistor } from "@/process/persistence"; -import { SyncHttpClient } from "@/http/http-client"; - -import { resolveCrawlerConfig } from "@/process/crawler"; +import { Article, HtmlSourceConfig, SourceKindSchema, WordPressSourceConfig } from "@/schema"; +import { createDateRange, formatDateRange, formatPageRange, resolveSourceConfig } from "@/utils"; export const collectHtmlListing = async ( payload: ListingTaskPayload, @@ -40,10 +39,10 @@ export const collectHtmlListing = async ( if (!url) continue; await manager.enqueueArticle({ - url, - sourceId: payload.sourceId, category: payload.category, dateRange: createDateRange(payload.dateRange), + sourceId: payload.sourceId, + url, } as DetailsTaskPayload); queued += 1; } @@ -79,11 +78,11 @@ export const collectWordPressListing = async ( if (!url) continue; await manager.enqueueArticle({ - url, - data, - sourceId: payload.sourceId, category: payload.category, + data, dateRange: createDateRange(payload.dateRange), + sourceId: payload.sourceId, + url, } as DetailsTaskPayload); queued += 1; } @@ -98,10 +97,10 @@ export const collectWordPressListing = async ( export const collectArticle = async (payload: DetailsTaskPayload): Promise => { const source = resolveSourceConfig(payload.sourceId); const settings = resolveCrawlerConfig(source, { - pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined, - dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined, - sourceId: payload.sourceId, category: payload.category, + dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined, + pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined, + sourceId: payload.sourceId, }); const persistors = [ new JsonlPersistor({ diff --git a/basango/apps/crawler/src/process/async/queue.ts b/basango/apps/crawler/src/process/async/queue.ts index 47e2002..d63f42a 100644 --- a/basango/apps/crawler/src/process/async/queue.ts +++ b/basango/apps/crawler/src/process/async/queue.ts @@ -1,8 +1,7 @@ import { randomUUID } from "node:crypto"; - -import IORedis from "ioredis"; import { JobsOptions, Queue, QueueOptions } from "bullmq"; - +import IORedis from "ioredis"; +import { config, FetchAsyncConfig } from "@/config"; import { DetailsTaskPayload, DetailsTaskPayloadSchema, @@ -12,7 +11,6 @@ import { ProcessingTaskPayloadSchema, } from "@/process/async/schemas"; import { parseRedisUrl } from "@/utils"; -import { config, FetchAsyncConfig } from "@/config"; export interface QueueBackend { add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>; @@ -26,7 +24,11 @@ export type QueueFactory = ( const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => { const redisConnection = - connection ?? new IORedis(settings.redisUrl, parseRedisUrl(settings.redisUrl)); + connection ?? + new IORedis(settings.redisUrl, { + ...parseRedisUrl(settings.redisUrl), + maxRetriesPerRequest: null, + }); const options: QueueOptions = { connection: redisConnection, prefix: settings.prefix, @@ -65,24 +67,30 @@ export const createQueueManager = (options: CreateQueueManagerOptions = {}): Que const settings = config.fetch.async; const connection = - options.connection ?? new IORedis(settings.redisUrl, parseRedisUrl(settings.redisUrl)); + options.connection ?? + new IORedis(settings.redisUrl, { + ...parseRedisUrl(settings.redisUrl), + maxRetriesPerRequest: null, + }); const factory = options.queueFactory ?? defaultQueueFactory; const ensureQueue = (queueName: string) => factory(queueName, settings, connection); return { - settings, - connection, - enqueueListing: (payload) => { - const data = ListingTaskPayloadSchema.parse(payload); - const queue = ensureQueue(settings.queues.listing); - return queue.add("collect_listing", data); + close: async () => { + await connection.quit(); }, + connection, enqueueArticle: (payload) => { const data = DetailsTaskPayloadSchema.parse(payload); const queue = ensureQueue(settings.queues.details); return queue.add("collect_article", data); }, + enqueueListing: (payload) => { + const data = ListingTaskPayloadSchema.parse(payload); + const queue = ensureQueue(settings.queues.listing); + return queue.add("collect_listing", data); + }, enqueueProcessed: (payload) => { const data = ProcessingTaskPayloadSchema.parse(payload); const queue = ensureQueue(settings.queues.processing); @@ -94,8 +102,6 @@ export const createQueueManager = (options: CreateQueueManagerOptions = {}): Que `${settings.prefix}:${settings.queues.processing}`, ], queueName: (suffix: string) => `${settings.prefix}:${suffix}`, - close: async () => { - await connection.quit(); - }, + settings, }; }; diff --git a/basango/apps/crawler/src/process/async/schemas.ts b/basango/apps/crawler/src/process/async/schemas.ts index 4101247..c6fdfc3 100644 --- a/basango/apps/crawler/src/process/async/schemas.ts +++ b/basango/apps/crawler/src/process/async/schemas.ts @@ -2,25 +2,25 @@ import { z } from "zod"; import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "@/schema"; export const ListingTaskPayloadSchema = z.object({ - sourceId: z.string(), - pageRange: z.string().optional(), - dateRange: z.string().optional(), category: z.string().optional(), + dateRange: z.string().optional(), + pageRange: z.string().optional(), + sourceId: z.string(), }); export const DetailsTaskPayloadSchema = z.object({ - sourceId: z.string(), - url: z.url(), + category: z.string().optional(), data: z.any().optional(), + dateRange: DateRangeSchema.optional(), page: z.number().int().nonnegative().optional(), pageRange: PageRangeSchema.optional(), - dateRange: DateRangeSchema.optional(), - category: z.string().optional(), + sourceId: z.string(), + url: z.url(), }); export const ProcessingTaskPayloadSchema = z.object({ - sourceId: z.string(), article: ArticleSchema, + sourceId: z.string(), }); export type ListingTaskPayload = z.infer; diff --git a/basango/apps/crawler/src/process/async/tasks.ts b/basango/apps/crawler/src/process/async/tasks.ts index 2941488..522958d 100644 --- a/basango/apps/crawler/src/process/async/tasks.ts +++ b/basango/apps/crawler/src/process/async/tasks.ts @@ -1,12 +1,11 @@ import { logger } from "@basango/logger"; - +import * as handlers from "@/process/async/handlers"; +import { createQueueManager } from "@/process/async/queue"; import { DetailsTaskPayloadSchema, ListingTaskPayloadSchema, ProcessingTaskPayloadSchema, } from "@/process/async/schemas"; -import { createQueueManager } from "@/process/async/queue"; -import * as handlers from "@/process/async/handlers"; import { CrawlingOptions } from "@/process/crawler"; export const collectListing = async (payload: unknown): Promise => { @@ -41,10 +40,10 @@ export const forwardForProcessing = async (payload: unknown): Promise = export const scheduleAsyncCrawl = async (options: CrawlingOptions): Promise => { const payload = ListingTaskPayloadSchema.parse({ - sourceId: options.sourceId, - pageRange: options.pageRange, - dateRange: options.dateRange, category: options.category, + dateRange: options.dateRange, + pageRange: options.pageRange, + sourceId: options.sourceId, }); const manager = createQueueManager(); diff --git a/basango/apps/crawler/src/process/async/worker.ts b/basango/apps/crawler/src/process/async/worker.ts index 27e51dd..01132e4 100644 --- a/basango/apps/crawler/src/process/async/worker.ts +++ b/basango/apps/crawler/src/process/async/worker.ts @@ -1,5 +1,5 @@ -import IORedis from "ioredis"; import { QueueEvents, Worker } from "bullmq"; +import IORedis from "ioredis"; import { QueueFactory, QueueManager } from "@/process/async/queue"; import { collectArticle, collectListing, forwardForProcessing } from "@/process/async/tasks"; @@ -43,8 +43,8 @@ export const startWorker = (options: WorkerOptions): WorkerHandle => { } }, { - connection, concurrency: options.concurrency ?? 5, + connection, }, ); @@ -60,8 +60,6 @@ export const startWorker = (options: WorkerOptions): WorkerHandle => { } return { - workers, - events, close: async () => { await Promise.all(workers.map((worker) => worker.close())); await Promise.all(events.map((event) => event.close())); @@ -70,5 +68,7 @@ export const startWorker = (options: WorkerOptions): WorkerHandle => { await manager.close(); } }, + events, + workers, }; }; diff --git a/basango/apps/crawler/src/process/crawler.ts b/basango/apps/crawler/src/process/crawler.ts index f0e71ef..067ec33 100644 --- a/basango/apps/crawler/src/process/crawler.ts +++ b/basango/apps/crawler/src/process/crawler.ts @@ -1,7 +1,7 @@ +import logger from "@basango/logger"; import { config, FetchCrawlerConfig } from "@/config"; import { JsonlPersistor, Persistor } from "@/process/persistence"; import { AnySourceConfig } from "@/schema"; -import logger from "@basango/logger"; import { createDateRange, createPageRange } from "@/utils"; export interface CrawlingOptions { @@ -17,10 +17,10 @@ export const resolveCrawlerConfig = ( ): FetchCrawlerConfig => { return { ...config.fetch.crawler, - source, + category: options.category, dateRange: createDateRange(options.dateRange), pageRange: createPageRange(options.pageRange), - category: options.category, + source, }; }; diff --git a/basango/apps/crawler/src/process/parsers/base.ts b/basango/apps/crawler/src/process/parsers/base.ts index d7a4c4f..117680c 100644 --- a/basango/apps/crawler/src/process/parsers/base.ts +++ b/basango/apps/crawler/src/process/parsers/base.ts @@ -1,9 +1,8 @@ -import { parse as parseHtml, HTMLElement } from "node-html-parser"; - +import { HTMLElement, parse as parseHtml } from "node-html-parser"; +import { config, FetchCrawlerConfig } from "@/config"; import { SyncHttpClient } from "@/http/http-client"; import { OpenGraph } from "@/http/open-graph"; import type { Persistor } from "@/process/persistence"; -import { config, FetchCrawlerConfig } from "@/config"; import { AnySourceConfig, Article } from "@/schema"; export interface CrawlerOptions { @@ -51,8 +50,8 @@ export abstract class BaseCrawler { protected textContent(node: HTMLElement | null | undefined): string | null { if (!node) return null; // innerText keeps spacing similar to browser rendering - const value = (node as any).innerText ?? node.text; - const text = typeof value === "string" ? value.trim() : String(value ?? "").trim(); + const value = node.innerText ?? node.text; + const text = value.trim(); return text.length ? text : null; } @@ -64,7 +63,7 @@ export abstract class BaseCrawler { protected extractFirst(root: HTMLElement, selector?: string | null): HTMLElement | null { if (!selector) return null; try { - return (root as any).querySelector?.(selector) ?? null; + return root.querySelector(selector) ?? null; } catch { return null; } @@ -78,7 +77,7 @@ export abstract class BaseCrawler { protected extractAll(root: HTMLElement, selector?: string | null): HTMLElement[] { if (!selector) return []; try { - return ((root as any).querySelectorAll?.(selector) ?? []) as HTMLElement[]; + return root.querySelectorAll(selector); } catch { return []; } diff --git a/basango/apps/crawler/src/process/parsers/html.ts b/basango/apps/crawler/src/process/parsers/html.ts index e5c4292..46d7863 100644 --- a/basango/apps/crawler/src/process/parsers/html.ts +++ b/basango/apps/crawler/src/process/parsers/html.ts @@ -1,18 +1,17 @@ import { logger } from "@basango/logger"; -import { HTMLElement } from "node-html-parser"; import { getUnixTime, isMatch as isDateMatch, parse as parseDateFns } from "date-fns"; - -import { isTimestampInRange, createAbsoluteUrl } from "@/utils"; -import { persist, Persistor } from "@/process/persistence"; -import { BaseCrawler } from "@/process/parsers/base"; +import { HTMLElement } from "node-html-parser"; import TurndownService from "turndown"; -import { DateRange, HtmlSourceConfig } from "@/schema"; import { FetchCrawlerConfig } from "@/config"; +import { BaseCrawler } from "@/process/parsers/base"; +import { Persistor, persist } from "@/process/persistence"; +import { DateRange, HtmlSourceConfig } from "@/schema"; +import { createAbsoluteUrl, isTimestampInRange } from "@/utils"; const md = new TurndownService({ + bulletListMarker: "-", headingStyle: "atx", hr: "---", - bulletListMarker: "-", }); /** @@ -136,7 +135,7 @@ export class HtmlCrawler extends BaseCrawler { if (dateRange && !isTimestampInRange(dateRange, timestamp)) { logger.info( - { title: titleText, link, date: rawDate, timestamp }, + { date: rawDate, link, timestamp, title: titleText }, "Skipping article outside date range", ); return null; @@ -144,12 +143,12 @@ export class HtmlCrawler extends BaseCrawler { const enriched = await this.enrichWithOpenGraph( { - title: titleText, - link, body, categories, + link, source: this.source.sourceId, timestamp, + title: titleText, }, link, ); @@ -172,7 +171,7 @@ export class HtmlCrawler extends BaseCrawler { * Get the pagination range (start and end page numbers). */ async getPagination(): Promise<{ start: number; end: number }> { - return { start: 0, end: await this.getLastPage() }; + return { end: await this.getLastPage(), start: 0 }; } /** @@ -187,7 +186,7 @@ export class HtmlCrawler extends BaseCrawler { const links = this.extractAll(root, this.source.sourceSelectors.pagination); if (!links.length) return 1; const last = links[links.length - 1]!; - const href = (last as any).getAttribute?.("href") as string | null; + const href = last.getAttribute("href") as string | null; if (!href) return 1; // Heuristic: prefer a number in the href, else "page" query param @@ -246,12 +245,10 @@ export class HtmlCrawler extends BaseCrawler { if (!target) return null; const href = - (target.getAttribute?.("href") as string | null) ?? - ((target as any).getAttribute?.("data-href") as string | null) ?? - ((target as any).getAttribute?.("src") as string | null); + target.getAttribute("href") ?? target.getAttribute("data-href") ?? target.getAttribute("src"); + if (!href) return null; - const absolute = createAbsoluteUrl(this.source.sourceUrl, href); - return absolute; + return createAbsoluteUrl(this.source.sourceUrl, href); } /** @@ -265,10 +262,10 @@ export class HtmlCrawler extends BaseCrawler { if (!target) return null; // If it's an image, prefer alt/title - const tag = (target as any).tagName?.toLowerCase?.() as string | undefined; + const tag = target.tagName.toLowerCase(); if (tag === "img") { - const alt = (target as any).getAttribute?.("alt") as string | null; - const title = (target as any).getAttribute?.("title") as string | null; + const alt = target.getAttribute("alt"); + const title = target.getAttribute("title"); const pick = (alt ?? title ?? "").trim(); if (pick.length > 0) return pick; } diff --git a/basango/apps/crawler/src/process/parsers/wordpress.ts b/basango/apps/crawler/src/process/parsers/wordpress.ts index c185a70..242143c 100644 --- a/basango/apps/crawler/src/process/parsers/wordpress.ts +++ b/basango/apps/crawler/src/process/parsers/wordpress.ts @@ -1,15 +1,14 @@ import { logger } from "@basango/logger"; - -import { DateRange, PageRange, WordPressSourceConfig } from "@/schema"; -import { BaseCrawler } from "@/process/parsers/base"; -import { persist, Persistor } from "@/process/persistence"; import TurndownService from "turndown"; import { FetchCrawlerConfig } from "@/config"; +import { BaseCrawler } from "@/process/parsers/base"; +import { Persistor, persist } from "@/process/persistence"; +import { DateRange, PageRange, WordPressSourceConfig } from "@/schema"; const md = new TurndownService({ + bulletListMarker: "-", headingStyle: "atx", hr: "---", - bulletListMarker: "-", }); interface WordPressPost { @@ -59,7 +58,7 @@ export class WordPressCrawler extends BaseCrawler { const data = (await response.json()) as unknown; const articles = Array.isArray(data) ? (data as WordPressPost[]) : []; if (!Array.isArray(data)) { - logger.warn({ type: typeof data, page }, "Unexpected WordPress payload type"); + logger.warn({ page, type: typeof data }, "Unexpected WordPress payload type"); } for (const entry of articles) { @@ -132,7 +131,7 @@ export class WordPressCrawler extends BaseCrawler { const { isTimestampInRange } = await import("@/utils"); if (!isTimestampInRange(dateRange, timestamp)) { logger.info( - { title, link, date: data.date, timestamp }, + { date: data.date, link, timestamp, title }, "Skipping article outside date range", ); return null; @@ -141,12 +140,12 @@ export class WordPressCrawler extends BaseCrawler { const enriched = await this.enrichWithOpenGraph( { - title, - link, body, categories, + link, source: this.source.sourceId, timestamp, + title, }, link, ); @@ -169,11 +168,11 @@ export class WordPressCrawler extends BaseCrawler { response.headers.get(WordPressCrawler.TOTAL_POSTS_HEADER) ?? "0", 10, ); - logger.info({ posts, pages }, "WordPress pagination"); + logger.info({ pages, posts }, "WordPress pagination"); const end = Number.isFinite(pages) && pages > 0 ? pages : 1; - return { start: 1, end }; + return { end, start: 1 }; } catch { - return { start: 1, end: 1 }; + return { end: 1, start: 1 }; } } diff --git a/basango/apps/crawler/src/process/persistence.ts b/basango/apps/crawler/src/process/persistence.ts index d36c41d..1315ced 100644 --- a/basango/apps/crawler/src/process/persistence.ts +++ b/basango/apps/crawler/src/process/persistence.ts @@ -1,8 +1,8 @@ import fs from "node:fs"; import path from "node:path"; +import logger from "@basango/logger"; import { Article } from "@/schema"; import { countTokens } from "@/utils"; -import logger from "@basango/logger"; export interface Persistor { persist(record: Article): Promise | void; @@ -16,14 +16,38 @@ export interface PersistorOptions { encoding?: BufferEncoding; } +const sanitize = (text: string): string => { + if (!text) return text; + + let s = text.replace(/\u00A0/g, " "); // remove NBSP + s = s.replace(" ", " "); // remove other NBSP + s = s.replace(" ", " "); // remove NARROW NO-BREAK SPACE + s = s.replace(/\u200B/g, ""); // remove ZERO WIDTH SPACE + s = s.replace(/\u200C/g, ""); // remove ZERO WIDTH NON-JOINER + s = s.replace(/\u200D/g, ""); // remove ZERO WIDTH JOINER + s = s.replace(/\uFEFF/g, ""); // remove ZERO WIDTH NO-BREAK SPACE + s = s.replace(/\r\n/g, "\n"); // normalize CRLF to LF + s = s.replace(/\n{2,}/g, "\n"); // collapse multiple newlines to one + // s = s.replace(/[ \t]{2,}/g, " "); // collapse multiple spaces/tabs + + return s.trim(); +}; + export const persist = async (payload: Article, persistors: Persistor[]): Promise
=> { - const article = { + const data = { ...payload, + body: sanitize(payload.body), + categories: payload.categories.map(sanitize), + title: sanitize(payload.title), + }; + + const article = { + ...data, tokenStatistics: { - title: countTokens(payload.title), body: countTokens(payload.body), - excerpt: countTokens(payload.body.substring(0, 200)), categories: countTokens(payload.categories.join(",")), + excerpt: countTokens(payload.body.substring(0, 200)), + title: countTokens(payload.title), }, } as Article; @@ -65,10 +89,7 @@ export class JsonlPersistor implements Persistor { const payload = `${JSON.stringify(record)}\n`; this.pending = this.pending.then(async () => { - fs.writeFileSync(this.filePath, payload, { - encoding: this.encoding, - mode: "a", - }); + fs.appendFileSync(this.filePath, payload, { encoding: this.encoding }); }); return this.pending; diff --git a/basango/apps/crawler/src/process/sync/tasks.ts b/basango/apps/crawler/src/process/sync/tasks.ts index bcf6fd3..b79ef67 100644 --- a/basango/apps/crawler/src/process/sync/tasks.ts +++ b/basango/apps/crawler/src/process/sync/tasks.ts @@ -1,13 +1,13 @@ -import { resolveSourceConfig } from "@/utils"; +import logger from "@basango/logger"; import { - closePersistors, CrawlingOptions, + closePersistors, createPersistors, resolveCrawlerConfig, } from "@/process/crawler"; -import logger from "@basango/logger"; -import { WordPressCrawler } from "@/process/parsers/wordpress"; import { HtmlCrawler } from "@/process/parsers/html"; +import { WordPressCrawler } from "@/process/parsers/wordpress"; +import { resolveSourceConfig } from "@/utils"; export const runSyncCrawl = async (options: CrawlingOptions): Promise => { const source = resolveSourceConfig(options.sourceId); diff --git a/basango/apps/crawler/src/schema.ts b/basango/apps/crawler/src/schema.ts index 1483e5a..33d6587 100644 --- a/basango/apps/crawler/src/schema.ts +++ b/basango/apps/crawler/src/schema.ts @@ -5,8 +5,8 @@ export const SourceKindSchema = z.enum(["wordpress", "html"]); export const DateRangeSchema = z .object({ - start: z.number().int(), end: z.number().int(), + start: z.number().int(), }) .superRefine((value, ctx) => { if (value.start === 0 || value.end === 0) { @@ -25,8 +25,8 @@ export const DateRangeSchema = z export const PageRangeSchema = z .object({ - start: z.number().int().min(0), end: z.number().int().min(0), + start: z.number().int().min(0), }) .superRefine((value, ctx) => { if (value.end < value.start) { @@ -43,8 +43,8 @@ export const PageRangeSpecSchema = z .transform((spec) => { const [startText, endText] = spec.split(":"); return { - start: Number.parseInt(String(startText), 10), end: Number.parseInt(String(endText), 10), + start: Number.parseInt(String(startText), 10), }; }); @@ -53,7 +53,7 @@ export const DateRangeSpecSchema = z .regex(/.+:.+/, "Expected start:end format") .transform((spec) => { const [startRaw, endRaw] = spec.split(":"); - return { startRaw: String(startRaw), endRaw: String(endRaw) }; + return { endRaw: String(endRaw), startRaw: String(startRaw) }; }); export const SourceDateSchema = z.object({ @@ -63,57 +63,57 @@ export const SourceDateSchema = z.object({ }); const BaseSourceSchema = z.object({ - sourceId: z.string(), - sourceUrl: z.url(), - sourceDate: SourceDateSchema, - sourceKind: SourceKindSchema, categories: z.array(z.string()).default([]), - supportsCategories: z.boolean().default(false), requiresDetails: z.boolean().default(false), requiresRateLimit: z.boolean().default(false), + sourceDate: SourceDateSchema, + sourceId: z.string(), + sourceKind: SourceKindSchema, + sourceUrl: z.url(), + supportsCategories: z.boolean().default(false), }); export const HtmlSourceConfigSchema = BaseSourceSchema.extend({ + paginationTemplate: z.string(), sourceKind: z.literal("html"), sourceSelectors: z.object({ + articleBody: z.string(), + articleCategories: z.string().optional(), + articleDate: z.string(), + articleLink: z.string(), articles: z.string(), articleTitle: z.string(), - articleLink: z.string(), - articleBody: z.string(), - articleDate: z.string(), - articleCategories: z.string().optional(), pagination: z.string().default("ul.pagination > li a"), }), - paginationTemplate: z.string(), }); export const WordPressSourceConfigSchema = BaseSourceSchema.extend({ - sourceKind: z.literal("wordpress"), sourceDate: SourceDateSchema.default(SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" })), + sourceKind: z.literal("wordpress"), }); export const ArticleMetadataSchema = z.object({ - title: z.string().optional(), description: z.string().optional(), image: z.string().optional(), + title: z.string().optional(), url: z.url().optional(), }); export const ArticleTokenStatisticsSchema = z.object({ - title: z.number().int().nonnegative().default(0), body: z.number().int().nonnegative().default(0), - excerpt: z.number().int().nonnegative().default(0), categories: z.number().int().nonnegative().default(0), + excerpt: z.number().int().nonnegative().default(0), + title: z.number().int().nonnegative().default(0), }); export const ArticleSchema = z.object({ - title: z.string(), - link: z.url(), body: z.string(), categories: z.array(z.string()).default([]), + link: z.url(), + metadata: ArticleMetadataSchema.optional(), source: z.string(), timestamp: z.number().int(), - metadata: ArticleMetadataSchema.optional(), + title: z.string(), tokenStatistics: ArticleTokenStatisticsSchema.optional(), }); diff --git a/basango/apps/crawler/src/scripts/crawl.ts b/basango/apps/crawler/src/scripts/crawl.ts index d92aa6c..b45d2e3 100644 --- a/basango/apps/crawler/src/scripts/crawl.ts +++ b/basango/apps/crawler/src/scripts/crawl.ts @@ -1,6 +1,6 @@ import { logger } from "@basango/logger"; import { runSyncCrawl } from "@/process/sync/tasks"; -import { parseCrawlingCliArgs, CRAWLING_USAGE } from "@/scripts/utils"; +import { CRAWLING_USAGE, parseCrawlingCliArgs } from "@/scripts/utils"; const main = async (): Promise => { const options = parseCrawlingCliArgs(); diff --git a/basango/apps/crawler/src/scripts/queue.ts b/basango/apps/crawler/src/scripts/queue.ts index 364eb77..cc828c3 100644 --- a/basango/apps/crawler/src/scripts/queue.ts +++ b/basango/apps/crawler/src/scripts/queue.ts @@ -1,6 +1,6 @@ import { logger } from "@basango/logger"; import { scheduleAsyncCrawl } from "@/process/async/tasks"; -import { parseCrawlingCliArgs, CRAWLING_USAGE } from "@/scripts/utils"; +import { CRAWLING_USAGE, parseCrawlingCliArgs } from "@/scripts/utils"; const main = async (): Promise => { const options = parseCrawlingCliArgs(); diff --git a/basango/apps/crawler/src/scripts/utils.ts b/basango/apps/crawler/src/scripts/utils.ts index 77bdb0e..bbbe036 100644 --- a/basango/apps/crawler/src/scripts/utils.ts +++ b/basango/apps/crawler/src/scripts/utils.ts @@ -9,8 +9,8 @@ export const CRAWLING_USAGE = ` Usage: bun run crawl:[async|sync] -- --sourceId [options] Options: - --page Optional page range filter (e.g. 1:5) - --date Optional date range filter (e.g. 2024-01-01:2024-01-31) + --pageRange Optional page range filter (e.g. 1:5) + --dateRange Optional date range filter (e.g. 2024-01-01:2024-01-31) --category Optional category to crawl -h, --help Show this message `; @@ -18,7 +18,7 @@ export const CRAWLING_USAGE = ` export const parseWorkerCliArgs = (): WorkerCliOptions => { const { values } = parseArgs({ options: { - queue: { type: "string", multiple: true, short: "q" }, + queue: { multiple: true, short: "q", type: "string" }, }, }); @@ -28,10 +28,10 @@ export const parseWorkerCliArgs = (): WorkerCliOptions => { export const parseCrawlingCliArgs = (): CrawlingOptions => { const { values } = parseArgs({ options: { - sourceId: { type: "string" }, - page: { type: "string" }, - date: { type: "string" }, category: { type: "string" }, + dateRange: { type: "string" }, + pageRange: { type: "string" }, + sourceId: { type: "string" }, }, }); diff --git a/basango/apps/crawler/src/utils.ts b/basango/apps/crawler/src/utils.ts index de6bc02..6af4607 100644 --- a/basango/apps/crawler/src/utils.ts +++ b/basango/apps/crawler/src/utils.ts @@ -1,19 +1,20 @@ +import { format, getUnixTime, isMatch, parse } from "date-fns"; import type { RedisOptions } from "ioredis"; import { get_encoding, TiktokenEncoding } from "tiktoken"; -import { format, getUnixTime, isMatch, parse } from "date-fns"; - +import { config } from "@/config"; +import { DEFAULT_DATE_FORMAT } from "@/constants"; import { AnySourceConfig, CreateDateRangeOptions, DateRange, DateRangeSchema, DateRangeSpecSchema, + HtmlSourceConfig, PageRange, PageRangeSchema, PageRangeSpecSchema, + WordPressSourceConfig, } from "@/schema"; -import { DEFAULT_DATE_FORMAT } from "@/constants"; -import { config } from "@/config"; /** * Resolve a source configuration by its ID. @@ -21,8 +22,8 @@ import { config } from "@/config"; */ export const resolveSourceConfig = (id: string): AnySourceConfig => { const source = - config.sources.html.find((s) => s.sourceId === id) || - config.sources.wordpress.find((s) => s.sourceId === id); + config.sources.html.find((s: HtmlSourceConfig) => s.sourceId === id) || + config.sources.wordpress.find((s: WordPressSourceConfig) => s.sourceId === id); if (source === undefined) { throw new Error(`Source '${id}' not found in configuration`); @@ -41,10 +42,10 @@ export const parseRedisUrl = (url: string): RedisOptions => { } const parsed = new URL(url); return { - host: parsed.hostname, - port: Number(parsed.port || 6379), - password: parsed.password || undefined, db: Number(parsed.pathname?.replace("/", "") || 0), + host: parsed.hostname, + password: parsed.password || undefined, + port: Number(parsed.port || 6379), }; }; @@ -112,8 +113,8 @@ export const createDateRange = ( const endDate = parseDate(parsedSpec.endRaw, format); const range = { - start: getUnixTime(startDate), end: getUnixTime(endDate), + start: getUnixTime(startDate), }; return DateRangeSchema.parse(range); diff --git a/basango/apps/crawler/tsconfig.json b/basango/apps/crawler/tsconfig.json index 66b4caa..d14326b 100644 --- a/basango/apps/crawler/tsconfig.json +++ b/basango/apps/crawler/tsconfig.json @@ -1,11 +1,11 @@ { - "extends": "@basango/tsconfig/base.json", "compilerOptions": { "baseUrl": ".", "paths": { "@/*": ["./src/*"] } }, + "extends": "@basango/tsconfig/base.json", "include": ["src"], "references": [] } diff --git a/basango/apps/crawler/vitest.config.ts b/basango/apps/crawler/vitest.config.ts index 14393f8..34e8a41 100644 --- a/basango/apps/crawler/vitest.config.ts +++ b/basango/apps/crawler/vitest.config.ts @@ -3,15 +3,15 @@ import path from "node:path"; import { defineConfig } from "vitest/config"; export default defineConfig({ + resolve: { + alias: { + "@": path.resolve(__dirname, "src"), + }, + }, test: { environment: "node", globals: true, include: ["src/**/*.test.ts"], setupFiles: ["./vitest.setup.ts"], }, - resolve: { - alias: { - "@": path.resolve(__dirname, "src"), - }, - }, }); diff --git a/basango/biome.json b/basango/biome.json index 77cbd27..ef79fc9 100644 --- a/basango/biome.json +++ b/basango/biome.json @@ -20,7 +20,8 @@ "rules": { "recommended": true, "style": { - "useImportType": "off" + "useImportType": "off", + "noNonNullAssertion": "off" }, "correctness": { "noUnusedImports": "on", diff --git a/basango/bun.lock b/basango/bun.lock index 005b238..82db8c2 100644 --- a/basango/bun.lock +++ b/basango/bun.lock @@ -15,15 +15,18 @@ "name": "@basango/crawler", "dependencies": { "@basango/logger": "workspace:*", - "@devscast/config": "^1.0.2", - "bullmq": "^4.17.0", - "date-fns": "catalog:", - "ioredis": "^5.3.2", + "@devscast/config": "^1.0.3", + "bullmq": "^4.18.3", + "date-fns": "^3.6.0", + "ioredis": "^5.8.2", "node-html-parser": "^7.0.1", - "tiktoken": "^1.0.14", + "tiktoken": "^1.0.22", "turndown": "^7.2.2", - "yaml": "^2.8.1", - "zod": "catalog:", + "zod": "^4.1.12", + }, + "devDependencies": { + "@types/turndown": "^5.0.6", + "vitest": "^4.0.7", }, }, "packages/db": { @@ -92,7 +95,7 @@ "@date-fns/utc": ["@date-fns/utc@2.1.1", "", {}, "sha512-SlJDfG6RPeEX8wEVv6ZB3kak4MmbtyiI2qX/5zuKdordbrhB/iaJ58GVMZgJ6P1sJaM1gMgENFYYeg1JWrCFrA=="], - "@devscast/config": ["@devscast/config@1.0.2", "", { "peerDependencies": { "ini": "^6.0.0", "yaml": "^2.8.1", "zod": "^4.1.12" }, "optionalPeers": ["ini", "yaml"] }, "sha512-1DR8GQogAOrR4B9mtZ24YIKlEZNvKOFeovw+XepfkXVx0MB1f1fAHtPAAXppV7RPMLSyQEMFJzve17x2HbohEw=="], + "@devscast/config": ["@devscast/config@1.0.3", "", { "peerDependencies": { "ini": "^6.0.0", "yaml": "^2.8.1", "zod": "^4.1.12" }, "optionalPeers": ["ini", "yaml"] }, "sha512-/FjCA/MV1KR2tY44YBA4tdXNzQgoF75O+RQ4fbzvVWY77PXOama2Hf6YXeLcQsvxfItaXi2cFz8BaaVdqZYS8w=="], "@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="], @@ -154,6 +157,8 @@ "@ioredis/commands": ["@ioredis/commands@1.4.0", "", {}, "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ=="], + "@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="], + "@manypkg/cli": ["@manypkg/cli@0.25.1", "", { "dependencies": { "@manypkg/get-packages": "^3.1.0", "detect-indent": "^7.0.1", "normalize-path": "^3.0.0", "p-limit": "^6.2.0", "package-json": "^10.0.1", "parse-github-url": "^1.0.3", "picocolors": "^1.1.1", "sembear": "^0.7.0", "semver": "^7.7.1", "tinyexec": "^1.0.1", "validate-npm-package-name": "^6.0.0" }, "bin": { "manypkg": "bin.js" } }, "sha512-lag906FyiNxzZjsRErkUD5/to174I2JzPk5bZubuJp6loMKKJn73zrtqeU7nHlVkHBg3tgXDTJj22HxUDxLRXw=="], "@manypkg/find-root": ["@manypkg/find-root@3.1.0", "", { "dependencies": { "@manypkg/tools": "^2.1.0" } }, "sha512-BcSqCyKhBVZ5YkSzOiheMCV41kqAFptW6xGqYSTjkVTl9XQpr+pqHhwgGCOHQtjDCv7Is6EFyA14Sm5GVbVABA=="], @@ -184,16 +189,86 @@ "@pnpm/npm-conf": ["@pnpm/npm-conf@2.3.1", "", { "dependencies": { "@pnpm/config.env-replace": "^1.1.0", "@pnpm/network.ca-file": "^1.0.1", "config-chain": "^1.1.11" } }, "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw=="], + "@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.52.5", "", { "os": "android", "cpu": "arm" }, "sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ=="], + + "@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.52.5", "", { "os": "android", "cpu": "arm64" }, "sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA=="], + + "@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.52.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA=="], + + "@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.52.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA=="], + + "@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.52.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA=="], + + "@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.52.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ=="], + + "@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ=="], + + "@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ=="], + + "@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg=="], + + "@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q=="], + + "@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA=="], + + "@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.52.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw=="], + + "@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw=="], + + "@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg=="], + + "@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.52.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ=="], + + "@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q=="], + + "@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg=="], + + "@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.52.5", "", { "os": "none", "cpu": "arm64" }, "sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw=="], + + "@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.52.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w=="], + + "@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.52.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg=="], + + "@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ=="], + + "@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg=="], + + "@standard-schema/spec": ["@standard-schema/spec@1.0.0", "", {}, "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA=="], + "@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="], + "@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="], + + "@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="], + + "@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="], + "@types/node": ["@types/node@24.9.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA=="], "@types/pg": ["@types/pg@8.15.6", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ=="], "@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="], + "@types/turndown": ["@types/turndown@5.0.6", "", {}, "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg=="], + + "@vitest/expect": ["@vitest/expect@4.0.7", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.0.7", "@vitest/utils": "4.0.7", "chai": "^6.0.1", "tinyrainbow": "^3.0.3" } }, "sha512-jGRG6HghnJDjljdjYIoVzX17S6uCVCBRFnsgdLGJ6CaxfPh8kzUKe/2n533y4O/aeZ/sIr7q7GbuEbeGDsWv4Q=="], + + "@vitest/mocker": ["@vitest/mocker@4.0.7", "", { "dependencies": { "@vitest/spy": "4.0.7", "estree-walker": "^3.0.3", "magic-string": "^0.30.19" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-OsDwLS7WnpuNslOV6bJkXVYVV/6RSc4eeVxV7h9wxQPNxnjRvTTrIikfwCbMyl8XJmW6oOccBj2Q07YwZtQcCw=="], + + "@vitest/pretty-format": ["@vitest/pretty-format@4.0.7", "", { "dependencies": { "tinyrainbow": "^3.0.3" } }, "sha512-YY//yxqTmk29+/pK+Wi1UB4DUH3lSVgIm+M10rAJ74pOSMgT7rydMSc+vFuq9LjZLhFvVEXir8EcqMke3SVM6Q=="], + + "@vitest/runner": ["@vitest/runner@4.0.7", "", { "dependencies": { "@vitest/utils": "4.0.7", "pathe": "^2.0.3" } }, "sha512-orU1lsu4PxLEcDWfjVCNGIedOSF/YtZ+XMrd1PZb90E68khWCNzD8y1dtxtgd0hyBIQk8XggteKN/38VQLvzuw=="], + + "@vitest/snapshot": ["@vitest/snapshot@4.0.7", "", { "dependencies": { "@vitest/pretty-format": "4.0.7", "magic-string": "^0.30.19", "pathe": "^2.0.3" } }, "sha512-xJL+Nkw0OjaUXXQf13B8iKK5pI9QVtN9uOtzNHYuG/o/B7fIEg0DQ+xOe0/RcqwDEI15rud1k7y5xznBKGUXAA=="], + + "@vitest/spy": ["@vitest/spy@4.0.7", "", {}, "sha512-FW4X8hzIEn4z+HublB4hBF/FhCVaXfIHm8sUfvlznrcy1MQG7VooBgZPMtVCGZtHi0yl3KESaXTqsKh16d8cFg=="], + + "@vitest/utils": ["@vitest/utils@4.0.7", "", { "dependencies": { "@vitest/pretty-format": "4.0.7", "tinyrainbow": "^3.0.3" } }, "sha512-HNrg9CM/Z4ZWB6RuExhuC6FPmLipiShKVMnT9JlQvfhwR47JatWLChA6mtZqVHqypE6p/z6ofcjbyWpM7YLxPQ=="], + "argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="], + "assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="], + "atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="], "balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="], @@ -208,6 +283,8 @@ "bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="], + "chai": ["chai@6.2.0", "", {}, "sha512-aUTnJc/JipRzJrNADXVvpVqi6CO0dn3nx4EVPxijri+fj3LUUDyZQOgVeW54Ob3Y1Xh9Iz8f+CgaCl8v0mn9bA=="], + "change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="], "cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="], @@ -254,10 +331,16 @@ "entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="], + "es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="], + "esbuild": ["esbuild@0.25.11", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.11", "@esbuild/android-arm": "0.25.11", "@esbuild/android-arm64": "0.25.11", "@esbuild/android-x64": "0.25.11", "@esbuild/darwin-arm64": "0.25.11", "@esbuild/darwin-x64": "0.25.11", "@esbuild/freebsd-arm64": "0.25.11", "@esbuild/freebsd-x64": "0.25.11", "@esbuild/linux-arm": "0.25.11", "@esbuild/linux-arm64": "0.25.11", "@esbuild/linux-ia32": "0.25.11", "@esbuild/linux-loong64": "0.25.11", "@esbuild/linux-mips64el": "0.25.11", "@esbuild/linux-ppc64": "0.25.11", "@esbuild/linux-riscv64": "0.25.11", "@esbuild/linux-s390x": "0.25.11", "@esbuild/linux-x64": "0.25.11", "@esbuild/netbsd-arm64": "0.25.11", "@esbuild/netbsd-x64": "0.25.11", "@esbuild/openbsd-arm64": "0.25.11", "@esbuild/openbsd-x64": "0.25.11", "@esbuild/openharmony-arm64": "0.25.11", "@esbuild/sunos-x64": "0.25.11", "@esbuild/win32-arm64": "0.25.11", "@esbuild/win32-ia32": "0.25.11", "@esbuild/win32-x64": "0.25.11" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q=="], "esbuild-register": ["esbuild-register@3.6.0", "", { "dependencies": { "debug": "^4.3.4" }, "peerDependencies": { "esbuild": ">=0.12 <1" } }, "sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg=="], + "estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="], + + "expect-type": ["expect-type@1.2.2", "", {}, "sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA=="], + "fast-copy": ["fast-copy@3.0.2", "", {}, "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="], "fast-safe-stringify": ["fast-safe-stringify@2.1.1", "", {}, "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="], @@ -266,6 +349,8 @@ "fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="], + "fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="], + "get-tsconfig": ["get-tsconfig@4.13.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ=="], "glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="], @@ -300,6 +385,8 @@ "luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="], + "magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="], + "map-obj": ["map-obj@5.0.2", "", {}, "sha512-K6K2NgKnTXimT3779/4KxSvobxOtMmx1LBZ3NwRxT/MDIR3Br/fQ4Q+WCX5QxjyUR8zg5+RV9Tbf2c5pAWTD2A=="], "minimatch": ["minimatch@5.1.6", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g=="], @@ -312,6 +399,8 @@ "msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="], + "nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="], + "node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="], "node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="], @@ -332,6 +421,8 @@ "parse-github-url": ["parse-github-url@1.0.3", "", { "bin": { "parse-github-url": "cli.js" } }, "sha512-tfalY5/4SqGaV/GIGzWyHnFjlpTPTNpENR9Ea2lLldSJ8EWXMsvacWucqY3m3I4YPtas15IxTLQVQ5NSYXPrww=="], + "pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="], + "pg": ["pg@8.16.3", "", { "dependencies": { "pg-connection-string": "^2.9.1", "pg-pool": "^3.10.1", "pg-protocol": "^1.10.3", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.2.7" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw=="], "pg-cloudflare": ["pg-cloudflare@1.2.7", "", {}, "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg=="], @@ -360,6 +451,8 @@ "pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="], + "postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="], + "postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="], "postgres-bytea": ["postgres-bytea@1.0.0", "", {}, "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w=="], @@ -390,6 +483,8 @@ "resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="], + "rollup": ["rollup@4.52.5", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.52.5", "@rollup/rollup-android-arm64": "4.52.5", "@rollup/rollup-darwin-arm64": "4.52.5", "@rollup/rollup-darwin-x64": "4.52.5", "@rollup/rollup-freebsd-arm64": "4.52.5", "@rollup/rollup-freebsd-x64": "4.52.5", "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", "@rollup/rollup-linux-arm-musleabihf": "4.52.5", "@rollup/rollup-linux-arm64-gnu": "4.52.5", "@rollup/rollup-linux-arm64-musl": "4.52.5", "@rollup/rollup-linux-loong64-gnu": "4.52.5", "@rollup/rollup-linux-ppc64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-musl": "4.52.5", "@rollup/rollup-linux-s390x-gnu": "4.52.5", "@rollup/rollup-linux-x64-gnu": "4.52.5", "@rollup/rollup-linux-x64-musl": "4.52.5", "@rollup/rollup-openharmony-arm64": "4.52.5", "@rollup/rollup-win32-arm64-msvc": "4.52.5", "@rollup/rollup-win32-ia32-msvc": "4.52.5", "@rollup/rollup-win32-x64-gnu": "4.52.5", "@rollup/rollup-win32-x64-msvc": "4.52.5", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw=="], + "safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="], "secure-json-parse": ["secure-json-parse@4.1.0", "", {}, "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA=="], @@ -398,28 +493,40 @@ "semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="], + "siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="], + "snakecase-keys": ["snakecase-keys@9.0.2", "", { "dependencies": { "change-case": "^5.4.4", "map-obj": "^5.0.2", "type-fest": "^4.15.0" } }, "sha512-Tr4gONsDj1Pa6HJH9D3b411r6tuRyCGgb1l7YpzDFp/thjVSWs7rcbNjyTyRqJi5SUV23sFpzf9epIJRbLR6Yw=="], "sonic-boom": ["sonic-boom@4.2.0", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww=="], "source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="], + "source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="], + "source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="], "split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="], + "stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="], + "standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="], + "std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="], + "strip-json-comments": ["strip-json-comments@5.0.3", "", {}, "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw=="], "thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="], "tiktoken": ["tiktoken@1.0.22", "", {}, "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA=="], + "tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="], + "tinyexec": ["tinyexec@1.0.1", "", {}, "sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw=="], "tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="], + "tinyrainbow": ["tinyrainbow@3.0.3", "", {}, "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q=="], + "tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], "turbo": ["turbo@2.5.8", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.8", "turbo-darwin-arm64": "2.5.8", "turbo-linux-64": "2.5.8", "turbo-linux-arm64": "2.5.8", "turbo-windows-64": "2.5.8", "turbo-windows-arm64": "2.5.8" }, "bin": { "turbo": "bin/turbo" } }, "sha512-5c9Fdsr9qfpT3hA0EyYSFRZj1dVVsb6KIWubA9JBYZ/9ZEAijgUEae0BBR/Xl/wekt4w65/lYLTFaP3JmwSO8w=="], @@ -448,6 +555,12 @@ "validate-npm-package-name": ["validate-npm-package-name@6.0.2", "", {}, "sha512-IUoow1YUtvoBBC06dXs8bR8B9vuA3aJfmQNKMoaPG/OFsPmoQvw8xh+6Ye25Gx9DQhoEom3Pcu9MKHerm/NpUQ=="], + "vite": ["vite@7.2.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-BxAKBWmIbrDgrokdGZH1IgkIk/5mMHDreLDmCJ0qpyJaAteP8NvMhkwr/ZCQNqNH97bw/dANTE9PDzqwJghfMQ=="], + + "vitest": ["vitest@4.0.7", "", { "dependencies": { "@vitest/expect": "4.0.7", "@vitest/mocker": "4.0.7", "@vitest/pretty-format": "4.0.7", "@vitest/runner": "4.0.7", "@vitest/snapshot": "4.0.7", "@vitest/spy": "4.0.7", "@vitest/utils": "4.0.7", "debug": "^4.4.3", "es-module-lexer": "^1.7.0", "expect-type": "^1.2.2", "magic-string": "^0.30.19", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.0.3", "vite": "^6.0.0 || ^7.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.0.7", "@vitest/browser-preview": "4.0.7", "@vitest/browser-webdriverio": "4.0.7", "@vitest/ui": "4.0.7", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-xQroKAadK503CrmbzCISvQUjeuvEZzv6U0wlnlVFOi5i3gnzfH4onyQ29f3lzpe0FresAiTAd3aqK0Bi/jLI8w=="], + + "why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="], + "wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="], "xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="], @@ -462,6 +575,8 @@ "rc/strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="], + "vitest/tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="], + "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="], "@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="], diff --git a/basango/packages/logger/src/index.ts b/basango/packages/logger/src/index.ts index 85cee15..c6c59f2 100644 --- a/basango/packages/logger/src/index.ts +++ b/basango/packages/logger/src/index.ts @@ -3,7 +3,7 @@ import pino from "pino"; export const logger = pino({ level: process.env.LOG_LEVEL || "info", // Use pretty printing in development, structured JSON in production - ...(process.env.NODE_ENV === "development" && { + ...(process.env.NODE_ENV !== "production" && { transport: { target: "pino-pretty", options: { diff --git a/basango/packages/tsconfig/base.json b/basango/packages/tsconfig/base.json index f389323..0bb2ea1 100644 --- a/basango/packages/tsconfig/base.json +++ b/basango/packages/tsconfig/base.json @@ -8,7 +8,7 @@ "incremental": false, "isolatedModules": true, "lib": ["es2022", "DOM", "DOM.Iterable"], - "module": "NodeNext", + "module": "esnext", "moduleDetection": "force", "moduleResolution": "node", "forceConsistentCasingInFileNames": true,