bernard-ng/basango
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

[crawler] full migration to typescript

Browse Source
This commit is contained in:
bernard-ng
2025-11-07 13:28:46 +02:00
committed by BernardNganduDev
parent 13dd1b09ee
commit 1c478ae443
28 changed files with 468 additions and 304 deletions
Download Patch File Download Diff File Expand all files Collapse all files
basango/apps/crawler/config/pipeline.json
+31 -32
View File
@@ -1,42 +1,41 @@
{
"paths": {
"root": "%env(BASANGO_CRAWLER_ROOT_PATH)%",
"data": "%env(BASANGO_CRAWLER_DATA_PATH)%",
"logs": "%env(BASANGO_CRAWLER_LOGS_PATH)%",
"config": "%env(BASANGO_CRAWLER_CONFIG_PATH)%"
},
"fetch": {
"client": {
"timeout": 20,
"userAgent": "%env(BASANGO_CRAWLER_FETCH_USER_AGENT)%",
"followRedirects": true,
"verifySsl": true,
"rotate": true,
"maxRetries": "%env(BASANGO_CRAWLER_FETCH_MAX_RETRIES)%",
"backoffInitial": 1,
"backoffMultiplier": 2,
"backoffMax": 30,
"respectRetryAfter": "%env(BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER)%"
},
"crawler": {
"notify": false,
"useMultiThreading": false,
"maxWorkers": 5,
"direction": "%env(BASANGO_CRAWLER_UPDATE_DIRECTION)%"
},
"async": {
"redisUrl": "%env(BASANGO_CRAWLER_ASYNC_REDIS_URL)%",
"prefix": "basango:crawler",
"queues": {
"details": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS)%",
"listing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_LISTING)%",
"processing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING)%"
},
"redisUrl": "%env(BASANGO_CRAWLER_ASYNC_REDIS_URL)%",
"ttl": {
"default": 600,
"result": "%env(BASANGO_CRAWLER_ASYNC_TTL_RESULT)%",
"failure": "%env(BASANGO_CRAWLER_ASYNC_TTL_FAILURE)%"
},
"queues": {
"listing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_LISTING)%",
"details": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS)%",
"processing": "%env(BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING)%"
"failure": "%env(number:BASANGO_CRAWLER_ASYNC_TTL_FAILURE)%",
"result": "%env(number:BASANGO_CRAWLER_ASYNC_TTL_RESULT)%"
}
},
"client": {
"backoffInitial": 1,
"backoffMax": 30,
"backoffMultiplier": 2,
"followRedirects": true,
"maxRetries": "%env(number:BASANGO_CRAWLER_FETCH_MAX_RETRIES)%",
"respectRetryAfter": "%env(boolean:BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER)%",
"rotate": true,
"timeout": 20,
"userAgent": "%env(BASANGO_CRAWLER_FETCH_USER_AGENT)%",
"verifySsl": true
},
"crawler": {
"direction": "%env(BASANGO_CRAWLER_UPDATE_DIRECTION)%",
"maxWorkers": 5,
"notify": false,
"useMultiThreading": false
}
},
"paths": {
"config": "%env(BASANGO_CRAWLER_CONFIG_PATH)%",
"data": "%env(BASANGO_CRAWLER_DATA_PATH)%",
"root": "%env(BASANGO_CRAWLER_ROOT_PATH)%"
}
}
basango/apps/crawler/config/sources.json
+75 -47
View File
@@ -2,188 +2,216 @@
"sources": {
"html": [
{
"sourceId": "radiookapi.net",
"sourceUrl": "https://www.radiookapi.net",
"paginationTemplate": "actualite",
"requiresDetails": true,
"requiresRateLimit": false,
"sourceDate": {
"pattern": "/(\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
"replacement": "$3-$2-$1 $4"
},
"sourceId": "radiookapi.net",
"sourceKind": "html",
"sourceSelectors": {
"articleBody": ".field-name-body",
"articleCategories": ".views-field-field-cat-gorie a",
"articleDate": ".views-field-created",
"articleLink": ".views-field-title a",
"articles": ".view-content > .views-row.content-row",
"articleTitle": "h1.page-header",
"articleLink": ".views-field-title a",
"articleBody": ".field-name-body",
"articleDate": ".views-field-created",
"articleCategories": ".views-field-field-cat-gorie a",
"pagination": "ul.pagination > li.pager-last > a"
},
"paginationTemplate": "actualite",
"supportsCategories": false,
"requiresDetails": true,
"requiresRateLimit": false
"sourceUrl": "https://www.radiookapi.net",
"supportsCategories": false
},
{
"sourceId": "7sur7.cd",
"sourceUrl": "https://7sur7.cd",
"categories": ["politique", "economie", "culture", "sport", "societe"],
"paginationTemplate": "index.php/category/{category}",
"requiresDetails": false,
"requiresRateLimit": false,
"sourceDate": {
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
"replacement": "$3-$2-$1 $4"
},
"categories": ["politique", "economie", "culture", "sport", "societe"],
"sourceId": "7sur7.cd",
"sourceKind": "html",
"sourceSelectors": {
"articles": ".view-content > .row.views-row",
"articleTitle": ".views-field-title a",
"articleLink": ".views-field-title a",
"articleBody": ".field.field--name-body",
"articleDate": ".views-field-created",
"articleLink": ".views-field-title a",
"articles": ".view-content > .row.views-row",
"articleTitle": ".views-field-title a",
"pagination": "ul.pagination > li.pager__item.pager__item--last > a"
},
"paginationTemplate": "index.php/category/{category}",
"supportsCategories": true,
"requiresDetails": false,
"requiresRateLimit": false
"sourceUrl": "https://7sur7.cd",
"supportsCategories": true
},
{
"sourceId": "mediacongo.net",
"sourceUrl": "https://www.mediacongo.net",
"paginationTemplate": "articles.html",
"requiresDetails": true,
"requiresRateLimit": false,
"sourceDate": {
"format": "%d.%m.%Y %H:%M"
},
"sourceId": "mediacongo.net",
"sourceKind": "html",
"sourceSelectors": {
"articles": ".for_aitems > .article_other_item",
"articleTitle": "img",
"articleLink": "a:first-child",
"articleCategories": "a.color_link",
"articleBody": ".article_ttext",
"articleCategories": "a.color_link",
"articleDate": ".article_other_about",
"articleLink": "a:first-child",
"articles": ".for_aitems > .article_other_item",
"articleTitle": "h1",
"pagination": "div.pagination > div > a:last-child"
},
"paginationTemplate": "articles.html",
"supportsCategories": false,
"requiresDetails": true,
"requiresRateLimit": false
"sourceUrl": "https://www.mediacongo.net",
"supportsCategories": false
},
{
"sourceId": "actualite.cd",
"sourceUrl": "https://actualite.cd",
"paginationTemplate": "actualite",
"requiresDetails": true,
"requiresRateLimit": false,
"sourceDate": {
"pattern": "/(\\d{1}) (\\d{1,2}) (\\d{2}) (\\d{4}) - (\\d{2}:\\d{2})/",
"replacement": "$4-$3-$2 $5"
},
"sourceId": "actualite.cd",
"sourceKind": "html",
"sourceSelectors": {
"articles": "#views-bootstrap-taxonomy-term-page-2 > div > div",
"articleTitle": "#actu-titre a",
"articleLink": "#actu-titre a",
"articleCategories": "#actu-cat a",
"articleBody": ".views-field.views-field-body",
"articleDate": "#p-date"
"articleCategories": "#actu-cat",
"articleDate": "#p-date",
"articleLink": "#actu-titre a",
"articles": "#views-bootstrap-taxonomy-term-page-2 > div > div",
"articleTitle": "h1.page-title"
},
"paginationTemplate": "actualite",
"supportsCategories": false,
"requiresDetails": true,
"requiresRateLimit": false
"sourceUrl": "https://actualite.cd",
"supportsCategories": false
}
],
"wordpress": [
{
"requiresRateLimit": true,
"sourceId": "beto.cd",
"sourceUrl": "https://beto.cd",
"requiresRateLimit": true
"sourceKind": "wordpress",
"sourceUrl": "https://beto.cd"
},
{ "sourceId": "newscd.net", "sourceUrl": "https://newscd.net" },
{ "sourceId": "newscd.net", "sourceKind": "wordpress", "sourceUrl": "https://newscd.net" },
{
"sourceId": "africanewsrdc.net",
"sourceKind": "wordpress",
"sourceUrl": "https://www.africanewsrdc.net"
},
{
"sourceId": "angazainstitute.ac.cd",
"sourceKind": "wordpress",
"sourceUrl": "https://angazainstitute.ac.cd"
},
{ "sourceId": "b-onetv.cd", "sourceUrl": "https://b-onetv.cd" },
{ "sourceId": "b-onetv.cd", "sourceKind": "wordpress", "sourceUrl": "https://b-onetv.cd" },
{
"sourceId": "bukavufm.com",
"sourceKind": "wordpress",
"sourceUrl": "https://bukavufm.com"
},
{
"sourceId": "changement7.net",
"sourceKind": "wordpress",
"sourceUrl": "https://changement7.net"
},
{
"sourceId": "congoactu.net",
"sourceKind": "wordpress",
"sourceUrl": "https://congoactu.net"
},
{
"sourceId": "congoindependant.com",
"sourceKind": "wordpress",
"sourceUrl": "https://www.congoindependant.com"
},
{
"sourceId": "congoquotidien.com",
"sourceKind": "wordpress",
"sourceUrl": "https://www.congoquotidien.com"
},
{
"sourceId": "cumulard.cd",
"sourceKind": "wordpress",
"sourceUrl": "https://www.cumulard.cd"
},
{
"sourceId": "environews-rdc.net",
"sourceKind": "wordpress",
"sourceUrl": "https://environews-rdc.net"
},
{
"sourceId": "freemediardc.info",
"sourceKind": "wordpress",
"sourceUrl": "https://www.freemediardc.info"
},
{
"sourceId": "geopolismagazine.org",
"sourceKind": "wordpress",
"sourceUrl": "https://geopolismagazine.org"
},
{
"sourceId": "habarirdc.net",
"sourceKind": "wordpress",
"sourceUrl": "https://habarirdc.net"
},
{ "sourceId": "infordc.com", "sourceUrl": "https://infordc.com" },
{ "sourceId": "infordc.com", "sourceKind": "wordpress", "sourceUrl": "https://infordc.com" },
{
"sourceId": "kilalopress.net",
"sourceKind": "wordpress",
"sourceUrl": "https://kilalopress.net"
},
{
"sourceId": "laprosperiteonline.net",
"sourceKind": "wordpress",
"sourceUrl": "https://laprosperiteonline.net"
},
{
"sourceId": "laprunellerdc.cd",
"sourceKind": "wordpress",
"sourceUrl": "https://laprunellerdc.cd"
},
{
"sourceId": "lesmedias.net",
"sourceKind": "wordpress",
"sourceUrl": "https://lesmedias.net"
},
{
"sourceId": "lesvolcansnews.net",
"sourceKind": "wordpress",
"sourceUrl": "https://lesvolcansnews.net"
},
{
"sourceId": "netic-news.net",
"sourceKind": "wordpress",
"sourceUrl": "https://www.netic-news.net"
},
{
"sourceId": "objectif-infos.cd",
"sourceKind": "wordpress",
"sourceUrl": "https://objectif-infos.cd"
},
{
"sourceId": "scooprdc.net",
"sourceKind": "wordpress",
"sourceUrl": "https://scooprdc.net"
},
{
"sourceId": "journaldekinshasa.com",
"sourceKind": "wordpress",
"sourceUrl": "https://www.journaldekinshasa.com"
},
{
"sourceId": "lepotentiel.cd",
"sourceKind": "wordpress",
"sourceUrl": "https://lepotentiel.cd"
},
{ "sourceId": "acturdc.com", "sourceUrl": "https://acturdc.com" },
{ "sourceId": "acturdc.com", "sourceKind": "wordpress", "sourceUrl": "https://acturdc.com" },
{
"sourceId": "matininfos.net",
"sourceKind": "wordpress",
"sourceUrl": "https://matininfos.net"
}
]
basango/apps/crawler/package.json
+22 -20
View File
@@ -1,28 +1,30 @@
{
"name": "@basango/crawler",
"private": true,
"type": "module",
"scripts": {
"=========== CODE STYLE ============": "",
"test": "vitest --run",
"lint": "biome check .",
"lint:fix": "biome check --write .",
"format": "biome format --write .",
"============= CLI =============": "",
"crawl:sync": "bun run src/scripts/crawl.ts",
"crawl:async": "bun run src/scripts/queue.ts",
"crawl:worker": "bun run src/scripts/worker.ts"
},
"dependencies": {
"@basango/logger": "workspace:*",
"@devscast/config": "^1.0.2",
"bullmq": "^4.17.0",
"date-fns": "catalog:",
"ioredis": "^5.3.2",
"@devscast/config": "^1.0.3",
"bullmq": "^4.18.3",
"date-fns": "^3.6.0",
"ioredis": "^5.8.2",
"node-html-parser": "^7.0.1",
"tiktoken": "^1.0.14",
"tiktoken": "^1.0.22",
"turndown": "^7.2.2",
"yaml": "^2.8.1",
"zod": "catalog:"
}
"zod": "^4.1.12"
},
"devDependencies": {
"@types/turndown": "^5.0.6",
"vitest": "^4.0.7"
},
"scripts": {
"crawler:async": "bun run src/scripts/queue.ts",
"crawler:sync": "bun run src/scripts/crawl.ts",
"crawler:worker": "bun run src/scripts/worker.ts",
"format": "biome format --write .",
"lint": "biome check .",
"lint:fix": "biome check --write .",
"test": "vitest --run",
"typecheck": "tsc --noEmit"
},
"type": "module"
}
basango/apps/crawler/src/config.ts
+38 -38
View File
@@ -1,6 +1,6 @@
import path from "node:path";
import { loadConfig } from "@devscast/config";
import { loadConfig as defineConfig } from "@devscast/config";
import { z } from "zod";
import {
DateRangeSchema,
@@ -13,49 +13,49 @@ import {
export const PROJECT_DIR = path.resolve(__dirname, "../");
export const PipelineConfigSchema = z.object({
paths: z.object({
root: z.string().default(PROJECT_DIR),
data: z.string().default(path.join(PROJECT_DIR, "data", "dataset")),
config: z.string().default(path.join(PROJECT_DIR, "config")),
}),
fetch: z.object({
client: z.object({
timeout: z.number().positive().default(20),
userAgent: z.string().default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
followRedirects: z.boolean().default(true),
verifySsl: z.boolean().default(true),
rotate: z.boolean().default(true),
maxRetries: z.number().int().nonnegative().default(3),
backoffInitial: z.number().nonnegative().default(1),
backoffMultiplier: z.number().positive().default(2),
backoffMax: z.number().nonnegative().default(30),
respectRetryAfter: z.boolean().default(true),
}),
crawler: z.object({
source: z.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema]).optional(),
pageRange: PageRangeSchema.optional(),
dateRange: DateRangeSchema.optional(),
category: z.string().optional(),
notify: z.boolean().default(false),
isUpdate: z.boolean().default(false),
useMultiThreading: z.boolean().default(false),
maxWorkers: z.number().int().positive().default(5),
direction: UpdateDirectionSchema.default("forward"),
}),
async: z.object({
redisUrl: z.string().default("redis://localhost:6379/0"),
prefix: z.string().default("basango:crawler:queue"),
ttl: z.object({
default: z.number().int().positive().default(600),
result: z.number().int().nonnegative().default(3600),
failure: z.number().int().nonnegative().default(3600),
}),
queues: z.object({
listing: z.string().default("listing"),
details: z.string().default("details"),
listing: z.string().default("listing"),
processing: z.string().default("processing"),
}),
redisUrl: z.string().default("redis://localhost:6379/0"),
ttl: z.object({
default: z.number().int().positive().default(600),
failure: z.number().int().nonnegative().default(3600),
result: z.number().int().nonnegative().default(3600),
}),
}),
client: z.object({
backoffInitial: z.number().nonnegative().default(1),
backoffMax: z.number().nonnegative().default(30),
backoffMultiplier: z.number().positive().default(2),
followRedirects: z.boolean().default(true),
maxRetries: z.number().int().nonnegative().default(3),
respectRetryAfter: z.boolean().default(true),
rotate: z.boolean().default(true),
timeout: z.number().positive().default(20),
userAgent: z.string().default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
verifySsl: z.boolean().default(true),
}),
crawler: z.object({
category: z.string().optional(),
dateRange: DateRangeSchema.optional(),
direction: UpdateDirectionSchema.default("forward"),
isUpdate: z.boolean().default(false),
maxWorkers: z.number().int().positive().default(5),
notify: z.boolean().default(false),
pageRange: PageRangeSchema.optional(),
source: z.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema]).optional(),
useMultiThreading: z.boolean().default(false),
}),
}),
paths: z.object({
config: z.string().default(path.join(PROJECT_DIR, "config")),
data: z.string().default(path.join(PROJECT_DIR, "data", "datasets")),
root: z.string().default(PROJECT_DIR),
}),
sources: z.object({
html: z.array(HtmlSourceConfigSchema).default([]),
@@ -63,12 +63,12 @@ export const PipelineConfigSchema = z.object({
}),
});
export const { config, env } = loadConfig({
schema: PipelineConfigSchema,
export const { config, env } = defineConfig({
cwd: process.cwd(),
env: {
path: path.join(PROJECT_DIR, ".env"),
},
schema: PipelineConfigSchema,
sources: [
path.join(PROJECT_DIR, "config", "pipeline.json"),
path.join(PROJECT_DIR, "config", "sources.json"),
basango/apps/crawler/src/http/http-client.ts
+5 -6
View File
@@ -1,12 +1,11 @@
import { setTimeout as delay } from "node:timers/promises";
import { FetchClientConfig } from "@/config";
import {
DEFAULT_RETRY_AFTER_HEADER,
DEFAULT_USER_AGENT,
TRANSIENT_HTTP_STATUSES,
} from "@/constants";
import { UserAgents } from "@/http/user-agent";
import { FetchClientConfig } from "@/config";
export type HttpHeaders = Record<string, string>;
export type HttpParams = Record<string, string | number | boolean | null | undefined>;
@@ -72,7 +71,7 @@ const buildUrl = (url: string, params?: HttpParams): string => {
*/
const computeBackoff = (config: FetchClientConfig, attempt: number): number => {
const base = Math.min(
config.backoffInitial * Math.pow(config.backoffMultiplier, attempt),
config.backoffInitial * config.backoffMultiplier ** attempt,
config.backoffMax,
);
const jitter = Math.random() * base * 0.25;
@@ -172,11 +171,11 @@ export class SyncHttpClient extends BaseHttpClient {
const headers = this.buildHeaders(options.headers);
const init: RequestInit = {
method,
headers,
body: options.data as BodyInit | undefined,
signal: controller.signal,
headers,
method,
redirect: this.config.followRedirects ? "follow" : "manual",
signal: controller.signal,
};
if (options.json !== undefined) {
basango/apps/crawler/src/http/open-graph.ts
+3 -4
View File
@@ -1,9 +1,8 @@
import { parse } from "node-html-parser";
import { config } from "@/config";
import { OPEN_GRAPH_USER_AGENT } from "@/constants";
import { SyncHttpClient } from "@/http/http-client";
import { UserAgents } from "@/http/user-agent";
import { config } from "@/config";
import { ArticleMetadata } from "@/schema";
/**
@@ -47,8 +46,8 @@ export class OpenGraph {
const provider = new UserAgents(true, OPEN_GRAPH_USER_AGENT);
this.client = new SyncHttpClient(settings, {
userAgentProvider: provider,
defaultHeaders: { "User-Agent": provider.og() },
userAgentProvider: provider,
});
}
@@ -94,9 +93,9 @@ export class OpenGraph {
}
return {
title,
description,
image,
title,
url: canonical,
};
}
basango/apps/crawler/src/process/async/handlers.ts
+13 -14
View File
@@ -1,20 +1,19 @@
import { logger } from "@basango/logger";
import { config, env } from "@/config";
import { Article, HtmlSourceConfig, SourceKindSchema, WordPressSourceConfig } from "@/schema";
import { createDateRange, formatDateRange, formatPageRange, resolveSourceConfig } from "@/utils";
import { SyncHttpClient } from "@/http/http-client";
import { createQueueManager, QueueManager } from "@/process/async/queue";
import {
DetailsTaskPayload,
ListingTaskPayload,
ProcessingTaskPayload,
} from "@/process/async/schemas";
import { createQueueManager, QueueManager } from "@/process/async/queue";
import { resolveCrawlerConfig } from "@/process/crawler";
import { HtmlCrawler } from "@/process/parsers/html";
import { WordPressCrawler } from "@/process/parsers/wordpress";
import { JsonlPersistor } from "@/process/persistence";
import { SyncHttpClient } from "@/http/http-client";
import { resolveCrawlerConfig } from "@/process/crawler";
import { Article, HtmlSourceConfig, SourceKindSchema, WordPressSourceConfig } from "@/schema";
import { createDateRange, formatDateRange, formatPageRange, resolveSourceConfig } from "@/utils";
export const collectHtmlListing = async (
payload: ListingTaskPayload,
@@ -40,10 +39,10 @@ export const collectHtmlListing = async (
if (!url) continue;
await manager.enqueueArticle({
url,
sourceId: payload.sourceId,
category: payload.category,
dateRange: createDateRange(payload.dateRange),
sourceId: payload.sourceId,
url,
} as DetailsTaskPayload);
queued += 1;
}
@@ -79,11 +78,11 @@ export const collectWordPressListing = async (
if (!url) continue;
await manager.enqueueArticle({
url,
data,
sourceId: payload.sourceId,
category: payload.category,
data,
dateRange: createDateRange(payload.dateRange),
sourceId: payload.sourceId,
url,
} as DetailsTaskPayload);
queued += 1;
}
@@ -98,10 +97,10 @@ export const collectWordPressListing = async (
export const collectArticle = async (payload: DetailsTaskPayload): Promise<unknown> => {
const source = resolveSourceConfig(payload.sourceId);
const settings = resolveCrawlerConfig(source, {
pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined,
dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined,
sourceId: payload.sourceId,
category: payload.category,
dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined,
pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined,
sourceId: payload.sourceId,
});
const persistors = [
new JsonlPersistor({
basango/apps/crawler/src/process/async/queue.ts
+21 -15
View File
@@ -1,8 +1,7 @@
import { randomUUID } from "node:crypto";
import IORedis from "ioredis";
import { JobsOptions, Queue, QueueOptions } from "bullmq";
import IORedis from "ioredis";
import { config, FetchAsyncConfig } from "@/config";
import {
DetailsTaskPayload,
DetailsTaskPayloadSchema,
@@ -12,7 +11,6 @@ import {
ProcessingTaskPayloadSchema,
} from "@/process/async/schemas";
import { parseRedisUrl } from "@/utils";
import { config, FetchAsyncConfig } from "@/config";
export interface QueueBackend<T = unknown> {
add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>;
@@ -26,7 +24,11 @@ export type QueueFactory = (
const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => {
const redisConnection =
connection ?? new IORedis(settings.redisUrl, parseRedisUrl(settings.redisUrl));
connection ??
new IORedis(settings.redisUrl, {
...parseRedisUrl(settings.redisUrl),
maxRetriesPerRequest: null,
});
const options: QueueOptions = {
connection: redisConnection,
prefix: settings.prefix,
@@ -65,24 +67,30 @@ export const createQueueManager = (options: CreateQueueManagerOptions = {}): Que
const settings = config.fetch.async;
const connection =
options.connection ?? new IORedis(settings.redisUrl, parseRedisUrl(settings.redisUrl));
options.connection ??
new IORedis(settings.redisUrl, {
...parseRedisUrl(settings.redisUrl),
maxRetriesPerRequest: null,
});
const factory = options.queueFactory ?? defaultQueueFactory;
const ensureQueue = (queueName: string) => factory(queueName, settings, connection);
return {
settings,
connection,
enqueueListing: (payload) => {
const data = ListingTaskPayloadSchema.parse(payload);
const queue = ensureQueue(settings.queues.listing);
return queue.add("collect_listing", data);
close: async () => {
await connection.quit();
},
connection,
enqueueArticle: (payload) => {
const data = DetailsTaskPayloadSchema.parse(payload);
const queue = ensureQueue(settings.queues.details);
return queue.add("collect_article", data);
},
enqueueListing: (payload) => {
const data = ListingTaskPayloadSchema.parse(payload);
const queue = ensureQueue(settings.queues.listing);
return queue.add("collect_listing", data);
},
enqueueProcessed: (payload) => {
const data = ProcessingTaskPayloadSchema.parse(payload);
const queue = ensureQueue(settings.queues.processing);
@@ -94,8 +102,6 @@ export const createQueueManager = (options: CreateQueueManagerOptions = {}): Que
`${settings.prefix}:${settings.queues.processing}`,
],
queueName: (suffix: string) => `${settings.prefix}:${suffix}`,
close: async () => {
await connection.quit();
},
settings,
};
};
basango/apps/crawler/src/process/async/schemas.ts
+8 -8
View File
@@ -2,25 +2,25 @@ import { z } from "zod";
import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "@/schema";
export const ListingTaskPayloadSchema = z.object({
sourceId: z.string(),
pageRange: z.string().optional(),
dateRange: z.string().optional(),
category: z.string().optional(),
dateRange: z.string().optional(),
pageRange: z.string().optional(),
sourceId: z.string(),
});
export const DetailsTaskPayloadSchema = z.object({
sourceId: z.string(),
url: z.url(),
category: z.string().optional(),
data: z.any().optional(),
dateRange: DateRangeSchema.optional(),
page: z.number().int().nonnegative().optional(),
pageRange: PageRangeSchema.optional(),
dateRange: DateRangeSchema.optional(),
category: z.string().optional(),
sourceId: z.string(),
url: z.url(),
});
export const ProcessingTaskPayloadSchema = z.object({
sourceId: z.string(),
article: ArticleSchema,
sourceId: z.string(),
});
export type ListingTaskPayload = z.infer<typeof ListingTaskPayloadSchema>;
basango/apps/crawler/src/process/async/tasks.ts
+5 -6
View File
@@ -1,12 +1,11 @@
import { logger } from "@basango/logger";
import * as handlers from "@/process/async/handlers";
import { createQueueManager } from "@/process/async/queue";
import {
DetailsTaskPayloadSchema,
ListingTaskPayloadSchema,
ProcessingTaskPayloadSchema,
} from "@/process/async/schemas";
import { createQueueManager } from "@/process/async/queue";
import * as handlers from "@/process/async/handlers";
import { CrawlingOptions } from "@/process/crawler";
export const collectListing = async (payload: unknown): Promise<number> => {
@@ -41,10 +40,10 @@ export const forwardForProcessing = async (payload: unknown): Promise<unknown> =
export const scheduleAsyncCrawl = async (options: CrawlingOptions): Promise<string> => {
const payload = ListingTaskPayloadSchema.parse({
sourceId: options.sourceId,
pageRange: options.pageRange,
dateRange: options.dateRange,
category: options.category,
dateRange: options.dateRange,
pageRange: options.pageRange,
sourceId: options.sourceId,
});
const manager = createQueueManager();
basango/apps/crawler/src/process/async/worker.ts
+4 -4
View File
@@ -1,5 +1,5 @@
import IORedis from "ioredis";
import { QueueEvents, Worker } from "bullmq";
import IORedis from "ioredis";
import { QueueFactory, QueueManager } from "@/process/async/queue";
import { collectArticle, collectListing, forwardForProcessing } from "@/process/async/tasks";
@@ -43,8 +43,8 @@ export const startWorker = (options: WorkerOptions): WorkerHandle => {
}
},
{
connection,
concurrency: options.concurrency ?? 5,
connection,
},
);
@@ -60,8 +60,6 @@ export const startWorker = (options: WorkerOptions): WorkerHandle => {
}
return {
workers,
events,
close: async () => {
await Promise.all(workers.map((worker) => worker.close()));
await Promise.all(events.map((event) => event.close()));
@@ -70,5 +68,7 @@ export const startWorker = (options: WorkerOptions): WorkerHandle => {
await manager.close();
}
},
events,
workers,
};
};
basango/apps/crawler/src/process/crawler.ts
+3 -3
View File
@@ -1,7 +1,7 @@
import logger from "@basango/logger";
import { config, FetchCrawlerConfig } from "@/config";
import { JsonlPersistor, Persistor } from "@/process/persistence";
import { AnySourceConfig } from "@/schema";
import logger from "@basango/logger";
import { createDateRange, createPageRange } from "@/utils";
export interface CrawlingOptions {
@@ -17,10 +17,10 @@ export const resolveCrawlerConfig = (
): FetchCrawlerConfig => {
return {
...config.fetch.crawler,
source,
category: options.category,
dateRange: createDateRange(options.dateRange),
pageRange: createPageRange(options.pageRange),
category: options.category,
source,
};
};
basango/apps/crawler/src/process/parsers/base.ts
+6 -7
View File
@@ -1,9 +1,8 @@
import { parse as parseHtml, HTMLElement } from "node-html-parser";
import { HTMLElement, parse as parseHtml } from "node-html-parser";
import { config, FetchCrawlerConfig } from "@/config";
import { SyncHttpClient } from "@/http/http-client";
import { OpenGraph } from "@/http/open-graph";
import type { Persistor } from "@/process/persistence";
import { config, FetchCrawlerConfig } from "@/config";
import { AnySourceConfig, Article } from "@/schema";
export interface CrawlerOptions {
@@ -51,8 +50,8 @@ export abstract class BaseCrawler {
protected textContent(node: HTMLElement | null | undefined): string | null {
if (!node) return null;
// innerText keeps spacing similar to browser rendering
const value = (node as any).innerText ?? node.text;
const text = typeof value === "string" ? value.trim() : String(value ?? "").trim();
const value = node.innerText ?? node.text;
const text = value.trim();
return text.length ? text : null;
}
@@ -64,7 +63,7 @@ export abstract class BaseCrawler {
protected extractFirst(root: HTMLElement, selector?: string | null): HTMLElement | null {
if (!selector) return null;
try {
return (root as any).querySelector?.(selector) ?? null;
return root.querySelector(selector) ?? null;
} catch {
return null;
}
@@ -78,7 +77,7 @@ export abstract class BaseCrawler {
protected extractAll(root: HTMLElement, selector?: string | null): HTMLElement[] {
if (!selector) return [];
try {
return ((root as any).querySelectorAll?.(selector) ?? []) as HTMLElement[];
return root.querySelectorAll(selector);
} catch {
return [];
}
basango/apps/crawler/src/process/parsers/html.ts
+17 -20
View File
@@ -1,18 +1,17 @@
import { logger } from "@basango/logger";
import { HTMLElement } from "node-html-parser";
import { getUnixTime, isMatch as isDateMatch, parse as parseDateFns } from "date-fns";
import { isTimestampInRange, createAbsoluteUrl } from "@/utils";
import { persist, Persistor } from "@/process/persistence";
import { BaseCrawler } from "@/process/parsers/base";
import { HTMLElement } from "node-html-parser";
import TurndownService from "turndown";
import { DateRange, HtmlSourceConfig } from "@/schema";
import { FetchCrawlerConfig } from "@/config";
import { BaseCrawler } from "@/process/parsers/base";
import { Persistor, persist } from "@/process/persistence";
import { DateRange, HtmlSourceConfig } from "@/schema";
import { createAbsoluteUrl, isTimestampInRange } from "@/utils";
const md = new TurndownService({
bulletListMarker: "-",
headingStyle: "atx",
hr: "---",
bulletListMarker: "-",
});
/**
@@ -136,7 +135,7 @@ export class HtmlCrawler extends BaseCrawler {
if (dateRange && !isTimestampInRange(dateRange, timestamp)) {
logger.info(
{ title: titleText, link, date: rawDate, timestamp },
{ date: rawDate, link, timestamp, title: titleText },
"Skipping article outside date range",
);
return null;
@@ -144,12 +143,12 @@ export class HtmlCrawler extends BaseCrawler {
const enriched = await this.enrichWithOpenGraph(
{
title: titleText,
link,
body,
categories,
link,
source: this.source.sourceId,
timestamp,
title: titleText,
},
link,
);
@@ -172,7 +171,7 @@ export class HtmlCrawler extends BaseCrawler {
* Get the pagination range (start and end page numbers).
*/
async getPagination(): Promise<{ start: number; end: number }> {
return { start: 0, end: await this.getLastPage() };
return { end: await this.getLastPage(), start: 0 };
}
/**
@@ -187,7 +186,7 @@ export class HtmlCrawler extends BaseCrawler {
const links = this.extractAll(root, this.source.sourceSelectors.pagination);
if (!links.length) return 1;
const last = links[links.length - 1]!;
const href = (last as any).getAttribute?.("href") as string | null;
const href = last.getAttribute("href") as string | null;
if (!href) return 1;
// Heuristic: prefer a number in the href, else "page" query param
@@ -246,12 +245,10 @@ export class HtmlCrawler extends BaseCrawler {
if (!target) return null;
const href =
(target.getAttribute?.("href") as string | null) ??
((target as any).getAttribute?.("data-href") as string | null) ??
((target as any).getAttribute?.("src") as string | null);
target.getAttribute("href") ?? target.getAttribute("data-href") ?? target.getAttribute("src");
if (!href) return null;
const absolute = createAbsoluteUrl(this.source.sourceUrl, href);
return absolute;
return createAbsoluteUrl(this.source.sourceUrl, href);
}
/**
@@ -265,10 +262,10 @@ export class HtmlCrawler extends BaseCrawler {
if (!target) return null;
// If it's an image, prefer alt/title
const tag = (target as any).tagName?.toLowerCase?.() as string | undefined;
const tag = target.tagName.toLowerCase();
if (tag === "img") {
const alt = (target as any).getAttribute?.("alt") as string | null;
const title = (target as any).getAttribute?.("title") as string | null;
const alt = target.getAttribute("alt");
const title = target.getAttribute("title");
const pick = (alt ?? title ?? "").trim();
if (pick.length > 0) return pick;
}
basango/apps/crawler/src/process/parsers/wordpress.ts
+11 -12
View File
@@ -1,15 +1,14 @@
import { logger } from "@basango/logger";
import { DateRange, PageRange, WordPressSourceConfig } from "@/schema";
import { BaseCrawler } from "@/process/parsers/base";
import { persist, Persistor } from "@/process/persistence";
import TurndownService from "turndown";
import { FetchCrawlerConfig } from "@/config";
import { BaseCrawler } from "@/process/parsers/base";
import { Persistor, persist } from "@/process/persistence";
import { DateRange, PageRange, WordPressSourceConfig } from "@/schema";
const md = new TurndownService({
bulletListMarker: "-",
headingStyle: "atx",
hr: "---",
bulletListMarker: "-",
});
interface WordPressPost {
@@ -59,7 +58,7 @@ export class WordPressCrawler extends BaseCrawler {
const data = (await response.json()) as unknown;
const articles = Array.isArray(data) ? (data as WordPressPost[]) : [];
if (!Array.isArray(data)) {
logger.warn({ type: typeof data, page }, "Unexpected WordPress payload type");
logger.warn({ page, type: typeof data }, "Unexpected WordPress payload type");
}
for (const entry of articles) {
@@ -132,7 +131,7 @@ export class WordPressCrawler extends BaseCrawler {
const { isTimestampInRange } = await import("@/utils");
if (!isTimestampInRange(dateRange, timestamp)) {
logger.info(
{ title, link, date: data.date, timestamp },
{ date: data.date, link, timestamp, title },
"Skipping article outside date range",
);
return null;
@@ -141,12 +140,12 @@ export class WordPressCrawler extends BaseCrawler {
const enriched = await this.enrichWithOpenGraph(
{
title,
link,
body,
categories,
link,
source: this.source.sourceId,
timestamp,
title,
},
link,
);
@@ -169,11 +168,11 @@ export class WordPressCrawler extends BaseCrawler {
response.headers.get(WordPressCrawler.TOTAL_POSTS_HEADER) ?? "0",
10,
);
logger.info({ posts, pages }, "WordPress pagination");
logger.info({ pages, posts }, "WordPress pagination");
const end = Number.isFinite(pages) && pages > 0 ? pages : 1;
return { start: 1, end };
return { end, start: 1 };
} catch {
return { start: 1, end: 1 };
return { end: 1, start: 1 };
}
}
basango/apps/crawler/src/process/persistence.ts
+29 -8
View File
@@ -1,8 +1,8 @@
import fs from "node:fs";
import path from "node:path";
import logger from "@basango/logger";
import { Article } from "@/schema";
import { countTokens } from "@/utils";
import logger from "@basango/logger";
export interface Persistor {
persist(record: Article): Promise<void> | void;
@@ -16,14 +16,38 @@ export interface PersistorOptions {
encoding?: BufferEncoding;
}
const sanitize = (text: string): string => {
if (!text) return text;
let s = text.replace(/\u00A0/g, " "); // remove NBSP
s = s.replace(" ", " "); // remove other NBSP
s = s.replace(" ", " "); // remove NARROW NO-BREAK SPACE
s = s.replace(/\u200B/g, ""); // remove ZERO WIDTH SPACE
s = s.replace(/\u200C/g, ""); // remove ZERO WIDTH NON-JOINER
s = s.replace(/\u200D/g, ""); // remove ZERO WIDTH JOINER
s = s.replace(/\uFEFF/g, ""); // remove ZERO WIDTH NO-BREAK SPACE
s = s.replace(/\r\n/g, "\n"); // normalize CRLF to LF
s = s.replace(/\n{2,}/g, "\n"); // collapse multiple newlines to one
// s = s.replace(/[ \t]{2,}/g, " "); // collapse multiple spaces/tabs
return s.trim();
};
export const persist = async (payload: Article, persistors: Persistor[]): Promise<Article> => {
const article = {
const data = {
...payload,
body: sanitize(payload.body),
categories: payload.categories.map(sanitize),
title: sanitize(payload.title),
};
const article = {
...data,
tokenStatistics: {
title: countTokens(payload.title),
body: countTokens(payload.body),
excerpt: countTokens(payload.body.substring(0, 200)),
categories: countTokens(payload.categories.join(",")),
excerpt: countTokens(payload.body.substring(0, 200)),
title: countTokens(payload.title),
},
} as Article;
@@ -65,10 +89,7 @@ export class JsonlPersistor implements Persistor {
const payload = `${JSON.stringify(record)}\n`;
this.pending = this.pending.then(async () => {
fs.writeFileSync(this.filePath, payload, {
encoding: this.encoding,
mode: "a",
});
fs.appendFileSync(this.filePath, payload, { encoding: this.encoding });
});
return this.pending;
basango/apps/crawler/src/process/sync/tasks.ts
+4 -4
View File
@@ -1,13 +1,13 @@
import { resolveSourceConfig } from "@/utils";
import logger from "@basango/logger";
import {
closePersistors,
CrawlingOptions,
closePersistors,
createPersistors,
resolveCrawlerConfig,
} from "@/process/crawler";
import logger from "@basango/logger";
import { WordPressCrawler } from "@/process/parsers/wordpress";
import { HtmlCrawler } from "@/process/parsers/html";
import { WordPressCrawler } from "@/process/parsers/wordpress";
import { resolveSourceConfig } from "@/utils";
export const runSyncCrawl = async (options: CrawlingOptions): Promise<void> => {
const source = resolveSourceConfig(options.sourceId);
basango/apps/crawler/src/schema.ts
+21 -21
View File
@@ -5,8 +5,8 @@ export const SourceKindSchema = z.enum(["wordpress", "html"]);
export const DateRangeSchema = z
.object({
start: z.number().int(),
end: z.number().int(),
start: z.number().int(),
})
.superRefine((value, ctx) => {
if (value.start === 0 || value.end === 0) {
@@ -25,8 +25,8 @@ export const DateRangeSchema = z
export const PageRangeSchema = z
.object({
start: z.number().int().min(0),
end: z.number().int().min(0),
start: z.number().int().min(0),
})
.superRefine((value, ctx) => {
if (value.end < value.start) {
@@ -43,8 +43,8 @@ export const PageRangeSpecSchema = z
.transform((spec) => {
const [startText, endText] = spec.split(":");
return {
start: Number.parseInt(String(startText), 10),
end: Number.parseInt(String(endText), 10),
start: Number.parseInt(String(startText), 10),
};
});
@@ -53,7 +53,7 @@ export const DateRangeSpecSchema = z
.regex(/.+:.+/, "Expected start:end format")
.transform((spec) => {
const [startRaw, endRaw] = spec.split(":");
return { startRaw: String(startRaw), endRaw: String(endRaw) };
return { endRaw: String(endRaw), startRaw: String(startRaw) };
});
export const SourceDateSchema = z.object({
@@ -63,57 +63,57 @@ export const SourceDateSchema = z.object({
});
const BaseSourceSchema = z.object({
sourceId: z.string(),
sourceUrl: z.url(),
sourceDate: SourceDateSchema,
sourceKind: SourceKindSchema,
categories: z.array(z.string()).default([]),
supportsCategories: z.boolean().default(false),
requiresDetails: z.boolean().default(false),
requiresRateLimit: z.boolean().default(false),
sourceDate: SourceDateSchema,
sourceId: z.string(),
sourceKind: SourceKindSchema,
sourceUrl: z.url(),
supportsCategories: z.boolean().default(false),
});
export const HtmlSourceConfigSchema = BaseSourceSchema.extend({
paginationTemplate: z.string(),
sourceKind: z.literal("html"),
sourceSelectors: z.object({
articleBody: z.string(),
articleCategories: z.string().optional(),
articleDate: z.string(),
articleLink: z.string(),
articles: z.string(),
articleTitle: z.string(),
articleLink: z.string(),
articleBody: z.string(),
articleDate: z.string(),
articleCategories: z.string().optional(),
pagination: z.string().default("ul.pagination > li a"),
}),
paginationTemplate: z.string(),
});
export const WordPressSourceConfigSchema = BaseSourceSchema.extend({
sourceKind: z.literal("wordpress"),
sourceDate: SourceDateSchema.default(SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" })),
sourceKind: z.literal("wordpress"),
});
export const ArticleMetadataSchema = z.object({
title: z.string().optional(),
description: z.string().optional(),
image: z.string().optional(),
title: z.string().optional(),
url: z.url().optional(),
});
export const ArticleTokenStatisticsSchema = z.object({
title: z.number().int().nonnegative().default(0),
body: z.number().int().nonnegative().default(0),
excerpt: z.number().int().nonnegative().default(0),
categories: z.number().int().nonnegative().default(0),
excerpt: z.number().int().nonnegative().default(0),
title: z.number().int().nonnegative().default(0),
});
export const ArticleSchema = z.object({
title: z.string(),
link: z.url(),
body: z.string(),
categories: z.array(z.string()).default([]),
link: z.url(),
metadata: ArticleMetadataSchema.optional(),
source: z.string(),
timestamp: z.number().int(),
metadata: ArticleMetadataSchema.optional(),
title: z.string(),
tokenStatistics: ArticleTokenStatisticsSchema.optional(),
});
basango/apps/crawler/src/scripts/crawl.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { logger } from "@basango/logger";
import { runSyncCrawl } from "@/process/sync/tasks";
import { parseCrawlingCliArgs, CRAWLING_USAGE } from "@/scripts/utils";
import { CRAWLING_USAGE, parseCrawlingCliArgs } from "@/scripts/utils";
const main = async (): Promise<void> => {
const options = parseCrawlingCliArgs();
basango/apps/crawler/src/scripts/queue.ts
+1 -1
View File
@@ -1,6 +1,6 @@
import { logger } from "@basango/logger";
import { scheduleAsyncCrawl } from "@/process/async/tasks";
import { parseCrawlingCliArgs, CRAWLING_USAGE } from "@/scripts/utils";
import { CRAWLING_USAGE, parseCrawlingCliArgs } from "@/scripts/utils";
const main = async (): Promise<void> => {
const options = parseCrawlingCliArgs();
basango/apps/crawler/src/scripts/utils.ts
+6 -6
View File
@@ -9,8 +9,8 @@ export const CRAWLING_USAGE = `
Usage: bun run crawl:[async|sync] -- --sourceId <id> [options]
Options:
--page <range> Optional page range filter (e.g. 1:5)
--date <range> Optional date range filter (e.g. 2024-01-01:2024-01-31)
--pageRange <range> Optional page range filter (e.g. 1:5)
--dateRange <range> Optional date range filter (e.g. 2024-01-01:2024-01-31)
--category <slug> Optional category to crawl
-h, --help Show this message
`;
@@ -18,7 +18,7 @@ export const CRAWLING_USAGE = `
export const parseWorkerCliArgs = (): WorkerCliOptions => {
const { values } = parseArgs({
options: {
queue: { type: "string", multiple: true, short: "q" },
queue: { multiple: true, short: "q", type: "string" },
},
});
@@ -28,10 +28,10 @@ export const parseWorkerCliArgs = (): WorkerCliOptions => {
export const parseCrawlingCliArgs = (): CrawlingOptions => {
const { values } = parseArgs({
options: {
sourceId: { type: "string" },
page: { type: "string" },
date: { type: "string" },
category: { type: "string" },
dateRange: { type: "string" },
pageRange: { type: "string" },
sourceId: { type: "string" },
},
});
basango/apps/crawler/src/utils.ts
+11 -10
View File
@@ -1,19 +1,20 @@
import { format, getUnixTime, isMatch, parse } from "date-fns";
import type { RedisOptions } from "ioredis";
import { get_encoding, TiktokenEncoding } from "tiktoken";
import { format, getUnixTime, isMatch, parse } from "date-fns";
import { config } from "@/config";
import { DEFAULT_DATE_FORMAT } from "@/constants";
import {
AnySourceConfig,
CreateDateRangeOptions,
DateRange,
DateRangeSchema,
DateRangeSpecSchema,
HtmlSourceConfig,
PageRange,
PageRangeSchema,
PageRangeSpecSchema,
WordPressSourceConfig,
} from "@/schema";
import { DEFAULT_DATE_FORMAT } from "@/constants";
import { config } from "@/config";
/**
* Resolve a source configuration by its ID.
@@ -21,8 +22,8 @@ import { config } from "@/config";
*/
export const resolveSourceConfig = (id: string): AnySourceConfig => {
const source =
config.sources.html.find((s) => s.sourceId === id) ||
config.sources.wordpress.find((s) => s.sourceId === id);
config.sources.html.find((s: HtmlSourceConfig) => s.sourceId === id) ||
config.sources.wordpress.find((s: WordPressSourceConfig) => s.sourceId === id);
if (source === undefined) {
throw new Error(`Source '${id}' not found in configuration`);
@@ -41,10 +42,10 @@ export const parseRedisUrl = (url: string): RedisOptions => {
}
const parsed = new URL(url);
return {
host: parsed.hostname,
port: Number(parsed.port || 6379),
password: parsed.password || undefined,
db: Number(parsed.pathname?.replace("/", "") || 0),
host: parsed.hostname,
password: parsed.password || undefined,
port: Number(parsed.port || 6379),
};
};
@@ -112,8 +113,8 @@ export const createDateRange = (
const endDate = parseDate(parsedSpec.endRaw, format);
const range = {
start: getUnixTime(startDate),
end: getUnixTime(endDate),
start: getUnixTime(startDate),
};
return DateRangeSchema.parse(range);
basango/apps/crawler/tsconfig.json
+1 -1
View File
@@ -1,11 +1,11 @@
{
"extends": "@basango/tsconfig/base.json",
"compilerOptions": {
"baseUrl": ".",
"paths": {
"@/*": ["./src/*"]
}
},
"extends": "@basango/tsconfig/base.json",
"include": ["src"],
"references": []
}
basango/apps/crawler/vitest.config.ts
+5 -5
View File
@@ -3,15 +3,15 @@ import path from "node:path";
import { defineConfig } from "vitest/config";
export default defineConfig({
resolve: {
alias: {
"@": path.resolve(__dirname, "src"),
},
},
test: {
environment: "node",
globals: true,
include: ["src/**/*.test.ts"],
setupFiles: ["./vitest.setup.ts"],
},
resolve: {
alias: {
"@": path.resolve(__dirname, "src"),
},
},
});
basango/biome.json
+2 -1
View File
@@ -20,7 +20,8 @@
"rules": {
"recommended": true,
"style": {
"useImportType": "off"
"useImportType": "off",
"noNonNullAssertion": "off"
},
"correctness": {
"noUnusedImports": "on",
basango/bun.lock
+123 -8
View File
@@ -15,15 +15,18 @@
"name": "@basango/crawler",
"dependencies": {
"@basango/logger": "workspace:*",
"@devscast/config": "^1.0.2",
"bullmq": "^4.17.0",
"date-fns": "catalog:",
"ioredis": "^5.3.2",
"@devscast/config": "^1.0.3",
"bullmq": "^4.18.3",
"date-fns": "^3.6.0",
"ioredis": "^5.8.2",
"node-html-parser": "^7.0.1",
"tiktoken": "^1.0.14",
"tiktoken": "^1.0.22",
"turndown": "^7.2.2",
"yaml": "^2.8.1",
"zod": "catalog:",
"zod": "^4.1.12",
},
"devDependencies": {
"@types/turndown": "^5.0.6",
"vitest": "^4.0.7",
},
},
"packages/db": {
@@ -92,7 +95,7 @@
"@date-fns/utc": ["@date-fns/utc@2.1.1", "", {}, "sha512-SlJDfG6RPeEX8wEVv6ZB3kak4MmbtyiI2qX/5zuKdordbrhB/iaJ58GVMZgJ6P1sJaM1gMgENFYYeg1JWrCFrA=="],
"@devscast/config": ["@devscast/config@1.0.2", "", { "peerDependencies": { "ini": "^6.0.0", "yaml": "^2.8.1", "zod": "^4.1.12" }, "optionalPeers": ["ini", "yaml"] }, "sha512-1DR8GQogAOrR4B9mtZ24YIKlEZNvKOFeovw+XepfkXVx0MB1f1fAHtPAAXppV7RPMLSyQEMFJzve17x2HbohEw=="],
"@devscast/config": ["@devscast/config@1.0.3", "", { "peerDependencies": { "ini": "^6.0.0", "yaml": "^2.8.1", "zod": "^4.1.12" }, "optionalPeers": ["ini", "yaml"] }, "sha512-/FjCA/MV1KR2tY44YBA4tdXNzQgoF75O+RQ4fbzvVWY77PXOama2Hf6YXeLcQsvxfItaXi2cFz8BaaVdqZYS8w=="],
"@drizzle-team/brocli": ["@drizzle-team/brocli@0.10.2", "", {}, "sha512-z33Il7l5dKjUgGULTqBsQBQwckHh5AbIuxhdsIxDDiZAzBOrZO6q9ogcWC65kU382AfynTfgNumVcNIjuIua6w=="],
@@ -154,6 +157,8 @@
"@ioredis/commands": ["@ioredis/commands@1.4.0", "", {}, "sha512-aFT2yemJJo+TZCmieA7qnYGQooOS7QfNmYrzGtsYd3g9j5iDP8AimYYAesf79ohjbLG12XxC4nG5DyEnC88AsQ=="],
"@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="],
"@manypkg/cli": ["@manypkg/cli@0.25.1", "", { "dependencies": { "@manypkg/get-packages": "^3.1.0", "detect-indent": "^7.0.1", "normalize-path": "^3.0.0", "p-limit": "^6.2.0", "package-json": "^10.0.1", "parse-github-url": "^1.0.3", "picocolors": "^1.1.1", "sembear": "^0.7.0", "semver": "^7.7.1", "tinyexec": "^1.0.1", "validate-npm-package-name": "^6.0.0" }, "bin": { "manypkg": "bin.js" } }, "sha512-lag906FyiNxzZjsRErkUD5/to174I2JzPk5bZubuJp6loMKKJn73zrtqeU7nHlVkHBg3tgXDTJj22HxUDxLRXw=="],
"@manypkg/find-root": ["@manypkg/find-root@3.1.0", "", { "dependencies": { "@manypkg/tools": "^2.1.0" } }, "sha512-BcSqCyKhBVZ5YkSzOiheMCV41kqAFptW6xGqYSTjkVTl9XQpr+pqHhwgGCOHQtjDCv7Is6EFyA14Sm5GVbVABA=="],
@@ -184,16 +189,86 @@
"@pnpm/npm-conf": ["@pnpm/npm-conf@2.3.1", "", { "dependencies": { "@pnpm/config.env-replace": "^1.1.0", "@pnpm/network.ca-file": "^1.0.1", "config-chain": "^1.1.11" } }, "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw=="],
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.52.5", "", { "os": "android", "cpu": "arm" }, "sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ=="],
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.52.5", "", { "os": "android", "cpu": "arm64" }, "sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA=="],
"@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.52.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA=="],
"@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.52.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA=="],
"@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.52.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA=="],
"@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.52.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ=="],
"@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ=="],
"@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ=="],
"@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg=="],
"@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q=="],
"@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA=="],
"@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.52.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw=="],
"@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw=="],
"@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg=="],
"@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.52.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ=="],
"@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q=="],
"@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg=="],
"@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.52.5", "", { "os": "none", "cpu": "arm64" }, "sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw=="],
"@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.52.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w=="],
"@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.52.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg=="],
"@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ=="],
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg=="],
"@standard-schema/spec": ["@standard-schema/spec@1.0.0", "", {}, "sha512-m2bOd0f2RT9k8QJx1JN85cZYyH1RqFBdlwtkSlf4tBDYLCiiZnv1fIIwacK6cqwXavOydf0NPToMQgpKq+dVlA=="],
"@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="],
"@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="],
"@types/deep-eql": ["@types/deep-eql@4.0.2", "", {}, "sha512-c9h9dVVMigMPc4bwTvC5dxqtqJZwQPePsWjPlpSOnojbor6pGqdk541lfA7AqFQr5pB1BRdq0juY9db81BwyFw=="],
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
"@types/node": ["@types/node@24.9.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA=="],
"@types/pg": ["@types/pg@8.15.6", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ=="],
"@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="],
"@types/turndown": ["@types/turndown@5.0.6", "", {}, "sha512-ru00MoyeeouE5BX4gRL+6m/BsDfbRayOskWqUvh7CLGW+UXxHQItqALa38kKnOiZPqJrtzJUgAC2+F0rL1S4Pg=="],
"@vitest/expect": ["@vitest/expect@4.0.7", "", { "dependencies": { "@standard-schema/spec": "^1.0.0", "@types/chai": "^5.2.2", "@vitest/spy": "4.0.7", "@vitest/utils": "4.0.7", "chai": "^6.0.1", "tinyrainbow": "^3.0.3" } }, "sha512-jGRG6HghnJDjljdjYIoVzX17S6uCVCBRFnsgdLGJ6CaxfPh8kzUKe/2n533y4O/aeZ/sIr7q7GbuEbeGDsWv4Q=="],
"@vitest/mocker": ["@vitest/mocker@4.0.7", "", { "dependencies": { "@vitest/spy": "4.0.7", "estree-walker": "^3.0.3", "magic-string": "^0.30.19" }, "peerDependencies": { "msw": "^2.4.9", "vite": "^6.0.0 || ^7.0.0-0" }, "optionalPeers": ["msw", "vite"] }, "sha512-OsDwLS7WnpuNslOV6bJkXVYVV/6RSc4eeVxV7h9wxQPNxnjRvTTrIikfwCbMyl8XJmW6oOccBj2Q07YwZtQcCw=="],
"@vitest/pretty-format": ["@vitest/pretty-format@4.0.7", "", { "dependencies": { "tinyrainbow": "^3.0.3" } }, "sha512-YY//yxqTmk29+/pK+Wi1UB4DUH3lSVgIm+M10rAJ74pOSMgT7rydMSc+vFuq9LjZLhFvVEXir8EcqMke3SVM6Q=="],
"@vitest/runner": ["@vitest/runner@4.0.7", "", { "dependencies": { "@vitest/utils": "4.0.7", "pathe": "^2.0.3" } }, "sha512-orU1lsu4PxLEcDWfjVCNGIedOSF/YtZ+XMrd1PZb90E68khWCNzD8y1dtxtgd0hyBIQk8XggteKN/38VQLvzuw=="],
"@vitest/snapshot": ["@vitest/snapshot@4.0.7", "", { "dependencies": { "@vitest/pretty-format": "4.0.7", "magic-string": "^0.30.19", "pathe": "^2.0.3" } }, "sha512-xJL+Nkw0OjaUXXQf13B8iKK5pI9QVtN9uOtzNHYuG/o/B7fIEg0DQ+xOe0/RcqwDEI15rud1k7y5xznBKGUXAA=="],
"@vitest/spy": ["@vitest/spy@4.0.7", "", {}, "sha512-FW4X8hzIEn4z+HublB4hBF/FhCVaXfIHm8sUfvlznrcy1MQG7VooBgZPMtVCGZtHi0yl3KESaXTqsKh16d8cFg=="],
"@vitest/utils": ["@vitest/utils@4.0.7", "", { "dependencies": { "@vitest/pretty-format": "4.0.7", "tinyrainbow": "^3.0.3" } }, "sha512-HNrg9CM/Z4ZWB6RuExhuC6FPmLipiShKVMnT9JlQvfhwR47JatWLChA6mtZqVHqypE6p/z6ofcjbyWpM7YLxPQ=="],
"argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
"assertion-error": ["assertion-error@2.0.1", "", {}, "sha512-Izi8RQcffqCeNVgFigKli1ssklIbpHnCYc6AknXGYoB6grJqyeby7jv12JUQgmTAnIDnbck1uxksT4dzN3PWBA=="],
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
"balanced-match": ["balanced-match@1.0.2", "", {}, "sha512-3oSeUO0TMV67hN1AmbXsK4yaqU7tjiHlbxRDZOpH0KW9+CeX4bRAaX0Anxt0tx2MrpRpWwQaPwIlISEJhYU5Pw=="],
@@ -208,6 +283,8 @@
"bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="],
"chai": ["chai@6.2.0", "", {}, "sha512-aUTnJc/JipRzJrNADXVvpVqi6CO0dn3nx4EVPxijri+fj3LUUDyZQOgVeW54Ob3Y1Xh9Iz8f+CgaCl8v0mn9bA=="],
"change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="],
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
@@ -254,10 +331,16 @@
"entities": ["entities@4.5.0", "", {}, "sha512-V0hjH4dGPh9Ao5p0MoRY6BVqtwCjhz6vI5LT8AJ55H+4g9/4vbHx1I54fS0XuclLhDHArPQCiMjDxjaL8fPxhw=="],
"es-module-lexer": ["es-module-lexer@1.7.0", "", {}, "sha512-jEQoCwk8hyb2AZziIOLhDqpm5+2ww5uIE6lkO/6jcOCusfk6LhMHpXXfBLXTZ7Ydyt0j4VoUQv6uGNYbdW+kBA=="],
"esbuild": ["esbuild@0.25.11", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.25.11", "@esbuild/android-arm": "0.25.11", "@esbuild/android-arm64": "0.25.11", "@esbuild/android-x64": "0.25.11", "@esbuild/darwin-arm64": "0.25.11", "@esbuild/darwin-x64": "0.25.11", "@esbuild/freebsd-arm64": "0.25.11", "@esbuild/freebsd-x64": "0.25.11", "@esbuild/linux-arm": "0.25.11", "@esbuild/linux-arm64": "0.25.11", "@esbuild/linux-ia32": "0.25.11", "@esbuild/linux-loong64": "0.25.11", "@esbuild/linux-mips64el": "0.25.11", "@esbuild/linux-ppc64": "0.25.11", "@esbuild/linux-riscv64": "0.25.11", "@esbuild/linux-s390x": "0.25.11", "@esbuild/linux-x64": "0.25.11", "@esbuild/netbsd-arm64": "0.25.11", "@esbuild/netbsd-x64": "0.25.11", "@esbuild/openbsd-arm64": "0.25.11", "@esbuild/openbsd-x64": "0.25.11", "@esbuild/openharmony-arm64": "0.25.11", "@esbuild/sunos-x64": "0.25.11", "@esbuild/win32-arm64": "0.25.11", "@esbuild/win32-ia32": "0.25.11", "@esbuild/win32-x64": "0.25.11" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-KohQwyzrKTQmhXDW1PjCv3Tyspn9n5GcY2RTDqeORIdIJY8yKIF7sTSopFmn/wpMPW4rdPXI0UE5LJLuq3bx0Q=="],
"esbuild-register": ["esbuild-register@3.6.0", "", { "dependencies": { "debug": "^4.3.4" }, "peerDependencies": { "esbuild": ">=0.12 <1" } }, "sha512-H2/S7Pm8a9CL1uhp9OvjwrBh5Pvx0H8qVOxNu8Wed9Y7qv56MPtq+GGM8RJpq6glYJn9Wspr8uw7l55uyinNeg=="],
"estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="],
"expect-type": ["expect-type@1.2.2", "", {}, "sha512-JhFGDVJ7tmDJItKhYgJCGLOWjuK9vPxiXoUFLwLDc99NlmklilbiQJwoctZtt13+xMw91MCk/REan6MWHqDjyA=="],
"fast-copy": ["fast-copy@3.0.2", "", {}, "sha512-dl0O9Vhju8IrcLndv2eU4ldt1ftXMqqfgN4H1cpmGV7P6jeB9FwpN9a2c8DPGE1Ys88rNUJVYDHq73CGAGOPfQ=="],
"fast-safe-stringify": ["fast-safe-stringify@2.1.1", "", {}, "sha512-W+KJc2dmILlPplD/H4K9l9LcAHAfPtP6BY84uVLXQ6Evcz9Lcg33Y2z1IVblT6xdY54PXYVHEv+0Wpq8Io6zkA=="],
@@ -266,6 +349,8 @@
"fs.realpath": ["fs.realpath@1.0.0", "", {}, "sha512-OO0pH2lK6a0hZnAdau5ItzHPI6pUlvI7jMVnxUQRtw4owF2wk8lOSabtGDCTP4Ggrg2MbGnWO9X8K1t4+fGMDw=="],
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
"get-tsconfig": ["get-tsconfig@4.13.0", "", { "dependencies": { "resolve-pkg-maps": "^1.0.0" } }, "sha512-1VKTZJCwBrvbd+Wn3AOgQP/2Av+TfTCOlE4AcRJE72W1ksZXbAx8PPBR9RzgTeSPzlPMHrbANMH3LbltH73wxQ=="],
"glob": ["glob@8.1.0", "", { "dependencies": { "fs.realpath": "^1.0.0", "inflight": "^1.0.4", "inherits": "2", "minimatch": "^5.0.1", "once": "^1.3.0" } }, "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ=="],
@@ -300,6 +385,8 @@
"luxon": ["luxon@3.7.2", "", {}, "sha512-vtEhXh/gNjI9Yg1u4jX/0YVPMvxzHuGgCm6tC5kZyb08yjGWGnqAjGJvcXbqQR2P3MyMEFnRbpcdFS6PBcLqew=="],
"magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="],
"map-obj": ["map-obj@5.0.2", "", {}, "sha512-K6K2NgKnTXimT3779/4KxSvobxOtMmx1LBZ3NwRxT/MDIR3Br/fQ4Q+WCX5QxjyUR8zg5+RV9Tbf2c5pAWTD2A=="],
"minimatch": ["minimatch@5.1.6", "", { "dependencies": { "brace-expansion": "^2.0.1" } }, "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g=="],
@@ -312,6 +399,8 @@
"msgpackr-extract": ["msgpackr-extract@3.0.3", "", { "dependencies": { "node-gyp-build-optional-packages": "5.2.2" }, "optionalDependencies": { "@msgpackr-extract/msgpackr-extract-darwin-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-darwin-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-arm64": "3.0.3", "@msgpackr-extract/msgpackr-extract-linux-x64": "3.0.3", "@msgpackr-extract/msgpackr-extract-win32-x64": "3.0.3" }, "bin": { "download-msgpackr-prebuilds": "bin/download-prebuilds.js" } }, "sha512-P0efT1C9jIdVRefqjzOQ9Xml57zpOXnIuS+csaB4MdZbTdmGDLo8XhzBG1N7aO11gKDDkJvBLULeFTo46wwreA=="],
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
"node-abort-controller": ["node-abort-controller@3.1.1", "", {}, "sha512-AGK2yQKIjRuqnc6VkX2Xj5d+QW8xZ87pa1UK6yA6ouUyuxfHuMP6umE5QK7UmTeOAymo+Zx1Fxiuw9rVx8taHQ=="],
"node-gyp-build-optional-packages": ["node-gyp-build-optional-packages@5.2.2", "", { "dependencies": { "detect-libc": "^2.0.1" }, "bin": { "node-gyp-build-optional-packages": "bin.js", "node-gyp-build-optional-packages-optional": "optional.js", "node-gyp-build-optional-packages-test": "build-test.js" } }, "sha512-s+w+rBWnpTMwSFbaE0UXsRlg7hU4FjekKU4eyAih5T8nJuNZT1nNsskXpxmeqSK9UzkBl6UgRlnKc8hz8IEqOw=="],
@@ -332,6 +421,8 @@
"parse-github-url": ["parse-github-url@1.0.3", "", { "bin": { "parse-github-url": "cli.js" } }, "sha512-tfalY5/4SqGaV/GIGzWyHnFjlpTPTNpENR9Ea2lLldSJ8EWXMsvacWucqY3m3I4YPtas15IxTLQVQ5NSYXPrww=="],
"pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
"pg": ["pg@8.16.3", "", { "dependencies": { "pg-connection-string": "^2.9.1", "pg-pool": "^3.10.1", "pg-protocol": "^1.10.3", "pg-types": "2.2.0", "pgpass": "1.0.5" }, "optionalDependencies": { "pg-cloudflare": "^1.2.7" }, "peerDependencies": { "pg-native": ">=3.0.1" }, "optionalPeers": ["pg-native"] }, "sha512-enxc1h0jA/aq5oSDMvqyW3q89ra6XIIDZgCX9vkMrnz5DFTw/Ny3Li2lFQ+pt3L6MCgm/5o2o8HW9hiJji+xvw=="],
"pg-cloudflare": ["pg-cloudflare@1.2.7", "", {}, "sha512-YgCtzMH0ptvZJslLM1ffsY4EuGaU0cx4XSdXLRFae8bPP4dS5xL1tNB3k2o/N64cHJpwU7dxKli/nZ2lUa5fLg=="],
@@ -360,6 +451,8 @@
"pino-std-serializers": ["pino-std-serializers@7.0.0", "", {}, "sha512-e906FRY0+tV27iq4juKzSYPbUj2do2X2JX4EzSca1631EB2QJQUqGbDuERal7LCtOpxl6x3+nvo9NPZcmjkiFA=="],
"postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="],
"postgres-array": ["postgres-array@2.0.0", "", {}, "sha512-VpZrUqU5A69eQyW2c5CA1jtLecCsN2U/bD6VilrFDWq5+5UIEVO7nazS3TEcHf1zuPYO/sqGvUvW62g86RXZuA=="],
"postgres-bytea": ["postgres-bytea@1.0.0", "", {}, "sha512-xy3pmLuQqRBZBXDULy7KbaitYqLcmxigw14Q5sj8QBVLqEwXfeybIKVWiqAXTlcvdvb0+xkOtDbfQMOf4lST1w=="],
@@ -390,6 +483,8 @@
"resolve-pkg-maps": ["resolve-pkg-maps@1.0.0", "", {}, "sha512-seS2Tj26TBVOC2NIc2rOe2y2ZO7efxITtLZcGSOnHHNOQ7CkiUBfw0Iw2ck6xkIhPwLhKNLS8BO+hEpngQlqzw=="],
"rollup": ["rollup@4.52.5", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.52.5", "@rollup/rollup-android-arm64": "4.52.5", "@rollup/rollup-darwin-arm64": "4.52.5", "@rollup/rollup-darwin-x64": "4.52.5", "@rollup/rollup-freebsd-arm64": "4.52.5", "@rollup/rollup-freebsd-x64": "4.52.5", "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", "@rollup/rollup-linux-arm-musleabihf": "4.52.5", "@rollup/rollup-linux-arm64-gnu": "4.52.5", "@rollup/rollup-linux-arm64-musl": "4.52.5", "@rollup/rollup-linux-loong64-gnu": "4.52.5", "@rollup/rollup-linux-ppc64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-musl": "4.52.5", "@rollup/rollup-linux-s390x-gnu": "4.52.5", "@rollup/rollup-linux-x64-gnu": "4.52.5", "@rollup/rollup-linux-x64-musl": "4.52.5", "@rollup/rollup-openharmony-arm64": "4.52.5", "@rollup/rollup-win32-arm64-msvc": "4.52.5", "@rollup/rollup-win32-ia32-msvc": "4.52.5", "@rollup/rollup-win32-x64-gnu": "4.52.5", "@rollup/rollup-win32-x64-msvc": "4.52.5", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw=="],
"safe-stable-stringify": ["safe-stable-stringify@2.5.0", "", {}, "sha512-b3rppTKm9T+PsVCBEOUR46GWI7fdOs00VKZ1+9c1EWDaDMvjQc6tUwuFyIprgGgTcWoVHSKrU8H31ZHA2e0RHA=="],
"secure-json-parse": ["secure-json-parse@4.1.0", "", {}, "sha512-l4KnYfEyqYJxDwlNVyRfO2E4NTHfMKAWdUuA8J0yve2Dz/E/PdBepY03RvyJpssIpRFwJoCD55wA+mEDs6ByWA=="],
@@ -398,28 +493,40 @@
"semver": ["semver@7.7.3", "", { "bin": { "semver": "bin/semver.js" } }, "sha512-SdsKMrI9TdgjdweUSR9MweHA4EJ8YxHn8DFaDisvhVlUOe4BF1tLD7GAj0lIqWVl+dPb/rExr0Btby5loQm20Q=="],
"siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="],
"snakecase-keys": ["snakecase-keys@9.0.2", "", { "dependencies": { "change-case": "^5.4.4", "map-obj": "^5.0.2", "type-fest": "^4.15.0" } }, "sha512-Tr4gONsDj1Pa6HJH9D3b411r6tuRyCGgb1l7YpzDFp/thjVSWs7rcbNjyTyRqJi5SUV23sFpzf9epIJRbLR6Yw=="],
"sonic-boom": ["sonic-boom@4.2.0", "", { "dependencies": { "atomic-sleep": "^1.0.0" } }, "sha512-INb7TM37/mAcsGmc9hyyI6+QR3rR1zVRu36B0NeGXKnOOLiZOfER5SA+N7X7k3yUYRzLWafduTDvJAfDswwEww=="],
"source-map": ["source-map@0.6.1", "", {}, "sha512-UjgapumWlbMhkBgzT7Ykc5YXUT46F0iKu8SGXq0bcwP5dz/h0Plj6enJqjz1Zbq2l5WaqYnrVbwWOWMyF3F47g=="],
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
"source-map-support": ["source-map-support@0.5.21", "", { "dependencies": { "buffer-from": "^1.0.0", "source-map": "^0.6.0" } }, "sha512-uBHU3L3czsIyYXKX88fdrGovxdSCoTGDRZ6SYXtSRxLZUzHg5P/66Ht6uoUlHu9EZod+inXhKo3qQgwXUT/y1w=="],
"split2": ["split2@4.2.0", "", {}, "sha512-UcjcJOWknrNkF6PLX83qcHM6KHgVKNkV62Y8a5uYDVv9ydGQVwAHMKqHdJje1VTWpljG0WYpCDhrCdAOYH4TWg=="],
"stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="],
"standard-as-callback": ["standard-as-callback@2.1.0", "", {}, "sha512-qoRRSyROncaz1z0mvYqIE4lCd9p2R90i6GxW3uZv5ucSu8tU7B5HXUP1gG8pVZsYNVaXjk8ClXHPttLyxAL48A=="],
"std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="],
"strip-json-comments": ["strip-json-comments@5.0.3", "", {}, "sha512-1tB5mhVo7U+ETBKNf92xT4hrQa3pm0MZ0PQvuDnWgAAGHDsfp4lPSpiS6psrSiet87wyGPh9ft6wmhOMQ0hDiw=="],
"thread-stream": ["thread-stream@3.1.0", "", { "dependencies": { "real-require": "^0.2.0" } }, "sha512-OqyPZ9u96VohAyMfJykzmivOrY2wfMSf3C5TtFJVgN+Hm6aj+voFhlK+kZEIv2FBh1X6Xp3DlnCOfEQ3B2J86A=="],
"tiktoken": ["tiktoken@1.0.22", "", {}, "sha512-PKvy1rVF1RibfF3JlXBSP0Jrcw2uq3yXdgcEXtKTYn3QJ/cBRBHDnrJ5jHky+MENZ6DIPwNUGWpkVx+7joCpNA=="],
"tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="],
"tinyexec": ["tinyexec@1.0.1", "", {}, "sha512-5uC6DDlmeqiOwCPmK9jMSdOuZTh8bU39Ys6yidB+UTt5hfZUPGAypSgFRiEp+jbi9qH40BLDvy85jIU88wKSqw=="],
"tinyglobby": ["tinyglobby@0.2.15", "", { "dependencies": { "fdir": "^6.5.0", "picomatch": "^4.0.3" } }, "sha512-j2Zq4NyQYG5XMST4cbs02Ak8iJUdxRM0XI5QyxXuZOzKOINmWurp3smXu3y5wDcJrptwpSjgXHzIQxR0omXljQ=="],
"tinyrainbow": ["tinyrainbow@3.0.3", "", {}, "sha512-PSkbLUoxOFRzJYjjxHJt9xro7D+iilgMX/C9lawzVuYiIdcihh9DXmVibBe8lmcFrRi/VzlPjBxbN7rH24q8/Q=="],
"tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="],
"turbo": ["turbo@2.5.8", "", { "optionalDependencies": { "turbo-darwin-64": "2.5.8", "turbo-darwin-arm64": "2.5.8", "turbo-linux-64": "2.5.8", "turbo-linux-arm64": "2.5.8", "turbo-windows-64": "2.5.8", "turbo-windows-arm64": "2.5.8" }, "bin": { "turbo": "bin/turbo" } }, "sha512-5c9Fdsr9qfpT3hA0EyYSFRZj1dVVsb6KIWubA9JBYZ/9ZEAijgUEae0BBR/Xl/wekt4w65/lYLTFaP3JmwSO8w=="],
@@ -448,6 +555,12 @@
"validate-npm-package-name": ["validate-npm-package-name@6.0.2", "", {}, "sha512-IUoow1YUtvoBBC06dXs8bR8B9vuA3aJfmQNKMoaPG/OFsPmoQvw8xh+6Ye25Gx9DQhoEom3Pcu9MKHerm/NpUQ=="],
"vite": ["vite@7.2.2", "", { "dependencies": { "esbuild": "^0.25.0", "fdir": "^6.5.0", "picomatch": "^4.0.3", "postcss": "^8.5.6", "rollup": "^4.43.0", "tinyglobby": "^0.2.15" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^20.19.0 || >=22.12.0", "jiti": ">=1.21.0", "less": "^4.0.0", "lightningcss": "^1.21.0", "sass": "^1.70.0", "sass-embedded": "^1.70.0", "stylus": ">=0.54.8", "sugarss": "^5.0.0", "terser": "^5.16.0", "tsx": "^4.8.1", "yaml": "^2.4.2" }, "optionalPeers": ["@types/node", "jiti", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser", "tsx", "yaml"], "bin": { "vite": "bin/vite.js" } }, "sha512-BxAKBWmIbrDgrokdGZH1IgkIk/5mMHDreLDmCJ0qpyJaAteP8NvMhkwr/ZCQNqNH97bw/dANTE9PDzqwJghfMQ=="],
"vitest": ["vitest@4.0.7", "", { "dependencies": { "@vitest/expect": "4.0.7", "@vitest/mocker": "4.0.7", "@vitest/pretty-format": "4.0.7", "@vitest/runner": "4.0.7", "@vitest/snapshot": "4.0.7", "@vitest/spy": "4.0.7", "@vitest/utils": "4.0.7", "debug": "^4.4.3", "es-module-lexer": "^1.7.0", "expect-type": "^1.2.2", "magic-string": "^0.30.19", "pathe": "^2.0.3", "picomatch": "^4.0.3", "std-env": "^3.9.0", "tinybench": "^2.9.0", "tinyexec": "^0.3.2", "tinyglobby": "^0.2.15", "tinyrainbow": "^3.0.3", "vite": "^6.0.0 || ^7.0.0", "why-is-node-running": "^2.3.0" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/debug": "^4.1.12", "@types/node": "^20.0.0 || ^22.0.0 || >=24.0.0", "@vitest/browser-playwright": "4.0.7", "@vitest/browser-preview": "4.0.7", "@vitest/browser-webdriverio": "4.0.7", "@vitest/ui": "4.0.7", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/debug", "@types/node", "@vitest/browser-playwright", "@vitest/browser-preview", "@vitest/browser-webdriverio", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-xQroKAadK503CrmbzCISvQUjeuvEZzv6U0wlnlVFOi5i3gnzfH4onyQ29f3lzpe0FresAiTAd3aqK0Bi/jLI8w=="],
"why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="],
"wrappy": ["wrappy@1.0.2", "", {}, "sha512-l4Sp/DRseor9wL6EvV2+TuQn63dMkPjZ/sp9XkghTEbV9KlPS1xUsZ3u7/IQO4wxtcFB4bgpQPRcR3QCvezPcQ=="],
"xtend": ["xtend@4.0.2", "", {}, "sha512-LKYU1iAXJXUgAXn9URjiu+MWhyUXHsvfp7mcuYm9dSUKK0/CjtrUwFAxD82/mCWbtLsGjFIad0wIsod4zrTAEQ=="],
@@ -462,6 +575,8 @@
"rc/strip-json-comments": ["strip-json-comments@2.0.1", "", {}, "sha512-4gB8na07fecVVkOI6Rs4e7T6NOTki5EmL7TUduTs6bu3EdnSycntVJ4re8kgZA+wx9IueI2Y11bfbgwtzuE0KQ=="],
"vitest/tinyexec": ["tinyexec@0.3.2", "", {}, "sha512-KQQR9yN7R5+OSwaK0XQoj22pwHoTlgYqmUscPYoknOoWCWfj/5/ABTMRi69FrKU5ffPVh5QcFikpWJI/P1ocHA=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm": ["@esbuild/android-arm@0.18.20", "", { "os": "android", "cpu": "arm" }, "sha512-fyi7TDI/ijKKNZTUJAQqiG5T7YjJXgnzkURqmGj13C6dCqckZBLdl4h7bkhHt/t0WP+zO9/zwroDvANaOqO5Sw=="],
"@esbuild-kit/core-utils/esbuild/@esbuild/android-arm64": ["@esbuild/android-arm64@0.18.20", "", { "os": "android", "cpu": "arm64" }, "sha512-Nz4rJcchGDtENV0eMKUNa6L12zz2zBDXuhj/Vjh18zGqB44Bi7MBMSXjgunJgjRhCmKOjnPuZp4Mb6OKqtMHLQ=="],
basango/packages/logger/src/index.ts
+1 -1
View File
@@ -3,7 +3,7 @@ import pino from "pino";
export const logger = pino({
level: process.env.LOG_LEVEL || "info",
// Use pretty printing in development, structured JSON in production
...(process.env.NODE_ENV === "development" && {
...(process.env.NODE_ENV !== "production" && {
transport: {
target: "pino-pretty",
options: {
basango/packages/tsconfig/base.json
+1 -1
View File
@@ -8,7 +8,7 @@
"incremental": false,
"isolatedModules": true,
"lib": ["es2022", "DOM", "DOM.Iterable"],
"module": "NodeNext",
"module": "esnext",
"moduleDetection": "force",
"moduleResolution": "node",
"forceConsistentCasingInFileNames": true,