[crawler]: fix conflit
This commit is contained in:
@@ -0,0 +1,7 @@
|
||||
# porting to Typescript (./basango)
|
||||
- when working on a porting feature (working directory is basango, refer to docs/ for more instructions)
|
||||
- make sure to use `bun` 1.3 as runtime not `node`
|
||||
- prefer `const fn = () => {}` instead of `function`
|
||||
|
||||
# legacy (./projects)
|
||||
- when working on a legacy feature (working director is projects)
|
||||
@@ -57,13 +57,7 @@
|
||||
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
||||
"replacement": "$3-$2-$1 $4"
|
||||
},
|
||||
"categories": [
|
||||
"politique",
|
||||
"economie",
|
||||
"culture",
|
||||
"sport",
|
||||
"societe"
|
||||
],
|
||||
"categories": ["politique", "economie", "culture", "sport", "societe"],
|
||||
"source_selectors": {
|
||||
"articles": ".view-content > .row.views-row",
|
||||
"article_title": ".views-field-title a",
|
||||
@@ -119,31 +113,80 @@
|
||||
}
|
||||
],
|
||||
"wordpress": [
|
||||
{ "source_id": "beto.cd", "source_url": "https://beto.cd", "requires_rate_limit": true },
|
||||
{
|
||||
"source_id": "beto.cd",
|
||||
"source_url": "https://beto.cd",
|
||||
"requires_rate_limit": true
|
||||
},
|
||||
{ "source_id": "newscd.net", "source_url": "https://newscd.net" },
|
||||
{ "source_id": "africanewsrdc.net", "source_url": "https://www.africanewsrdc.net" },
|
||||
{ "source_id": "angazainstitute.ac.cd", "source_url": "https://angazainstitute.ac.cd" },
|
||||
{
|
||||
"source_id": "africanewsrdc.net",
|
||||
"source_url": "https://www.africanewsrdc.net"
|
||||
},
|
||||
{
|
||||
"source_id": "angazainstitute.ac.cd",
|
||||
"source_url": "https://angazainstitute.ac.cd"
|
||||
},
|
||||
{ "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" },
|
||||
{ "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" },
|
||||
{ "source_id": "changement7.net", "source_url": "https://changement7.net" },
|
||||
{
|
||||
"source_id": "changement7.net",
|
||||
"source_url": "https://changement7.net"
|
||||
},
|
||||
{ "source_id": "congoactu.net", "source_url": "https://congoactu.net" },
|
||||
{ "source_id": "congoindependant.com", "source_url": "https://www.congoindependant.com" },
|
||||
{ "source_id": "congoquotidien.com", "source_url": "https://www.congoquotidien.com" },
|
||||
{
|
||||
"source_id": "congoindependant.com",
|
||||
"source_url": "https://www.congoindependant.com"
|
||||
},
|
||||
{
|
||||
"source_id": "congoquotidien.com",
|
||||
"source_url": "https://www.congoquotidien.com"
|
||||
},
|
||||
{ "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" },
|
||||
{ "source_id": "environews-rdc.net", "source_url": "https://environews-rdc.net" },
|
||||
{ "source_id": "freemediardc.info", "source_url": "https://www.freemediardc.info" },
|
||||
{ "source_id": "geopolismagazine.org", "source_url": "https://geopolismagazine.org" },
|
||||
{
|
||||
"source_id": "environews-rdc.net",
|
||||
"source_url": "https://environews-rdc.net"
|
||||
},
|
||||
{
|
||||
"source_id": "freemediardc.info",
|
||||
"source_url": "https://www.freemediardc.info"
|
||||
},
|
||||
{
|
||||
"source_id": "geopolismagazine.org",
|
||||
"source_url": "https://geopolismagazine.org"
|
||||
},
|
||||
{ "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" },
|
||||
{ "source_id": "infordc.com", "source_url": "https://infordc.com" },
|
||||
{ "source_id": "kilalopress.net", "source_url": "https://kilalopress.net" },
|
||||
{ "source_id": "laprosperiteonline.net", "source_url": "https://laprosperiteonline.net" },
|
||||
{ "source_id": "laprunellerdc.cd", "source_url": "https://laprunellerdc.cd" },
|
||||
{
|
||||
"source_id": "kilalopress.net",
|
||||
"source_url": "https://kilalopress.net"
|
||||
},
|
||||
{
|
||||
"source_id": "laprosperiteonline.net",
|
||||
"source_url": "https://laprosperiteonline.net"
|
||||
},
|
||||
{
|
||||
"source_id": "laprunellerdc.cd",
|
||||
"source_url": "https://laprunellerdc.cd"
|
||||
},
|
||||
{ "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" },
|
||||
{ "source_id": "lesvolcansnews.net", "source_url": "https://lesvolcansnews.net" },
|
||||
{ "source_id": "netic-news.net", "source_url": "https://www.netic-news.net" },
|
||||
{ "source_id": "objectif-infos.cd", "source_url": "https://objectif-infos.cd" },
|
||||
{
|
||||
"source_id": "lesvolcansnews.net",
|
||||
"source_url": "https://lesvolcansnews.net"
|
||||
},
|
||||
{
|
||||
"source_id": "netic-news.net",
|
||||
"source_url": "https://www.netic-news.net"
|
||||
},
|
||||
{
|
||||
"source_id": "objectif-infos.cd",
|
||||
"source_url": "https://objectif-infos.cd"
|
||||
},
|
||||
{ "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" },
|
||||
{ "source_id": "journaldekinshasa.com", "source_url": "https://www.journaldekinshasa.com" },
|
||||
{
|
||||
"source_id": "journaldekinshasa.com",
|
||||
"source_url": "https://www.journaldekinshasa.com"
|
||||
},
|
||||
{ "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" },
|
||||
{ "source_id": "acturdc.com", "source_url": "https://acturdc.com" },
|
||||
{ "source_id": "matininfos.net", "source_url": "https://matininfos.net" }
|
||||
|
||||
@@ -57,13 +57,7 @@
|
||||
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
||||
"replacement": "$3-$2-$1 $4"
|
||||
},
|
||||
"categories": [
|
||||
"politique",
|
||||
"economie",
|
||||
"culture",
|
||||
"sport",
|
||||
"societe"
|
||||
],
|
||||
"categories": ["politique", "economie", "culture", "sport", "societe"],
|
||||
"source_selectors": {
|
||||
"articles": ".view-content > .row.views-row",
|
||||
"article_title": ".views-field-title a",
|
||||
@@ -119,31 +113,80 @@
|
||||
}
|
||||
],
|
||||
"wordpress": [
|
||||
{ "source_id": "beto.cd", "source_url": "https://beto.cd", "requires_rate_limit": true },
|
||||
{
|
||||
"source_id": "beto.cd",
|
||||
"source_url": "https://beto.cd",
|
||||
"requires_rate_limit": true
|
||||
},
|
||||
{ "source_id": "newscd.net", "source_url": "https://newscd.net" },
|
||||
{ "source_id": "africanewsrdc.net", "source_url": "https://www.africanewsrdc.net" },
|
||||
{ "source_id": "angazainstitute.ac.cd", "source_url": "https://angazainstitute.ac.cd" },
|
||||
{
|
||||
"source_id": "africanewsrdc.net",
|
||||
"source_url": "https://www.africanewsrdc.net"
|
||||
},
|
||||
{
|
||||
"source_id": "angazainstitute.ac.cd",
|
||||
"source_url": "https://angazainstitute.ac.cd"
|
||||
},
|
||||
{ "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" },
|
||||
{ "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" },
|
||||
{ "source_id": "changement7.net", "source_url": "https://changement7.net" },
|
||||
{
|
||||
"source_id": "changement7.net",
|
||||
"source_url": "https://changement7.net"
|
||||
},
|
||||
{ "source_id": "congoactu.net", "source_url": "https://congoactu.net" },
|
||||
{ "source_id": "congoindependant.com", "source_url": "https://www.congoindependant.com" },
|
||||
{ "source_id": "congoquotidien.com", "source_url": "https://www.congoquotidien.com" },
|
||||
{
|
||||
"source_id": "congoindependant.com",
|
||||
"source_url": "https://www.congoindependant.com"
|
||||
},
|
||||
{
|
||||
"source_id": "congoquotidien.com",
|
||||
"source_url": "https://www.congoquotidien.com"
|
||||
},
|
||||
{ "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" },
|
||||
{ "source_id": "environews-rdc.net", "source_url": "https://environews-rdc.net" },
|
||||
{ "source_id": "freemediardc.info", "source_url": "https://www.freemediardc.info" },
|
||||
{ "source_id": "geopolismagazine.org", "source_url": "https://geopolismagazine.org" },
|
||||
{
|
||||
"source_id": "environews-rdc.net",
|
||||
"source_url": "https://environews-rdc.net"
|
||||
},
|
||||
{
|
||||
"source_id": "freemediardc.info",
|
||||
"source_url": "https://www.freemediardc.info"
|
||||
},
|
||||
{
|
||||
"source_id": "geopolismagazine.org",
|
||||
"source_url": "https://geopolismagazine.org"
|
||||
},
|
||||
{ "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" },
|
||||
{ "source_id": "infordc.com", "source_url": "https://infordc.com" },
|
||||
{ "source_id": "kilalopress.net", "source_url": "https://kilalopress.net" },
|
||||
{ "source_id": "laprosperiteonline.net", "source_url": "https://laprosperiteonline.net" },
|
||||
{ "source_id": "laprunellerdc.cd", "source_url": "https://laprunellerdc.cd" },
|
||||
{
|
||||
"source_id": "kilalopress.net",
|
||||
"source_url": "https://kilalopress.net"
|
||||
},
|
||||
{
|
||||
"source_id": "laprosperiteonline.net",
|
||||
"source_url": "https://laprosperiteonline.net"
|
||||
},
|
||||
{
|
||||
"source_id": "laprunellerdc.cd",
|
||||
"source_url": "https://laprunellerdc.cd"
|
||||
},
|
||||
{ "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" },
|
||||
{ "source_id": "lesvolcansnews.net", "source_url": "https://lesvolcansnews.net" },
|
||||
{ "source_id": "netic-news.net", "source_url": "https://www.netic-news.net" },
|
||||
{ "source_id": "objectif-infos.cd", "source_url": "https://objectif-infos.cd" },
|
||||
{
|
||||
"source_id": "lesvolcansnews.net",
|
||||
"source_url": "https://lesvolcansnews.net"
|
||||
},
|
||||
{
|
||||
"source_id": "netic-news.net",
|
||||
"source_url": "https://www.netic-news.net"
|
||||
},
|
||||
{
|
||||
"source_id": "objectif-infos.cd",
|
||||
"source_url": "https://objectif-infos.cd"
|
||||
},
|
||||
{ "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" },
|
||||
{ "source_id": "journaldekinshasa.com", "source_url": "https://www.journaldekinshasa.com" },
|
||||
{
|
||||
"source_id": "journaldekinshasa.com",
|
||||
"source_url": "https://www.journaldekinshasa.com"
|
||||
},
|
||||
{ "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" },
|
||||
{ "source_id": "acturdc.com", "source_url": "https://acturdc.com" },
|
||||
{ "source_id": "matininfos.net", "source_url": "https://matininfos.net" }
|
||||
|
||||
@@ -1,22 +1,22 @@
|
||||
{
|
||||
"name": "@basango/crawler",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc -b",
|
||||
"test": "vitest --run",
|
||||
"queue": "bun run src/scripts/queue.ts",
|
||||
"worker": "bun run src/scripts/worker.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@basango/logger": "workspace:*",
|
||||
"bullmq": "^4.17.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"ioredis": "^5.3.2",
|
||||
"tiktoken": "^1.0.14",
|
||||
"zod": "^4.0.0"
|
||||
}
|
||||
"name": "@basango/crawler",
|
||||
"version": "0.1.0",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "dist/index.js",
|
||||
"types": "dist/index.d.ts",
|
||||
"scripts": {
|
||||
"build": "tsc -b",
|
||||
"test": "vitest --run",
|
||||
"queue": "bun run src/scripts/queue.ts",
|
||||
"worker": "bun run src/scripts/worker.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@basango/logger": "workspace:*",
|
||||
"bullmq": "^4.17.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"ioredis": "^5.3.2",
|
||||
"tiktoken": "^1.0.14",
|
||||
"zod": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -8,74 +8,74 @@ import { loadConfig } from "./config";
|
||||
import { resolveConfigPath } from "./schema";
|
||||
|
||||
describe("loadConfig", () => {
|
||||
it("parses json configuration and ensures directories", () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
||||
const paths = {
|
||||
root: tempDir,
|
||||
data: path.join(tempDir, "data"),
|
||||
logs: path.join(tempDir, "logs"),
|
||||
configs: path.join(tempDir, "configs"),
|
||||
};
|
||||
it("parses json configuration and ensures directories", () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
||||
const paths = {
|
||||
root: tempDir,
|
||||
data: path.join(tempDir, "data"),
|
||||
logs: path.join(tempDir, "logs"),
|
||||
configs: path.join(tempDir, "configs"),
|
||||
};
|
||||
|
||||
const configPath = path.join(tempDir, "pipeline.json");
|
||||
fs.writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
paths,
|
||||
fetch: {
|
||||
client: { timeout: 10 },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
const configPath = path.join(tempDir, "pipeline.json");
|
||||
fs.writeFileSync(
|
||||
configPath,
|
||||
JSON.stringify(
|
||||
{
|
||||
paths,
|
||||
fetch: {
|
||||
client: { timeout: 10 },
|
||||
},
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const config = loadConfig({ configPath });
|
||||
const config = loadConfig({ configPath });
|
||||
|
||||
expect(config.fetch.client.timeout).toBe(10);
|
||||
expect(fs.existsSync(paths.data)).toBe(true);
|
||||
expect(fs.existsSync(paths.logs)).toBe(true);
|
||||
expect(fs.existsSync(paths.configs)).toBe(true);
|
||||
});
|
||||
expect(config.fetch.client.timeout).toBe(10);
|
||||
expect(fs.existsSync(paths.data)).toBe(true);
|
||||
expect(fs.existsSync(paths.logs)).toBe(true);
|
||||
expect(fs.existsSync(paths.configs)).toBe(true);
|
||||
});
|
||||
|
||||
it("merges environment override if available", () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
||||
const paths = {
|
||||
root: tempDir,
|
||||
data: path.join(tempDir, "data"),
|
||||
logs: path.join(tempDir, "logs"),
|
||||
configs: path.join(tempDir, "configs"),
|
||||
};
|
||||
it("merges environment override if available", () => {
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
||||
const paths = {
|
||||
root: tempDir,
|
||||
data: path.join(tempDir, "data"),
|
||||
logs: path.join(tempDir, "logs"),
|
||||
configs: path.join(tempDir, "configs"),
|
||||
};
|
||||
|
||||
const basePath = path.join(tempDir, "pipeline.json");
|
||||
fs.writeFileSync(
|
||||
basePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
paths,
|
||||
logging: { level: "INFO" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
const basePath = path.join(tempDir, "pipeline.json");
|
||||
fs.writeFileSync(
|
||||
basePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
paths,
|
||||
logging: { level: "INFO" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const overridePath = resolveConfigPath(basePath, "production");
|
||||
fs.writeFileSync(
|
||||
overridePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
logging: { level: "DEBUG" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
const overridePath = resolveConfigPath(basePath, "production");
|
||||
fs.writeFileSync(
|
||||
overridePath,
|
||||
JSON.stringify(
|
||||
{
|
||||
logging: { level: "DEBUG" },
|
||||
},
|
||||
null,
|
||||
2,
|
||||
),
|
||||
);
|
||||
|
||||
const config = loadConfig({ configPath: basePath, env: "production" });
|
||||
const config = loadConfig({ configPath: basePath, env: "production" });
|
||||
|
||||
expect(config.logging.level).toBe("DEBUG");
|
||||
});
|
||||
expect(config.logging.level).toBe("DEBUG");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -3,56 +3,56 @@ import { describe, expect, it } from "vitest";
|
||||
import { createQueueManager, createQueueSettings } from "./queue";
|
||||
|
||||
class InMemoryQueue {
|
||||
public jobs: Array<{ name: string; data: unknown }> = [];
|
||||
public jobs: Array<{ name: string; data: unknown }> = [];
|
||||
|
||||
async add(name: string, data: unknown) {
|
||||
this.jobs.push({ name, data });
|
||||
return { id: `${name}-${this.jobs.length}` };
|
||||
}
|
||||
async add(name: string, data: unknown) {
|
||||
this.jobs.push({ name, data });
|
||||
return { id: `${name}-${this.jobs.length}` };
|
||||
}
|
||||
}
|
||||
|
||||
describe("createQueueManager", () => {
|
||||
it("prefixes queue names", () => {
|
||||
const manager = createQueueManager({
|
||||
settings: createQueueSettings({ prefix: "test" }),
|
||||
queueFactory: (queueName) => {
|
||||
expect(queueName).toBe("listing");
|
||||
return new InMemoryQueue();
|
||||
},
|
||||
connection: {
|
||||
quit: async () => undefined,
|
||||
} as any,
|
||||
});
|
||||
it("prefixes queue names", () => {
|
||||
const manager = createQueueManager({
|
||||
settings: createQueueSettings({ prefix: "test" }),
|
||||
queueFactory: (queueName) => {
|
||||
expect(queueName).toBe("listing");
|
||||
return new InMemoryQueue();
|
||||
},
|
||||
connection: {
|
||||
quit: async () => undefined,
|
||||
} as any,
|
||||
});
|
||||
|
||||
expect(manager.iterQueueNames()).toEqual([
|
||||
"test:listing",
|
||||
"test:articles",
|
||||
"test:processed",
|
||||
]);
|
||||
});
|
||||
expect(manager.iterQueueNames()).toEqual([
|
||||
"test:listing",
|
||||
"test:articles",
|
||||
"test:processed",
|
||||
]);
|
||||
});
|
||||
|
||||
it("enqueues listing job with validated payload", async () => {
|
||||
const queue = new InMemoryQueue();
|
||||
const manager = createQueueManager({
|
||||
queueFactory: () => queue,
|
||||
connection: { quit: async () => undefined } as any,
|
||||
});
|
||||
it("enqueues listing job with validated payload", async () => {
|
||||
const queue = new InMemoryQueue();
|
||||
const manager = createQueueManager({
|
||||
queueFactory: () => queue,
|
||||
connection: { quit: async () => undefined } as any,
|
||||
});
|
||||
|
||||
const job = await manager.enqueueListing({
|
||||
source_id: "radiookapi",
|
||||
env: "test",
|
||||
});
|
||||
const job = await manager.enqueueListing({
|
||||
source_id: "radiookapi",
|
||||
env: "test",
|
||||
});
|
||||
|
||||
expect(job.id).toBe("collect_listing-1");
|
||||
expect(queue.jobs[0]).toEqual({
|
||||
name: "collect_listing",
|
||||
data: {
|
||||
source_id: "radiookapi",
|
||||
env: "test",
|
||||
page_range: undefined,
|
||||
date_range: undefined,
|
||||
category: undefined,
|
||||
},
|
||||
});
|
||||
});
|
||||
expect(job.id).toBe("collect_listing-1");
|
||||
expect(queue.jobs[0]).toEqual({
|
||||
name: "collect_listing",
|
||||
data: {
|
||||
source_id: "radiookapi",
|
||||
env: "test",
|
||||
page_range: undefined,
|
||||
date_range: undefined,
|
||||
category: undefined,
|
||||
},
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1 +1 @@
|
||||
export * from "../services/crawler/async/queue";
|
||||
export * from "@basango/crawler/services/async/queue";
|
||||
|
||||
@@ -1,35 +1,35 @@
|
||||
import { describe, expect, it } from "vitest";
|
||||
|
||||
import {
|
||||
PipelineConfigSchema,
|
||||
createDateRange,
|
||||
formatDateRange,
|
||||
isTimestampInRange,
|
||||
PageRangeSpecSchema,
|
||||
PageRangeSchema,
|
||||
schemaToJSON,
|
||||
PipelineConfigSchema,
|
||||
createDateRange,
|
||||
formatDateRange,
|
||||
isTimestampInRange,
|
||||
PageRangeSpecSchema,
|
||||
PageRangeSchema,
|
||||
schemaToJSON,
|
||||
} from "./schema";
|
||||
|
||||
describe("schema helpers", () => {
|
||||
it("creates date range from spec", () => {
|
||||
const range = createDateRange("2024-01-01:2024-01-31");
|
||||
expect(range.start).toBeLessThan(range.end);
|
||||
expect(formatDateRange(range)).toBe("2024-01-01:2024-01-31");
|
||||
});
|
||||
it("creates date range from spec", () => {
|
||||
const range = createDateRange("2024-01-01:2024-01-31");
|
||||
expect(range.start).toBeLessThan(range.end);
|
||||
expect(formatDateRange(range)).toBe("2024-01-01:2024-01-31");
|
||||
});
|
||||
|
||||
it("checks membership", () => {
|
||||
const range = createDateRange("2024-01-01:2024-01-02");
|
||||
expect(isTimestampInRange(range, range.start)).toBe(true);
|
||||
expect(isTimestampInRange(range, range.start - 1)).toBe(false);
|
||||
});
|
||||
it("checks membership", () => {
|
||||
const range = createDateRange("2024-01-01:2024-01-02");
|
||||
expect(isTimestampInRange(range, range.start)).toBe(true);
|
||||
expect(isTimestampInRange(range, range.start - 1)).toBe(false);
|
||||
});
|
||||
|
||||
it("parses page range spec", () => {
|
||||
const range = PageRangeSchema.parse(PageRangeSpecSchema.parse("1:10"));
|
||||
expect(range).toEqual({ start: 1, end: 10 });
|
||||
});
|
||||
it("parses page range spec", () => {
|
||||
const range = PageRangeSchema.parse(PageRangeSpecSchema.parse("1:10"));
|
||||
expect(range).toEqual({ start: 1, end: 10 });
|
||||
});
|
||||
|
||||
it("produces json schema", () => {
|
||||
const json = schemaToJSON(PipelineConfigSchema);
|
||||
expect(json.type).toBe("object");
|
||||
});
|
||||
it("produces json schema", () => {
|
||||
const json = schemaToJSON(PipelineConfigSchema);
|
||||
expect(json.type).toBe("object");
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1,50 +1,50 @@
|
||||
import { describe, expect, it, vi } from "vitest";
|
||||
|
||||
import {
|
||||
scheduleAsyncCrawl,
|
||||
registerCrawlerTaskHandlers,
|
||||
collectListing,
|
||||
scheduleAsyncCrawl,
|
||||
registerCrawlerTaskHandlers,
|
||||
collectListing,
|
||||
} from "./tasks";
|
||||
import { QueueManager } from "./queue";
|
||||
|
||||
describe("Async tasks", () => {
|
||||
it("schedules crawl with provided manager", async () => {
|
||||
const enqueueListing = vi.fn().mockResolvedValue({ id: "job-1" });
|
||||
const manager = {
|
||||
enqueueListing,
|
||||
} as unknown as QueueManager;
|
||||
it("schedules crawl with provided manager", async () => {
|
||||
const enqueueListing = vi.fn().mockResolvedValue({ id: "job-1" });
|
||||
const manager = {
|
||||
enqueueListing,
|
||||
} as unknown as QueueManager;
|
||||
|
||||
const jobId = await scheduleAsyncCrawl({
|
||||
sourceId: "radiookapi",
|
||||
queueManager: manager,
|
||||
});
|
||||
const jobId = await scheduleAsyncCrawl({
|
||||
sourceId: "radiookapi",
|
||||
queueManager: manager,
|
||||
});
|
||||
|
||||
expect(jobId).toBe("job-1");
|
||||
expect(enqueueListing).toHaveBeenCalledWith({
|
||||
source_id: "radiookapi",
|
||||
env: "development",
|
||||
page_range: undefined,
|
||||
date_range: undefined,
|
||||
category: undefined,
|
||||
});
|
||||
});
|
||||
expect(jobId).toBe("job-1");
|
||||
expect(enqueueListing).toHaveBeenCalledWith({
|
||||
source_id: "radiookapi",
|
||||
env: "development",
|
||||
page_range: undefined,
|
||||
date_range: undefined,
|
||||
category: undefined,
|
||||
});
|
||||
});
|
||||
|
||||
it("delegates listing collection to registered handler", async () => {
|
||||
const handler = vi.fn().mockResolvedValue(5);
|
||||
registerCrawlerTaskHandlers({ collectListing: handler });
|
||||
it("delegates listing collection to registered handler", async () => {
|
||||
const handler = vi.fn().mockResolvedValue(5);
|
||||
registerCrawlerTaskHandlers({ collectListing: handler });
|
||||
|
||||
const count = await collectListing({
|
||||
source_id: "radiookapi",
|
||||
env: "development",
|
||||
});
|
||||
const count = await collectListing({
|
||||
source_id: "radiookapi",
|
||||
env: "development",
|
||||
});
|
||||
|
||||
expect(count).toBe(5);
|
||||
expect(handler).toHaveBeenCalledWith({
|
||||
source_id: "radiookapi",
|
||||
env: "development",
|
||||
page_range: undefined,
|
||||
date_range: undefined,
|
||||
category: undefined,
|
||||
});
|
||||
});
|
||||
expect(count).toBe(5);
|
||||
expect(handler).toHaveBeenCalledWith({
|
||||
source_id: "radiookapi",
|
||||
env: "development",
|
||||
page_range: undefined,
|
||||
date_range: undefined,
|
||||
category: undefined,
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
@@ -1 +1 @@
|
||||
export * from "../services/crawler/async/tasks";
|
||||
export * from "@basango/crawler/services/async/tasks";
|
||||
|
||||
+125
-129
@@ -1,186 +1,182 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import * as fs from "node:fs";
|
||||
import * as path from "node:path";
|
||||
|
||||
import { logger } from "@basango/logger";
|
||||
import {logger} from "@basango/logger";
|
||||
|
||||
import {
|
||||
PipelineConfig,
|
||||
PipelineConfigSchema,
|
||||
mergePipelineConfig,
|
||||
resolveConfigPath,
|
||||
resolveProjectPaths,
|
||||
mergePipelineConfig,
|
||||
PipelineConfig,
|
||||
PipelineConfigSchema,
|
||||
resolveConfigPath,
|
||||
resolveProjectPaths,
|
||||
} from "./schema";
|
||||
import { ensureDirectories } from "./utils";
|
||||
import {ensureDirectories} from "./utils";
|
||||
|
||||
export interface LoadConfigOptions {
|
||||
configPath?: string;
|
||||
env?: string;
|
||||
configPath?: string;
|
||||
env?: string;
|
||||
}
|
||||
|
||||
const DEFAULT_CONFIG_FILES = [
|
||||
path.join(process.cwd(), "config", "pipeline.json"),
|
||||
path.join(process.cwd(), "pipeline.json"),
|
||||
path.join(process.cwd(), "config", "pipeline.json"),
|
||||
path.join(process.cwd(), "pipeline.json"),
|
||||
];
|
||||
|
||||
const readJsonFile = (filePath: string): unknown => {
|
||||
const contents = fs.readFileSync(filePath, "utf-8");
|
||||
return contents.trim() === "" ? {} : JSON.parse(contents);
|
||||
const contents = fs.readFileSync(filePath, "utf-8");
|
||||
return contents.trim() === "" ? {} : JSON.parse(contents);
|
||||
};
|
||||
|
||||
export const locateConfigFile = (explicit?: string): string => {
|
||||
if (explicit && fs.existsSync(explicit)) {
|
||||
return explicit;
|
||||
}
|
||||
const locateConfigFile = (explicit?: string): string => {
|
||||
if (explicit && fs.existsSync(explicit)) {
|
||||
return explicit;
|
||||
}
|
||||
|
||||
for (const candidate of DEFAULT_CONFIG_FILES) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
for (const candidate of DEFAULT_CONFIG_FILES) {
|
||||
if (fs.existsSync(candidate)) {
|
||||
return candidate;
|
||||
}
|
||||
}
|
||||
|
||||
return DEFAULT_CONFIG_FILES[0];
|
||||
return DEFAULT_CONFIG_FILES[0];
|
||||
};
|
||||
|
||||
const readPipelineConfig = (configPath: string): PipelineConfig => {
|
||||
if (!fs.existsSync(configPath)) {
|
||||
return PipelineConfigSchema.parse({
|
||||
paths: resolveProjectPaths(path.resolve(".")),
|
||||
});
|
||||
}
|
||||
if (!fs.existsSync(configPath)) {
|
||||
return PipelineConfigSchema.parse({
|
||||
paths: resolveProjectPaths(path.resolve(".")),
|
||||
});
|
||||
}
|
||||
|
||||
const raw = readJsonFile(configPath);
|
||||
return PipelineConfigSchema.parse(raw);
|
||||
const raw = readJsonFile(configPath);
|
||||
return PipelineConfigSchema.parse(raw);
|
||||
};
|
||||
|
||||
const applyEnvironmentOverride = (
|
||||
baseConfig: PipelineConfig,
|
||||
basePath: string,
|
||||
env?: string,
|
||||
baseConfig: PipelineConfig,
|
||||
basePath: string,
|
||||
env?: string,
|
||||
): PipelineConfig => {
|
||||
if (!env || env === "development") {
|
||||
return baseConfig;
|
||||
}
|
||||
if (!env || env === "development") {
|
||||
return baseConfig;
|
||||
}
|
||||
|
||||
const overridePath = resolveConfigPath(basePath, env);
|
||||
if (!fs.existsSync(overridePath)) {
|
||||
return baseConfig;
|
||||
}
|
||||
const overridePath = resolveConfigPath(basePath, env);
|
||||
if (!fs.existsSync(overridePath)) {
|
||||
return baseConfig;
|
||||
}
|
||||
|
||||
const overrides = PipelineConfigSchema.parse(readJsonFile(overridePath));
|
||||
return mergePipelineConfig(baseConfig, overrides);
|
||||
const overrides = PipelineConfigSchema.parse(readJsonFile(overridePath));
|
||||
return mergePipelineConfig(baseConfig, overrides);
|
||||
};
|
||||
|
||||
export const loadConfig = (options: LoadConfigOptions = {}): PipelineConfig => {
|
||||
const basePath = locateConfigFile(options.configPath);
|
||||
const config = applyEnvironmentOverride(
|
||||
readPipelineConfig(basePath),
|
||||
basePath,
|
||||
options.env,
|
||||
);
|
||||
const basePath = locateConfigFile(options.configPath);
|
||||
const config = applyEnvironmentOverride(
|
||||
readPipelineConfig(basePath),
|
||||
basePath,
|
||||
options.env,
|
||||
);
|
||||
|
||||
ensureDirectories(config.paths);
|
||||
return config;
|
||||
ensureDirectories(config.paths);
|
||||
return config;
|
||||
};
|
||||
|
||||
export const dumpConfig = (
|
||||
config: PipelineConfig,
|
||||
targetPath?: string,
|
||||
config: PipelineConfig,
|
||||
targetPath?: string,
|
||||
): void => {
|
||||
const destination = targetPath ?? locateConfigFile();
|
||||
const normalized = PipelineConfigSchema.parse(config);
|
||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
||||
fs.writeFileSync(destination, JSON.stringify(normalized, null, 2));
|
||||
const destination = targetPath ?? locateConfigFile();
|
||||
const normalized = PipelineConfigSchema.parse(config);
|
||||
fs.mkdirSync(path.dirname(destination), {recursive: true});
|
||||
fs.writeFileSync(destination, JSON.stringify(normalized, null, 2));
|
||||
};
|
||||
|
||||
export interface PipelineConfigManagerOptions {
|
||||
configPath?: string;
|
||||
env?: string;
|
||||
autoLoad?: boolean;
|
||||
configPath?: string;
|
||||
env?: string;
|
||||
autoLoad?: boolean;
|
||||
}
|
||||
|
||||
export class PipelineConfigManager {
|
||||
private readonly explicitPath?: string;
|
||||
private readonly explicitPath?: string;
|
||||
|
||||
private readonly defaultEnv: string;
|
||||
private readonly defaultEnv: string;
|
||||
|
||||
private cache?: PipelineConfig;
|
||||
private cache?: PipelineConfig;
|
||||
|
||||
constructor(options: PipelineConfigManagerOptions = {}) {
|
||||
this.explicitPath = options.configPath;
|
||||
this.defaultEnv = options.env ?? "development";
|
||||
constructor(options: PipelineConfigManagerOptions = {}) {
|
||||
this.explicitPath = options.configPath;
|
||||
this.defaultEnv = options.env ?? "development";
|
||||
|
||||
if (options.autoLoad !== false) {
|
||||
this.cache = loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: this.defaultEnv,
|
||||
});
|
||||
}
|
||||
}
|
||||
if (options.autoLoad !== false) {
|
||||
this.cache = loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: this.defaultEnv,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
get(env?: string): PipelineConfig {
|
||||
const resolvedEnv = env ?? this.defaultEnv;
|
||||
get(env?: string): PipelineConfig {
|
||||
const resolvedEnv = env ?? this.defaultEnv;
|
||||
|
||||
if (resolvedEnv !== this.defaultEnv) {
|
||||
return loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: resolvedEnv,
|
||||
});
|
||||
}
|
||||
if (resolvedEnv !== this.defaultEnv) {
|
||||
return loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: resolvedEnv,
|
||||
});
|
||||
}
|
||||
|
||||
if (!this.cache) {
|
||||
this.cache = loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: resolvedEnv,
|
||||
});
|
||||
}
|
||||
if (!this.cache) {
|
||||
this.cache = loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: resolvedEnv,
|
||||
});
|
||||
}
|
||||
|
||||
return this.cache;
|
||||
}
|
||||
return this.cache;
|
||||
}
|
||||
|
||||
reload(env?: string): PipelineConfig {
|
||||
const resolvedEnv = env ?? this.defaultEnv;
|
||||
const config = loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: resolvedEnv,
|
||||
});
|
||||
reload(env?: string): PipelineConfig {
|
||||
const resolvedEnv = env ?? this.defaultEnv;
|
||||
const config = loadConfig({
|
||||
configPath: this.explicitPath,
|
||||
env: resolvedEnv,
|
||||
});
|
||||
|
||||
if (resolvedEnv === this.defaultEnv) {
|
||||
this.cache = config;
|
||||
}
|
||||
if (resolvedEnv === this.defaultEnv) {
|
||||
this.cache = config;
|
||||
}
|
||||
|
||||
return config;
|
||||
}
|
||||
return config;
|
||||
}
|
||||
|
||||
ensureDirectories(config?: PipelineConfig): PipelineConfig {
|
||||
const pipeline = config ?? this.get();
|
||||
ensureDirectories(pipeline.paths);
|
||||
return pipeline;
|
||||
}
|
||||
ensureDirectories(config?: PipelineConfig): PipelineConfig {
|
||||
const pipeline = config ?? this.get();
|
||||
ensureDirectories(pipeline.paths);
|
||||
return pipeline;
|
||||
}
|
||||
|
||||
setupLogging(config?: PipelineConfig): void {
|
||||
const pipeline = config ?? this.get();
|
||||
this.ensureDirectories(pipeline);
|
||||
setupLogging(config?: PipelineConfig): void {
|
||||
const pipeline = config ?? this.get();
|
||||
this.ensureDirectories(pipeline);
|
||||
|
||||
const level = pipeline.logging.level.toLowerCase();
|
||||
process.env.LOG_LEVEL = level;
|
||||
const normalizedLevel = level as typeof logger.level;
|
||||
logger.level = normalizedLevel;
|
||||
const level = pipeline.logging.level.toLowerCase();
|
||||
process.env.LOG_LEVEL = level;
|
||||
logger.level = level as typeof logger.level;
|
||||
|
||||
if (pipeline.logging.file_logging) {
|
||||
const logDir = pipeline.paths.logs;
|
||||
const destination = path.join(
|
||||
logDir,
|
||||
pipeline.logging.log_file,
|
||||
);
|
||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
||||
if (!fs.existsSync(destination)) {
|
||||
fs.writeFileSync(destination, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
if (pipeline.logging.file_logging) {
|
||||
const logDir = pipeline.paths.logs;
|
||||
const destination = path.join(logDir, pipeline.logging.log_file);
|
||||
fs.mkdirSync(path.dirname(destination), {recursive: true});
|
||||
if (!fs.existsSync(destination)) {
|
||||
fs.writeFileSync(destination, "");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
resolveConfigPath(env?: string): string {
|
||||
const base = locateConfigFile(this.explicitPath);
|
||||
return resolveConfigPath(base, env ?? this.defaultEnv);
|
||||
}
|
||||
resolveConfigPath(env?: string): string {
|
||||
const base = locateConfigFile(this.explicitPath);
|
||||
return resolveConfigPath(base, env ?? this.defaultEnv);
|
||||
}
|
||||
}
|
||||
|
||||
+216
-221
@@ -1,7 +1,7 @@
|
||||
import path from "node:path";
|
||||
import * as path from "node:path";
|
||||
|
||||
import { getUnixTime, parse, isMatch, format as formatDate } from "date-fns";
|
||||
import { z } from "zod";
|
||||
import {format as formatDate, getUnixTime, isMatch, parse} from "date-fns";
|
||||
import {z} from "zod";
|
||||
|
||||
export const UpdateDirectionSchema = z.enum(["forward", "backward"]);
|
||||
export type UpdateDirection = z.infer<typeof UpdateDirectionSchema>;
|
||||
@@ -10,47 +10,47 @@ export const SourceKindSchema = z.enum(["wordpress", "html"]);
|
||||
export type SourceKind = z.infer<typeof SourceKindSchema>;
|
||||
|
||||
export const SourceDateSchema = z.object({
|
||||
format: z.string().default("yyyy-LL-dd HH:mm"),
|
||||
pattern: z.string().nullable().optional(),
|
||||
replacement: z.string().nullable().optional(),
|
||||
format: z.string().default("yyyy-LL-dd HH:mm"),
|
||||
pattern: z.string().nullable().optional(),
|
||||
replacement: z.string().nullable().optional(),
|
||||
});
|
||||
export type SourceDate = z.infer<typeof SourceDateSchema>;
|
||||
|
||||
export const SourceSelectorsSchema = z.object({
|
||||
articles: z.string().optional().nullable(),
|
||||
article_title: z.string().optional().nullable(),
|
||||
article_link: z.string().optional().nullable(),
|
||||
article_body: z.string().optional().nullable(),
|
||||
article_date: z.string().optional().nullable(),
|
||||
article_categories: z.string().optional().nullable(),
|
||||
pagination: z.string().default("ul.pagination > li a"),
|
||||
articles: z.string().optional().nullable(),
|
||||
article_title: z.string().optional().nullable(),
|
||||
article_link: z.string().optional().nullable(),
|
||||
article_body: z.string().optional().nullable(),
|
||||
article_date: z.string().optional().nullable(),
|
||||
article_categories: z.string().optional().nullable(),
|
||||
pagination: z.string().default("ul.pagination > li a"),
|
||||
});
|
||||
export type SourceSelectors = z.infer<typeof SourceSelectorsSchema>;
|
||||
|
||||
const BaseSourceSchema = z.object({
|
||||
source_id: z.string(),
|
||||
source_url: z.string().url(),
|
||||
source_date: SourceDateSchema.default(SourceDateSchema.parse({})),
|
||||
source_kind: SourceKindSchema,
|
||||
categories: z.array(z.string()).default([]),
|
||||
supports_categories: z.boolean().default(false),
|
||||
requires_details: z.boolean().default(false),
|
||||
requires_rate_limit: z.boolean().default(false),
|
||||
source_id: z.string(),
|
||||
source_url: z.url(),
|
||||
source_date: SourceDateSchema.default(SourceDateSchema.parse({})),
|
||||
source_kind: SourceKindSchema,
|
||||
categories: z.array(z.string()).default([]),
|
||||
supports_categories: z.boolean().default(false),
|
||||
requires_details: z.boolean().default(false),
|
||||
requires_rate_limit: z.boolean().default(false),
|
||||
});
|
||||
|
||||
export const HtmlSourceConfigSchema = BaseSourceSchema.extend({
|
||||
source_kind: z.literal("html"),
|
||||
source_selectors: SourceSelectorsSchema.default(
|
||||
SourceSelectorsSchema.parse({}),
|
||||
),
|
||||
pagination_template: z.string(),
|
||||
source_kind: z.literal("html"),
|
||||
source_selectors: SourceSelectorsSchema.default(
|
||||
SourceSelectorsSchema.parse({}),
|
||||
),
|
||||
pagination_template: z.string(),
|
||||
});
|
||||
|
||||
export const WordPressSourceConfigSchema = BaseSourceSchema.extend({
|
||||
source_kind: z.literal("wordpress"),
|
||||
source_date: SourceDateSchema.default(
|
||||
SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" }),
|
||||
),
|
||||
source_kind: z.literal("wordpress"),
|
||||
source_date: SourceDateSchema.default(
|
||||
SourceDateSchema.parse({format: "yyyy-LL-dd'T'HH:mm:ss"}),
|
||||
),
|
||||
});
|
||||
|
||||
export type HtmlSourceConfig = z.infer<typeof HtmlSourceConfigSchema>;
|
||||
@@ -58,279 +58,274 @@ export type WordPressSourceConfig = z.infer<typeof WordPressSourceConfigSchema>;
|
||||
export type AnySourceConfig = HtmlSourceConfig | WordPressSourceConfig;
|
||||
|
||||
export const DateRangeSchema = z
|
||||
.object({
|
||||
start: z.number().int(),
|
||||
end: z.number().int(),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.start === 0 || value.end === 0) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "Timestamp cannot be zero",
|
||||
});
|
||||
}
|
||||
if (value.end < value.start) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "End timestamp must be greater than or equal to start",
|
||||
});
|
||||
}
|
||||
});
|
||||
.object({
|
||||
start: z.number().int(),
|
||||
end: z.number().int(),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.start === 0 || value.end === 0) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "Timestamp cannot be zero",
|
||||
});
|
||||
}
|
||||
if (value.end < value.start) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "End timestamp must be greater than or equal to start",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type DateRange = z.infer<typeof DateRangeSchema>;
|
||||
|
||||
export const PageRangeSchema = z
|
||||
.object({
|
||||
start: z.number().int().min(0),
|
||||
end: z.number().int().min(0),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.end < value.start) {
|
||||
ctx.addIssue({
|
||||
code: z.ZodIssueCode.custom,
|
||||
message: "End page must be greater than or equal to start page",
|
||||
});
|
||||
}
|
||||
});
|
||||
.object({
|
||||
start: z.number().int().min(0),
|
||||
end: z.number().int().min(0),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.end < value.start) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "End page must be greater than or equal to start page",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export type PageRange = z.infer<typeof PageRangeSchema>;
|
||||
|
||||
export const PageRangeSpecSchema = z
|
||||
.string()
|
||||
.regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end")
|
||||
.transform((spec) => {
|
||||
const [startText, endText] = spec.split(":");
|
||||
return {
|
||||
start: Number.parseInt(startText, 10),
|
||||
end: Number.parseInt(endText, 10),
|
||||
};
|
||||
});
|
||||
.string()
|
||||
.regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end")
|
||||
.transform((spec) => {
|
||||
const [startText, endText] = spec.split(":");
|
||||
return {
|
||||
start: Number.parseInt(startText, 10),
|
||||
end: Number.parseInt(endText, 10),
|
||||
};
|
||||
});
|
||||
|
||||
const defaultDateFormat = "yyyy-LL-dd";
|
||||
|
||||
export const DateRangeSpecSchema = z
|
||||
.string()
|
||||
.regex(/.+:.+/, "Expected start:end format")
|
||||
.transform((spec) => {
|
||||
const [startRaw, endRaw] = spec.split(":");
|
||||
return { startRaw, endRaw };
|
||||
});
|
||||
.string()
|
||||
.regex(/.+:.+/, "Expected start:end format")
|
||||
.transform((spec) => {
|
||||
const [startRaw, endRaw] = spec.split(":");
|
||||
return {startRaw, endRaw};
|
||||
});
|
||||
|
||||
const parseDate = (value: string, format: string): Date => {
|
||||
if (!isMatch(value, format)) {
|
||||
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
||||
}
|
||||
const parsed = parse(value, format, new Date());
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
||||
}
|
||||
return parsed;
|
||||
if (!isMatch(value, format)) {
|
||||
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
||||
}
|
||||
const parsed = parse(value, format, new Date());
|
||||
if (Number.isNaN(parsed.getTime())) {
|
||||
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
||||
}
|
||||
return parsed;
|
||||
};
|
||||
|
||||
export interface CreateDateRangeOptions {
|
||||
format?: string;
|
||||
separator?: string;
|
||||
format?: string;
|
||||
separator?: string;
|
||||
}
|
||||
|
||||
export const createDateRange = (
|
||||
spec: string,
|
||||
options: CreateDateRangeOptions = {},
|
||||
spec: string,
|
||||
options: CreateDateRangeOptions = {},
|
||||
): DateRange => {
|
||||
const { format = defaultDateFormat, separator = ":" } = options;
|
||||
if (!separator) {
|
||||
throw new Error("Separator cannot be empty");
|
||||
}
|
||||
const {format = defaultDateFormat, separator = ":"} = options;
|
||||
if (!separator) {
|
||||
throw new Error("Separator cannot be empty");
|
||||
}
|
||||
|
||||
const normalized = spec.replace(separator, ":");
|
||||
const parsedSpec = DateRangeSpecSchema.parse(normalized);
|
||||
const normalized = spec.replace(separator, ":");
|
||||
const parsedSpec = DateRangeSpecSchema.parse(normalized);
|
||||
|
||||
const startDate = parseDate(parsedSpec.startRaw, format);
|
||||
const endDate = parseDate(parsedSpec.endRaw, format);
|
||||
const startDate = parseDate(parsedSpec.startRaw, format);
|
||||
const endDate = parseDate(parsedSpec.endRaw, format);
|
||||
|
||||
const range = {
|
||||
start: getUnixTime(startDate),
|
||||
end: getUnixTime(endDate),
|
||||
};
|
||||
const range = {
|
||||
start: getUnixTime(startDate),
|
||||
end: getUnixTime(endDate),
|
||||
};
|
||||
|
||||
return DateRangeSchema.parse(range);
|
||||
return DateRangeSchema.parse(range);
|
||||
};
|
||||
|
||||
export const formatDateRange = (
|
||||
range: DateRange,
|
||||
fmt = defaultDateFormat,
|
||||
range: DateRange,
|
||||
fmt = defaultDateFormat,
|
||||
): string => {
|
||||
const start = formatDate(new Date(range.start * 1000), fmt);
|
||||
const end = formatDate(new Date(range.end * 1000), fmt);
|
||||
return `${start}:${end}`;
|
||||
const start = formatDate(new Date(range.start * 1000), fmt);
|
||||
const end = formatDate(new Date(range.end * 1000), fmt);
|
||||
return `${start}:${end}`;
|
||||
};
|
||||
|
||||
export const isTimestampInRange = (
|
||||
range: DateRange,
|
||||
timestamp: number,
|
||||
range: DateRange,
|
||||
timestamp: number,
|
||||
): boolean => {
|
||||
return range.start <= timestamp && timestamp <= range.end;
|
||||
return range.start <= timestamp && timestamp <= range.end;
|
||||
};
|
||||
|
||||
export const ProjectPathsSchema = z.object({
|
||||
root: z.string(),
|
||||
data: z.string(),
|
||||
logs: z.string(),
|
||||
configs: z.string(),
|
||||
root: z.string(),
|
||||
data: z.string(),
|
||||
logs: z.string(),
|
||||
configs: z.string(),
|
||||
});
|
||||
export type ProjectPaths = z.infer<typeof ProjectPathsSchema>;
|
||||
|
||||
export const resolveProjectPaths = (rootDir: string): ProjectPaths => {
|
||||
return ProjectPathsSchema.parse({
|
||||
root: rootDir,
|
||||
data: path.join(rootDir, "data", "dataset"),
|
||||
logs: path.join(rootDir, "data", "logs"),
|
||||
configs: path.join(rootDir, "config"),
|
||||
});
|
||||
return ProjectPathsSchema.parse({
|
||||
root: rootDir,
|
||||
data: path.join(rootDir, "data", "dataset"),
|
||||
logs: path.join(rootDir, "data", "logs"),
|
||||
configs: path.join(rootDir, "config"),
|
||||
});
|
||||
};
|
||||
|
||||
export const LoggingConfigSchema = z.object({
|
||||
level: z.string().default("INFO"),
|
||||
format: z
|
||||
.string()
|
||||
.default("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
|
||||
console_logging: z.boolean().default(true),
|
||||
file_logging: z.boolean().default(false),
|
||||
log_file: z.string().default("crawler.log"),
|
||||
max_log_size: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(10 * 1024 * 1024),
|
||||
backup_count: z.number().int().nonnegative().default(5),
|
||||
level: z.string().default("INFO"),
|
||||
format: z
|
||||
.string()
|
||||
.default("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
|
||||
console_logging: z.boolean().default(true),
|
||||
file_logging: z.boolean().default(false),
|
||||
log_file: z.string().default("crawler.log"),
|
||||
max_log_size: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(10 * 1024 * 1024),
|
||||
backup_count: z.number().int().nonnegative().default(5),
|
||||
});
|
||||
export type LoggingConfig = z.infer<typeof LoggingConfigSchema>;
|
||||
|
||||
export const ClientConfigSchema = z.object({
|
||||
timeout: z.number().positive().default(20),
|
||||
user_agent: z
|
||||
.string()
|
||||
.default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
|
||||
follow_redirects: z.boolean().default(true),
|
||||
verify_ssl: z.boolean().default(true),
|
||||
rotate: z.boolean().default(true),
|
||||
max_retries: z.number().int().nonnegative().default(3),
|
||||
backoff_initial: z.number().nonnegative().default(1),
|
||||
backoff_multiplier: z.number().positive().default(2),
|
||||
backoff_max: z.number().nonnegative().default(30),
|
||||
respect_retry_after: z.boolean().default(true),
|
||||
timeout: z.number().positive().default(20),
|
||||
user_agent: z
|
||||
.string()
|
||||
.default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
|
||||
follow_redirects: z.boolean().default(true),
|
||||
verify_ssl: z.boolean().default(true),
|
||||
rotate: z.boolean().default(true),
|
||||
max_retries: z.number().int().nonnegative().default(3),
|
||||
backoff_initial: z.number().nonnegative().default(1),
|
||||
backoff_multiplier: z.number().positive().default(2),
|
||||
backoff_max: z.number().nonnegative().default(30),
|
||||
respect_retry_after: z.boolean().default(true),
|
||||
});
|
||||
|
||||
export const CrawlerConfigSchema = z.object({
|
||||
source: z
|
||||
.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema])
|
||||
.optional(),
|
||||
page_range: PageRangeSchema.optional(),
|
||||
date_range: DateRangeSchema.optional(),
|
||||
category: z.string().optional(),
|
||||
notify: z.boolean().default(false),
|
||||
is_update: z.boolean().default(false),
|
||||
use_multi_threading: z.boolean().default(false),
|
||||
max_workers: z.number().int().positive().default(5),
|
||||
direction: UpdateDirectionSchema.default("forward"),
|
||||
source: z
|
||||
.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema])
|
||||
.optional(),
|
||||
page_range: PageRangeSchema.optional(),
|
||||
date_range: DateRangeSchema.optional(),
|
||||
category: z.string().optional(),
|
||||
notify: z.boolean().default(false),
|
||||
is_update: z.boolean().default(false),
|
||||
use_multi_threading: z.boolean().default(false),
|
||||
max_workers: z.number().int().positive().default(5),
|
||||
direction: UpdateDirectionSchema.default("forward"),
|
||||
});
|
||||
|
||||
export type ClientConfig = z.infer<typeof ClientConfigSchema>;
|
||||
export type CrawlerConfig = z.infer<typeof CrawlerConfigSchema> & {
|
||||
source?: AnySourceConfig;
|
||||
source?: AnySourceConfig;
|
||||
};
|
||||
|
||||
export const FetchConfigSchema = z.object({
|
||||
client: ClientConfigSchema.default(ClientConfigSchema.parse({})),
|
||||
crawler: CrawlerConfigSchema.default(CrawlerConfigSchema.parse({})),
|
||||
client: ClientConfigSchema.default(ClientConfigSchema.parse({})),
|
||||
crawler: CrawlerConfigSchema.default(CrawlerConfigSchema.parse({})),
|
||||
});
|
||||
export type FetchConfig = z.infer<typeof FetchConfigSchema>;
|
||||
|
||||
const SourcesConfigSchema = z.object({
|
||||
html: z.array(HtmlSourceConfigSchema).default([]),
|
||||
wordpress: z.array(WordPressSourceConfigSchema).default([]),
|
||||
html: z.array(HtmlSourceConfigSchema).default([]),
|
||||
wordpress: z.array(WordPressSourceConfigSchema).default([]),
|
||||
});
|
||||
|
||||
export type SourcesConfig = z.infer<typeof SourcesConfigSchema> & {
|
||||
find: (sourceId: string) => AnySourceConfig | undefined;
|
||||
find: (sourceId: string) => AnySourceConfig | undefined;
|
||||
};
|
||||
|
||||
export const createSourcesConfig = (input: unknown): SourcesConfig => {
|
||||
const parsed = SourcesConfigSchema.parse(input);
|
||||
const resolver = (sourceId: string) =>
|
||||
[...parsed.html, ...parsed.wordpress].find(
|
||||
(source) => source.source_id === sourceId,
|
||||
);
|
||||
return Object.assign({ find: resolver }, parsed);
|
||||
const parsed = SourcesConfigSchema.parse(input);
|
||||
const resolver = (sourceId: string) =>
|
||||
[...parsed.html, ...parsed.wordpress].find(
|
||||
(source) => source.source_id === sourceId,
|
||||
);
|
||||
return Object.assign({find: resolver}, parsed);
|
||||
};
|
||||
|
||||
export const PipelineConfigSchema = z.object({
|
||||
paths: ProjectPathsSchema.default(resolveProjectPaths(process.cwd())),
|
||||
logging: LoggingConfigSchema.default(LoggingConfigSchema.parse({})),
|
||||
fetch: FetchConfigSchema.default(FetchConfigSchema.parse({})),
|
||||
sources: z
|
||||
.union([SourcesConfigSchema, z.undefined()])
|
||||
.transform((value) => createSourcesConfig(value ?? {})),
|
||||
paths: ProjectPathsSchema.default(resolveProjectPaths(process.cwd())),
|
||||
logging: LoggingConfigSchema.default(LoggingConfigSchema.parse({})),
|
||||
fetch: FetchConfigSchema.default(FetchConfigSchema.parse({})),
|
||||
sources: z
|
||||
.union([SourcesConfigSchema, z.undefined()])
|
||||
.transform((value) => createSourcesConfig(value ?? {})),
|
||||
});
|
||||
|
||||
export type PipelineConfig = z.infer<typeof PipelineConfigSchema> & {
|
||||
sources: SourcesConfig;
|
||||
};
|
||||
export type PipelineConfig = z.infer<typeof PipelineConfigSchema>
|
||||
|
||||
export const mergePipelineConfig = (
|
||||
base: PipelineConfig,
|
||||
overrides: Partial<PipelineConfig>,
|
||||
base: PipelineConfig,
|
||||
overrides: Partial<PipelineConfig>,
|
||||
): PipelineConfig => {
|
||||
const paths = overrides.paths ?? base.paths;
|
||||
const logging = { ...base.logging, ...(overrides.logging ?? {}) };
|
||||
const fetch = {
|
||||
client: { ...base.fetch.client, ...(overrides.fetch?.client ?? {}) },
|
||||
crawler: { ...base.fetch.crawler, ...(overrides.fetch?.crawler ?? {}) },
|
||||
};
|
||||
const paths = overrides.paths ?? base.paths;
|
||||
const logging = {...base.logging, ...(overrides.logging ?? {})};
|
||||
const fetch = {
|
||||
client: {...base.fetch.client, ...(overrides.fetch?.client ?? {})},
|
||||
crawler: {...base.fetch.crawler, ...(overrides.fetch?.crawler ?? {})},
|
||||
};
|
||||
|
||||
const sources = createSourcesConfig({
|
||||
html: overrides.sources?.html ?? base.sources.html,
|
||||
wordpress: overrides.sources?.wordpress ?? base.sources.wordpress,
|
||||
});
|
||||
const sources = createSourcesConfig({
|
||||
html: overrides.sources?.html ?? base.sources.html,
|
||||
wordpress: overrides.sources?.wordpress ?? base.sources.wordpress,
|
||||
});
|
||||
|
||||
return {
|
||||
paths,
|
||||
logging,
|
||||
fetch,
|
||||
sources,
|
||||
};
|
||||
return {
|
||||
paths,
|
||||
logging,
|
||||
fetch,
|
||||
sources,
|
||||
};
|
||||
};
|
||||
|
||||
export const resolveConfigPath = (basePath: string, env?: string): string => {
|
||||
if (!env || env === "development") {
|
||||
return basePath;
|
||||
}
|
||||
if (!env || env === "development") {
|
||||
return basePath;
|
||||
}
|
||||
|
||||
const ext = path.extname(basePath);
|
||||
const withoutExt = basePath.slice(0, basePath.length - ext.length);
|
||||
return `${withoutExt}.${env}${ext}`;
|
||||
const ext = path.extname(basePath);
|
||||
const withoutExt = basePath.slice(0, basePath.length - ext.length);
|
||||
return `${withoutExt}.${env}${ext}`;
|
||||
};
|
||||
|
||||
export const schemaToJSON = <T extends z.ZodTypeAny>(schema: T): unknown => {
|
||||
const candidate = schema as unknown as { toJSON?: () => unknown };
|
||||
if (typeof candidate.toJSON === "function") {
|
||||
return candidate.toJSON();
|
||||
}
|
||||
const toJSONSchema = (z as any).toJSONSchema as
|
||||
| ((s: z.ZodTypeAny, opts?: Record<string, unknown>) => unknown)
|
||||
| undefined;
|
||||
|
||||
const typeName = (schema as { _def?: { typeName?: z.ZodFirstPartyTypeKind } })._def
|
||||
?.typeName;
|
||||
if (typeof toJSONSchema === "function") {
|
||||
try {
|
||||
// target can be "draft-2020-12" | "draft-7" | "draft-4" | "openapi-3.0"
|
||||
return toJSONSchema(schema, {target: "draft-2020-12", unrepresentable: "any"});
|
||||
} catch {
|
||||
// fall through to minimal mapping
|
||||
}
|
||||
}
|
||||
|
||||
switch (typeName) {
|
||||
case z.ZodFirstPartyTypeKind.ZodObject:
|
||||
return { type: "object" };
|
||||
case z.ZodFirstPartyTypeKind.ZodArray:
|
||||
return { type: "array" };
|
||||
case z.ZodFirstPartyTypeKind.ZodString:
|
||||
return { type: "string" };
|
||||
case z.ZodFirstPartyTypeKind.ZodNumber:
|
||||
return { type: "number" };
|
||||
case z.ZodFirstPartyTypeKind.ZodBoolean:
|
||||
return { type: "boolean" };
|
||||
default:
|
||||
return { type: "unknown" };
|
||||
}
|
||||
};
|
||||
if (schema instanceof z.ZodObject) return {type: "object"};
|
||||
if (schema instanceof z.ZodArray) return {type: "array"};
|
||||
if (schema instanceof z.ZodString) return {type: "string"};
|
||||
if (schema instanceof z.ZodNumber) return {type: "number"};
|
||||
if (schema instanceof z.ZodBoolean) return {type: "boolean"};
|
||||
|
||||
return {type: "unknown"};
|
||||
};
|
||||
@@ -1,88 +1,88 @@
|
||||
import { parseArgs } from "node:util";
|
||||
import {parseArgs} from "node:util";
|
||||
|
||||
import { logger } from "@basango/logger";
|
||||
import {logger} from "@basango/logger";
|
||||
|
||||
import { PipelineConfigManager } from "@crawler/config";
|
||||
import { scheduleAsyncCrawl } from "@crawler/services/crawler";
|
||||
import { createQueueSettings } from "@crawler/services/crawler/async/queue";
|
||||
import {PipelineConfigManager} from "@crawler/config";
|
||||
import {createQueueSettings} from "@crawler/services/async/queue";
|
||||
import {scheduleAsyncCrawl} from "@crawler/services/async/tasks";
|
||||
|
||||
interface QueueCliOptions {
|
||||
"source-id"?: string;
|
||||
env: string;
|
||||
"page-range"?: string;
|
||||
"date-range"?: string;
|
||||
category?: string;
|
||||
"redis-url"?: string;
|
||||
help?: boolean;
|
||||
"source-id"?: string;
|
||||
env: string;
|
||||
"page-range"?: string;
|
||||
"date-range"?: string;
|
||||
category?: string;
|
||||
"redis-url"?: string;
|
||||
help?: boolean;
|
||||
}
|
||||
|
||||
const usage = `Usage: bun run src/scripts/queue.ts -- --source-id <id> [options]\n\nOptions:\n --env <env> Environment to load (default: development)\n --page-range <range> Optional page range filter (e.g. 1:5)\n --date-range <range> Optional date range filter (e.g. 2024-01-01:2024-01-31)\n --category <slug> Optional category to crawl\n --redis-url <url> Override Redis connection URL\n -h, --help Show this message`;
|
||||
|
||||
const parseCliArgs = (): QueueCliOptions => {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
"source-id": { type: "string" },
|
||||
env: { type: "string", default: "development" },
|
||||
"page-range": { type: "string" },
|
||||
"date-range": { type: "string" },
|
||||
category: { type: "string" },
|
||||
"redis-url": { type: "string" },
|
||||
help: { type: "boolean", short: "h" },
|
||||
},
|
||||
});
|
||||
const {values} = parseArgs({
|
||||
options: {
|
||||
"source-id": {type: "string"},
|
||||
env: {type: "string", default: "development"},
|
||||
"page-range": {type: "string"},
|
||||
"date-range": {type: "string"},
|
||||
category: {type: "string"},
|
||||
"redis-url": {type: "string"},
|
||||
help: {type: "boolean", short: "h"},
|
||||
},
|
||||
});
|
||||
|
||||
return values as QueueCliOptions;
|
||||
return values as QueueCliOptions;
|
||||
};
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const options = parseCliArgs();
|
||||
const options = parseCliArgs();
|
||||
|
||||
if (options.help || !options["source-id"]) {
|
||||
console.log(usage);
|
||||
if (!options["source-id"]) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (options.help || !options["source-id"]) {
|
||||
console.log(usage);
|
||||
if (!options["source-id"]) {
|
||||
process.exitCode = 1;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
const env = options.env ?? "development";
|
||||
const manager = new PipelineConfigManager({ env });
|
||||
const config = manager.ensureDirectories();
|
||||
manager.setupLogging(config);
|
||||
const env = options.env ?? "development";
|
||||
const manager = new PipelineConfigManager({env});
|
||||
const config = manager.ensureDirectories();
|
||||
manager.setupLogging(config);
|
||||
|
||||
const settings = options["redis-url"]
|
||||
? createQueueSettings({ redis_url: options["redis-url"] })
|
||||
: undefined;
|
||||
const settings = options["redis-url"]
|
||||
? createQueueSettings({redis_url: options["redis-url"]})
|
||||
: undefined;
|
||||
|
||||
try {
|
||||
const jobId = await scheduleAsyncCrawl({
|
||||
sourceId: options["source-id"],
|
||||
env,
|
||||
pageRange: options["page-range"] ?? null,
|
||||
dateRange: options["date-range"] ?? null,
|
||||
category: options.category ?? null,
|
||||
settings,
|
||||
});
|
||||
try {
|
||||
const jobId = await scheduleAsyncCrawl({
|
||||
sourceId: options["source-id"],
|
||||
env,
|
||||
pageRange: options["page-range"] ?? null,
|
||||
dateRange: options["date-range"] ?? null,
|
||||
category: options.category ?? null,
|
||||
settings,
|
||||
});
|
||||
|
||||
logger.info(
|
||||
{
|
||||
jobId,
|
||||
sourceId: options["source-id"],
|
||||
env,
|
||||
},
|
||||
"Scheduled asynchronous crawl job",
|
||||
);
|
||||
console.log(
|
||||
`Scheduled async crawl job ${jobId} for source '${options["source-id"]}' (env=${env})`,
|
||||
);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error instanceof Error ? error : { error },
|
||||
"Failed to schedule crawl job",
|
||||
);
|
||||
console.error(`Failed to schedule crawl job: ${(error as Error).message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
logger.info(
|
||||
{
|
||||
jobId,
|
||||
sourceId: options["source-id"],
|
||||
env,
|
||||
},
|
||||
"Scheduled asynchronous crawl job",
|
||||
);
|
||||
console.log(
|
||||
`Scheduled async crawl job ${jobId} for source '${options["source-id"]}' (env=${env})`,
|
||||
);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error instanceof Error ? error : {error},
|
||||
"Failed to schedule crawl job",
|
||||
);
|
||||
console.error(`Failed to schedule crawl job: ${(error as Error).message}`);
|
||||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
|
||||
void main();
|
||||
|
||||
@@ -1,112 +1,112 @@
|
||||
import { parseArgs } from "node:util";
|
||||
import {parseArgs} from "node:util";
|
||||
|
||||
import { logger } from "@basango/logger";
|
||||
import {logger} from "@basango/logger";
|
||||
|
||||
import { PipelineConfigManager } from "@crawler/config";
|
||||
import { createQueueManager, createQueueSettings } from "@crawler/services/crawler/async/queue";
|
||||
import { startWorker } from "@crawler/services/crawler/async/worker";
|
||||
import {PipelineConfigManager} from "@crawler/config";
|
||||
import {createQueueManager, createQueueSettings,} from "@crawler/services/async/queue";
|
||||
import {startWorker} from "@crawler/services/async/worker";
|
||||
|
||||
interface WorkerCliOptions {
|
||||
env: string;
|
||||
queue?: string[];
|
||||
concurrency?: string;
|
||||
"redis-url"?: string;
|
||||
help?: boolean;
|
||||
env: string;
|
||||
queue?: string[];
|
||||
concurrency?: string;
|
||||
"redis-url"?: string;
|
||||
help?: boolean;
|
||||
}
|
||||
|
||||
const usage = `Usage: bun run src/scripts/worker.ts [options]\n\nOptions:\n --env <env> Environment to load (default: development)\n -q, --queue <name> Queue name to listen on (repeatable)\n --concurrency <number> Number of concurrent jobs per worker\n --redis-url <url> Override Redis connection URL\n -h, --help Show this message`;
|
||||
|
||||
const parseCliArgs = (): WorkerCliOptions => {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
env: { type: "string", default: "development" },
|
||||
queue: { type: "string", multiple: true, short: "q" },
|
||||
concurrency: { type: "string" },
|
||||
"redis-url": { type: "string" },
|
||||
help: { type: "boolean", short: "h" },
|
||||
},
|
||||
});
|
||||
const {values} = parseArgs({
|
||||
options: {
|
||||
env: {type: "string", default: "development"},
|
||||
queue: {type: "string", multiple: true, short: "q"},
|
||||
concurrency: {type: "string"},
|
||||
"redis-url": {type: "string"},
|
||||
help: {type: "boolean", short: "h"},
|
||||
},
|
||||
});
|
||||
|
||||
return values as WorkerCliOptions;
|
||||
return values as WorkerCliOptions;
|
||||
};
|
||||
|
||||
const parseConcurrency = (value?: string): number | undefined => {
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
if (!value) {
|
||||
return undefined;
|
||||
}
|
||||
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (Number.isNaN(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid concurrency value: ${value}`);
|
||||
}
|
||||
const parsed = Number.parseInt(value, 10);
|
||||
if (Number.isNaN(parsed) || parsed <= 0) {
|
||||
throw new Error(`Invalid concurrency value: ${value}`);
|
||||
}
|
||||
|
||||
return parsed;
|
||||
return parsed;
|
||||
};
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const options = parseCliArgs();
|
||||
const options = parseCliArgs();
|
||||
|
||||
if (options.help) {
|
||||
console.log(usage);
|
||||
return;
|
||||
}
|
||||
if (options.help) {
|
||||
console.log(usage);
|
||||
return;
|
||||
}
|
||||
|
||||
const env = options.env ?? "development";
|
||||
const manager = new PipelineConfigManager({ env });
|
||||
const config = manager.ensureDirectories();
|
||||
manager.setupLogging(config);
|
||||
const env = options.env ?? "development";
|
||||
const manager = new PipelineConfigManager({env});
|
||||
const config = manager.ensureDirectories();
|
||||
manager.setupLogging(config);
|
||||
|
||||
let concurrency: number | undefined;
|
||||
try {
|
||||
concurrency = parseConcurrency(options.concurrency);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error instanceof Error ? error : { error },
|
||||
"Invalid concurrency value provided",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const settings = options["redis-url"]
|
||||
? createQueueSettings({ redis_url: options["redis-url"] })
|
||||
: undefined;
|
||||
const queueManager = createQueueManager({ settings });
|
||||
let concurrency: number | undefined;
|
||||
try {
|
||||
concurrency = parseConcurrency(options.concurrency);
|
||||
} catch (error) {
|
||||
logger.error(
|
||||
error instanceof Error ? error : {error},
|
||||
"Invalid concurrency value provided",
|
||||
);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
const settings = options["redis-url"]
|
||||
? createQueueSettings({redis_url: options["redis-url"]})
|
||||
: undefined;
|
||||
const queueManager = createQueueManager({settings});
|
||||
|
||||
const queueNames = options.queue?.length
|
||||
? options.queue.map((name) => queueManager.queueName(name))
|
||||
: undefined;
|
||||
const queueNames = options.queue?.length
|
||||
? options.queue.map((name) => queueManager.queueName(name))
|
||||
: undefined;
|
||||
|
||||
const handle = startWorker({
|
||||
queueManager,
|
||||
queueNames,
|
||||
concurrency,
|
||||
});
|
||||
const handle = startWorker({
|
||||
queueManager,
|
||||
queueNames,
|
||||
concurrency,
|
||||
});
|
||||
|
||||
const shutdown = async (signal: NodeJS.Signals) => {
|
||||
logger.info({ signal }, "Received shutdown signal, draining workers");
|
||||
try {
|
||||
await handle.close();
|
||||
} finally {
|
||||
await queueManager.close();
|
||||
process.exit(0);
|
||||
}
|
||||
};
|
||||
const shutdown = async (signal: NodeJS.Signals) => {
|
||||
logger.info({signal}, "Received shutdown signal, draining workers");
|
||||
try {
|
||||
await handle.close();
|
||||
} finally {
|
||||
await queueManager.close();
|
||||
process.exit(0);
|
||||
}
|
||||
};
|
||||
|
||||
process.once("SIGINT", (signal) => {
|
||||
void shutdown(signal);
|
||||
});
|
||||
process.once("SIGTERM", (signal) => {
|
||||
void shutdown(signal);
|
||||
});
|
||||
process.once("SIGINT", (signal) => {
|
||||
void shutdown(signal);
|
||||
});
|
||||
process.once("SIGTERM", (signal) => {
|
||||
void shutdown(signal);
|
||||
});
|
||||
|
||||
logger.info(
|
||||
{
|
||||
env,
|
||||
queueNames: queueNames ?? queueManager.iterQueueNames(),
|
||||
concurrency: concurrency ?? "default",
|
||||
},
|
||||
"Crawler workers started",
|
||||
);
|
||||
logger.info(
|
||||
{
|
||||
env,
|
||||
queueNames: queueNames ?? queueManager.iterQueueNames(),
|
||||
concurrency: concurrency ?? "default",
|
||||
},
|
||||
"Crawler workers started",
|
||||
);
|
||||
};
|
||||
|
||||
void main();
|
||||
|
||||
@@ -0,0 +1,142 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import IORedis from "ioredis";
|
||||
import { JobsOptions, Queue, QueueOptions } from "bullmq";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
ArticleTaskPayload,
|
||||
ArticleTaskPayloadSchema,
|
||||
ListingTaskPayload,
|
||||
ListingTaskPayloadSchema,
|
||||
ProcessedTaskPayload,
|
||||
ProcessedTaskPayloadSchema,
|
||||
} from "./schemas";
|
||||
import { parseRedisUrl } from "@crawler/utils";
|
||||
|
||||
const QueueSettingsSchema = z.object({
|
||||
redis_url: z
|
||||
.string()
|
||||
.default(process.env.BASANGO_REDIS_URL ?? "redis://localhost:6379/0"),
|
||||
prefix: z.string().default(process.env.BASANGO_QUEUE_PREFIX ?? "crawler"),
|
||||
default_timeout: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(Number(process.env.BASANGO_QUEUE_TIMEOUT ?? 600)),
|
||||
result_ttl: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.default(Number(process.env.BASANGO_QUEUE_RESULT_TTL ?? 3600)),
|
||||
failure_ttl: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.default(Number(process.env.BASANGO_QUEUE_FAILURE_TTL ?? 3600)),
|
||||
listing_queue: z.string().default("listing"),
|
||||
article_queue: z.string().default("articles"),
|
||||
processed_queue: z.string().default("processed"),
|
||||
});
|
||||
|
||||
export type QueueSettingsInput = z.input<typeof QueueSettingsSchema>;
|
||||
export type QueueSettings = z.output<typeof QueueSettingsSchema>;
|
||||
|
||||
export const createQueueSettings = (
|
||||
input?: QueueSettingsInput,
|
||||
): QueueSettings => QueueSettingsSchema.parse(input ?? {});
|
||||
|
||||
export interface QueueBackend<T = unknown> {
|
||||
add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
export type QueueFactory = (
|
||||
queueName: string,
|
||||
settings: QueueSettings,
|
||||
connection?: IORedis,
|
||||
) => QueueBackend;
|
||||
|
||||
const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => {
|
||||
const redisConnection =
|
||||
connection ??
|
||||
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
||||
const options: QueueOptions = {
|
||||
connection: redisConnection,
|
||||
prefix: settings.prefix,
|
||||
};
|
||||
|
||||
const queue = new Queue(queueName, options);
|
||||
return {
|
||||
add: async (name, data, opts) => {
|
||||
const job = await queue.add(name, data, {
|
||||
removeOnComplete: settings.result_ttl === 0 ? true : undefined,
|
||||
removeOnFail: settings.failure_ttl === 0 ? true : undefined,
|
||||
//timeout: settings.default_timeout * 1000,
|
||||
...opts,
|
||||
});
|
||||
return { id: job.id ?? randomUUID() };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export interface CreateQueueManagerOptions {
|
||||
settings?: QueueSettings | QueueSettingsInput;
|
||||
queueFactory?: QueueFactory;
|
||||
connection?: IORedis;
|
||||
}
|
||||
|
||||
export interface QueueManager {
|
||||
readonly settings: QueueSettings;
|
||||
readonly connection: IORedis;
|
||||
enqueueListing: (payload: ListingTaskPayload) => Promise<{ id: string }>;
|
||||
enqueueArticle: (payload: ArticleTaskPayload) => Promise<{ id: string }>;
|
||||
enqueueProcessed: (payload: ProcessedTaskPayload) => Promise<{ id: string }>;
|
||||
iterQueueNames: () => string[];
|
||||
queueName: (suffix: string) => string;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
export const createQueueManager = (
|
||||
options: CreateQueueManagerOptions = {},
|
||||
): QueueManager => {
|
||||
const settings = createQueueSettings(
|
||||
options.settings as QueueSettingsInput | undefined,
|
||||
);
|
||||
|
||||
const connection =
|
||||
options.connection ??
|
||||
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
||||
const factory = options.queueFactory ?? defaultQueueFactory;
|
||||
|
||||
const ensureQueue = (queueName: string) =>
|
||||
factory(queueName, settings, connection);
|
||||
|
||||
return {
|
||||
settings,
|
||||
connection,
|
||||
enqueueListing: (payload) => {
|
||||
const data = ListingTaskPayloadSchema.parse(payload);
|
||||
const queue = ensureQueue(settings.listing_queue);
|
||||
return queue.add("collect_listing", data);
|
||||
},
|
||||
enqueueArticle: (payload) => {
|
||||
const data = ArticleTaskPayloadSchema.parse(payload);
|
||||
const queue = ensureQueue(settings.article_queue);
|
||||
return queue.add("collect_article", data);
|
||||
},
|
||||
enqueueProcessed: (payload) => {
|
||||
const data = ProcessedTaskPayloadSchema.parse(payload);
|
||||
const queue = ensureQueue(settings.processed_queue);
|
||||
return queue.add("forward_for_processing", data);
|
||||
},
|
||||
iterQueueNames: () => [
|
||||
`${settings.prefix}:${settings.listing_queue}`,
|
||||
`${settings.prefix}:${settings.article_queue}`,
|
||||
`${settings.prefix}:${settings.processed_queue}`,
|
||||
],
|
||||
queueName: (suffix: string) => `${settings.prefix}:${suffix}`,
|
||||
close: async () => {
|
||||
await connection.quit();
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -0,0 +1,41 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
AnySourceConfig,
|
||||
DateRangeSchema,
|
||||
PageRangeSchema,
|
||||
} from "@crawler/schema";
|
||||
|
||||
export const ListingTaskPayloadSchema = z.object({
|
||||
source_id: z.string(),
|
||||
env: z.string().default("development"),
|
||||
page_range: z.string().optional().nullable(),
|
||||
date_range: z.string().optional().nullable(),
|
||||
category: z.string().optional().nullable(),
|
||||
});
|
||||
|
||||
export type ListingTaskPayload = z.infer<typeof ListingTaskPayloadSchema>;
|
||||
|
||||
export const ArticleTaskPayloadSchema = z.object({
|
||||
source_id: z.string(),
|
||||
env: z.string().default("development"),
|
||||
url: z.url(),
|
||||
page: z.number().int().nonnegative().optional(),
|
||||
page_range: PageRangeSchema.optional().nullable(),
|
||||
date_range: DateRangeSchema.optional().nullable(),
|
||||
category: z.string().optional().nullable(),
|
||||
});
|
||||
|
||||
export type ArticleTaskPayload = z.infer<typeof ArticleTaskPayloadSchema>;
|
||||
|
||||
export const ProcessedTaskPayloadSchema = z.object({
|
||||
source_id: z.string(),
|
||||
env: z.string().default("development"),
|
||||
article: z.any(),
|
||||
});
|
||||
|
||||
export type ProcessedTaskPayload = z.infer<typeof ProcessedTaskPayloadSchema>;
|
||||
|
||||
export interface ListingContext {
|
||||
source: AnySourceConfig;
|
||||
}
|
||||
@@ -0,0 +1,171 @@
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import {
|
||||
ArticleTaskPayload,
|
||||
ArticleTaskPayloadSchema,
|
||||
ListingTaskPayload,
|
||||
ListingTaskPayloadSchema,
|
||||
ProcessedTaskPayload,
|
||||
ProcessedTaskPayloadSchema,
|
||||
} from "./schemas";
|
||||
import {
|
||||
createQueueManager,
|
||||
QueueManager,
|
||||
QueueSettings,
|
||||
QueueSettingsInput,
|
||||
} from "./queue";
|
||||
|
||||
export interface CrawlerTaskHandlers {
|
||||
collectListing: (payload: ListingTaskPayload) => Promise<number> | number;
|
||||
collectArticle: (payload: ArticleTaskPayload) => Promise<unknown> | unknown;
|
||||
forwardForProcessing: (
|
||||
payload: ProcessedTaskPayload,
|
||||
) => Promise<unknown> | unknown;
|
||||
}
|
||||
|
||||
const notImplemented = (name: keyof CrawlerTaskHandlers) => () => {
|
||||
throw new Error(`Crawler task handler '${name}' is not implemented`);
|
||||
};
|
||||
|
||||
let handlers: CrawlerTaskHandlers = {
|
||||
collectListing: notImplemented("collectListing"),
|
||||
collectArticle: notImplemented("collectArticle"),
|
||||
forwardForProcessing: notImplemented("forwardForProcessing"),
|
||||
};
|
||||
|
||||
export const registerCrawlerTaskHandlers = (
|
||||
overrides: Partial<CrawlerTaskHandlers>,
|
||||
): void => {
|
||||
handlers = { ...handlers, ...overrides };
|
||||
};
|
||||
|
||||
export interface ScheduleAsyncCrawlOptions {
|
||||
sourceId: string;
|
||||
env?: string;
|
||||
pageRange?: string | null;
|
||||
dateRange?: string | null;
|
||||
category?: string | null;
|
||||
settings?: QueueSettings | QueueSettingsInput;
|
||||
queueManager?: QueueManager;
|
||||
}
|
||||
|
||||
export const scheduleAsyncCrawl = async ({
|
||||
sourceId,
|
||||
env = "development",
|
||||
pageRange,
|
||||
dateRange,
|
||||
category,
|
||||
settings,
|
||||
queueManager,
|
||||
}: ScheduleAsyncCrawlOptions): Promise<string> => {
|
||||
const payload = ListingTaskPayloadSchema.parse({
|
||||
source_id: sourceId,
|
||||
env,
|
||||
page_range: pageRange ?? undefined,
|
||||
date_range: dateRange ?? undefined,
|
||||
category: category ?? undefined,
|
||||
});
|
||||
|
||||
const manager = queueManager ?? createQueueManager({ settings });
|
||||
logger.debug(
|
||||
{
|
||||
sourceId,
|
||||
env: payload.env,
|
||||
pageRange: payload.page_range,
|
||||
dateRange: payload.date_range,
|
||||
category: payload.category,
|
||||
},
|
||||
"Scheduling listing collection job",
|
||||
);
|
||||
try {
|
||||
const job = await manager.enqueueListing(payload);
|
||||
logger.info(
|
||||
{ jobId: job.id, sourceId, env: payload.env },
|
||||
"Scheduled listing collection job",
|
||||
);
|
||||
return job.id;
|
||||
} finally {
|
||||
if (!queueManager) {
|
||||
await manager.close();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const collectListing = async (payload: unknown): Promise<number> => {
|
||||
const data = ListingTaskPayloadSchema.parse(payload);
|
||||
logger.debug(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
pageRange: data.page_range,
|
||||
dateRange: data.date_range,
|
||||
category: data.category,
|
||||
},
|
||||
"Collecting listing",
|
||||
);
|
||||
|
||||
const result = await handlers.collectListing(data);
|
||||
const count = typeof result === "number" ? result : 0;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
queuedArticles: count,
|
||||
},
|
||||
"Listing collection completed",
|
||||
);
|
||||
|
||||
return count;
|
||||
};
|
||||
|
||||
export const collectArticle = async (payload: unknown): Promise<unknown> => {
|
||||
const data = ArticleTaskPayloadSchema.parse(payload);
|
||||
logger.debug(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
url: data.url,
|
||||
page: data.page,
|
||||
},
|
||||
"Collecting article",
|
||||
);
|
||||
|
||||
const result = await handlers.collectArticle(data);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
url: data.url,
|
||||
},
|
||||
"Article collection completed",
|
||||
);
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const forwardForProcessing = async (
|
||||
payload: unknown,
|
||||
): Promise<unknown> => {
|
||||
const data = ProcessedTaskPayloadSchema.parse(payload);
|
||||
logger.debug(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
},
|
||||
"Forwarding article for processing",
|
||||
);
|
||||
|
||||
const result = await handlers.forwardForProcessing(data);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
},
|
||||
"Article forwarded for processing",
|
||||
);
|
||||
|
||||
return result;
|
||||
};
|
||||
@@ -0,0 +1,87 @@
|
||||
import IORedis from "ioredis";
|
||||
import { Worker, QueueEvents } from "bullmq";
|
||||
|
||||
import {
|
||||
createQueueManager,
|
||||
QueueFactory,
|
||||
QueueManager,
|
||||
QueueSettings,
|
||||
QueueSettingsInput,
|
||||
} from "./queue";
|
||||
import { collectArticle, collectListing, forwardForProcessing } from "./tasks";
|
||||
|
||||
export interface WorkerOptions {
|
||||
queueNames?: string[];
|
||||
settings?: QueueSettings | QueueSettingsInput;
|
||||
connection?: IORedis;
|
||||
queueFactory?: QueueFactory;
|
||||
concurrency?: number;
|
||||
onError?: (error: Error) => void;
|
||||
queueManager?: QueueManager;
|
||||
}
|
||||
|
||||
export interface WorkerHandle {
|
||||
readonly workers: Worker[];
|
||||
readonly events: QueueEvents[];
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
export const startWorker = (options: WorkerOptions = {}): WorkerHandle => {
|
||||
const manager =
|
||||
options.queueManager ??
|
||||
createQueueManager({
|
||||
settings: options.settings,
|
||||
connection: options.connection,
|
||||
queueFactory: options.queueFactory,
|
||||
});
|
||||
|
||||
const queueNames = options.queueNames ?? manager.iterQueueNames();
|
||||
const workers: Worker[] = [];
|
||||
const events: QueueEvents[] = [];
|
||||
|
||||
const connection = manager.connection;
|
||||
|
||||
for (const queueName of queueNames) {
|
||||
const worker = new Worker(
|
||||
queueName,
|
||||
async (job) => {
|
||||
switch (job.name) {
|
||||
case "collect_listing":
|
||||
return collectListing(job.data);
|
||||
case "collect_article":
|
||||
return collectArticle(job.data);
|
||||
case "forward_for_processing":
|
||||
return forwardForProcessing(job.data);
|
||||
default:
|
||||
throw new Error(`Unknown job name: ${job.name}`);
|
||||
}
|
||||
},
|
||||
{
|
||||
connection,
|
||||
concurrency: options.concurrency ?? 5,
|
||||
},
|
||||
);
|
||||
|
||||
if (options.onError) {
|
||||
worker.on("failed", (_, err) => options.onError?.(err as Error));
|
||||
worker.on("error", (err) => options.onError?.(err as Error));
|
||||
}
|
||||
|
||||
const queueEvents = new QueueEvents(queueName, { connection });
|
||||
|
||||
workers.push(worker);
|
||||
events.push(queueEvents);
|
||||
}
|
||||
|
||||
return {
|
||||
workers,
|
||||
events,
|
||||
close: async () => {
|
||||
await Promise.all(workers.map((worker) => worker.close()));
|
||||
await Promise.all(events.map((event) => event.close()));
|
||||
if (!options.queueManager) {
|
||||
await manager.close();
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,142 +0,0 @@
|
||||
import { randomUUID } from "node:crypto";
|
||||
|
||||
import IORedis from "ioredis";
|
||||
import { Queue, JobsOptions, QueueOptions } from "bullmq";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
ListingTaskPayload,
|
||||
ArticleTaskPayload,
|
||||
ProcessedTaskPayload,
|
||||
ListingTaskPayloadSchema,
|
||||
ArticleTaskPayloadSchema,
|
||||
ProcessedTaskPayloadSchema,
|
||||
} from "./schemas";
|
||||
import { parseRedisUrl } from "../../../utils";
|
||||
|
||||
const QueueSettingsSchema = z.object({
|
||||
redis_url: z
|
||||
.string()
|
||||
.default(process.env.BASANGO_REDIS_URL ?? "redis://localhost:6379/0"),
|
||||
prefix: z.string().default(process.env.BASANGO_QUEUE_PREFIX ?? "crawler"),
|
||||
default_timeout: z
|
||||
.number()
|
||||
.int()
|
||||
.positive()
|
||||
.default(Number(process.env.BASANGO_QUEUE_TIMEOUT ?? 600)),
|
||||
result_ttl: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.default(Number(process.env.BASANGO_QUEUE_RESULT_TTL ?? 3600)),
|
||||
failure_ttl: z
|
||||
.number()
|
||||
.int()
|
||||
.nonnegative()
|
||||
.default(Number(process.env.BASANGO_QUEUE_FAILURE_TTL ?? 3600)),
|
||||
listing_queue: z.string().default("listing"),
|
||||
article_queue: z.string().default("articles"),
|
||||
processed_queue: z.string().default("processed"),
|
||||
});
|
||||
|
||||
export type QueueSettingsInput = z.input<typeof QueueSettingsSchema>;
|
||||
export type QueueSettings = z.output<typeof QueueSettingsSchema>;
|
||||
|
||||
export const createQueueSettings = (
|
||||
input?: QueueSettingsInput,
|
||||
): QueueSettings => QueueSettingsSchema.parse(input ?? {});
|
||||
|
||||
export interface QueueBackend<T = unknown> {
|
||||
add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>;
|
||||
}
|
||||
|
||||
export type QueueFactory = (
|
||||
queueName: string,
|
||||
settings: QueueSettings,
|
||||
connection?: IORedis,
|
||||
) => QueueBackend;
|
||||
|
||||
const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => {
|
||||
const redisConnection =
|
||||
connection ??
|
||||
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
||||
const options: QueueOptions = {
|
||||
connection: redisConnection,
|
||||
prefix: settings.prefix,
|
||||
};
|
||||
|
||||
const queue = new Queue(queueName, options);
|
||||
return {
|
||||
add: async (name, data, opts) => {
|
||||
const job = await queue.add(name, data, {
|
||||
removeOnComplete: settings.result_ttl === 0 ? true : undefined,
|
||||
removeOnFail: settings.failure_ttl === 0 ? true : undefined,
|
||||
timeout: settings.default_timeout * 1000,
|
||||
...opts,
|
||||
});
|
||||
return { id: job.id ?? randomUUID() };
|
||||
},
|
||||
};
|
||||
};
|
||||
|
||||
export interface CreateQueueManagerOptions {
|
||||
settings?: QueueSettings | QueueSettingsInput;
|
||||
queueFactory?: QueueFactory;
|
||||
connection?: IORedis;
|
||||
}
|
||||
|
||||
export interface QueueManager {
|
||||
readonly settings: QueueSettings;
|
||||
readonly connection: IORedis;
|
||||
enqueueListing: (payload: ListingTaskPayload) => Promise<{ id: string }>;
|
||||
enqueueArticle: (payload: ArticleTaskPayload) => Promise<{ id: string }>;
|
||||
enqueueProcessed: (payload: ProcessedTaskPayload) => Promise<{ id: string }>;
|
||||
iterQueueNames: () => string[];
|
||||
queueName: (suffix: string) => string;
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
export const createQueueManager = (
|
||||
options: CreateQueueManagerOptions = {},
|
||||
): QueueManager => {
|
||||
const settings = createQueueSettings(
|
||||
options.settings as QueueSettingsInput | undefined,
|
||||
);
|
||||
|
||||
const connection =
|
||||
options.connection ??
|
||||
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
||||
const factory = options.queueFactory ?? defaultQueueFactory;
|
||||
|
||||
const ensureQueue = (queueName: string) =>
|
||||
factory(queueName, settings, connection);
|
||||
|
||||
return {
|
||||
settings,
|
||||
connection,
|
||||
enqueueListing: (payload) => {
|
||||
const data = ListingTaskPayloadSchema.parse(payload);
|
||||
const queue = ensureQueue(settings.listing_queue);
|
||||
return queue.add("collect_listing", data);
|
||||
},
|
||||
enqueueArticle: (payload) => {
|
||||
const data = ArticleTaskPayloadSchema.parse(payload);
|
||||
const queue = ensureQueue(settings.article_queue);
|
||||
return queue.add("collect_article", data);
|
||||
},
|
||||
enqueueProcessed: (payload) => {
|
||||
const data = ProcessedTaskPayloadSchema.parse(payload);
|
||||
const queue = ensureQueue(settings.processed_queue);
|
||||
return queue.add("forward_for_processing", data);
|
||||
},
|
||||
iterQueueNames: () => [
|
||||
`${settings.prefix}:${settings.listing_queue}`,
|
||||
`${settings.prefix}:${settings.article_queue}`,
|
||||
`${settings.prefix}:${settings.processed_queue}`,
|
||||
],
|
||||
queueName: (suffix: string) => `${settings.prefix}:${suffix}`,
|
||||
close: async () => {
|
||||
await connection.quit();
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,41 +0,0 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
AnySourceConfig,
|
||||
DateRangeSchema,
|
||||
PageRangeSchema,
|
||||
} from "../../../schema";
|
||||
|
||||
export const ListingTaskPayloadSchema = z.object({
|
||||
source_id: z.string(),
|
||||
env: z.string().default("development"),
|
||||
page_range: z.string().optional().nullable(),
|
||||
date_range: z.string().optional().nullable(),
|
||||
category: z.string().optional().nullable(),
|
||||
});
|
||||
|
||||
export type ListingTaskPayload = z.infer<typeof ListingTaskPayloadSchema>;
|
||||
|
||||
export const ArticleTaskPayloadSchema = z.object({
|
||||
source_id: z.string(),
|
||||
env: z.string().default("development"),
|
||||
url: z.string().url(),
|
||||
page: z.number().int().nonnegative().optional(),
|
||||
page_range: PageRangeSchema.optional().nullable(),
|
||||
date_range: DateRangeSchema.optional().nullable(),
|
||||
category: z.string().optional().nullable(),
|
||||
});
|
||||
|
||||
export type ArticleTaskPayload = z.infer<typeof ArticleTaskPayloadSchema>;
|
||||
|
||||
export const ProcessedTaskPayloadSchema = z.object({
|
||||
source_id: z.string(),
|
||||
env: z.string().default("development"),
|
||||
article: z.any(),
|
||||
});
|
||||
|
||||
export type ProcessedTaskPayload = z.infer<typeof ProcessedTaskPayloadSchema>;
|
||||
|
||||
export interface ListingContext {
|
||||
source: AnySourceConfig;
|
||||
}
|
||||
@@ -1,171 +0,0 @@
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import {
|
||||
ListingTaskPayloadSchema,
|
||||
ArticleTaskPayloadSchema,
|
||||
ProcessedTaskPayloadSchema,
|
||||
ListingTaskPayload,
|
||||
ArticleTaskPayload,
|
||||
ProcessedTaskPayload,
|
||||
} from "./schemas";
|
||||
import {
|
||||
createQueueManager,
|
||||
QueueManager,
|
||||
QueueSettings,
|
||||
QueueSettingsInput,
|
||||
} from "./queue";
|
||||
|
||||
export interface CrawlerTaskHandlers {
|
||||
collectListing: (payload: ListingTaskPayload) => Promise<number> | number;
|
||||
collectArticle: (payload: ArticleTaskPayload) => Promise<unknown> | unknown;
|
||||
forwardForProcessing: (
|
||||
payload: ProcessedTaskPayload,
|
||||
) => Promise<unknown> | unknown;
|
||||
}
|
||||
|
||||
const notImplemented = (name: keyof CrawlerTaskHandlers) => () => {
|
||||
throw new Error(`Crawler task handler '${name}' is not implemented`);
|
||||
};
|
||||
|
||||
let handlers: CrawlerTaskHandlers = {
|
||||
collectListing: notImplemented("collectListing"),
|
||||
collectArticle: notImplemented("collectArticle"),
|
||||
forwardForProcessing: notImplemented("forwardForProcessing"),
|
||||
};
|
||||
|
||||
export const registerCrawlerTaskHandlers = (
|
||||
overrides: Partial<CrawlerTaskHandlers>,
|
||||
): void => {
|
||||
handlers = { ...handlers, ...overrides };
|
||||
};
|
||||
|
||||
export interface ScheduleAsyncCrawlOptions {
|
||||
sourceId: string;
|
||||
env?: string;
|
||||
pageRange?: string | null;
|
||||
dateRange?: string | null;
|
||||
category?: string | null;
|
||||
settings?: QueueSettings | QueueSettingsInput;
|
||||
queueManager?: QueueManager;
|
||||
}
|
||||
|
||||
export const scheduleAsyncCrawl = async ({
|
||||
sourceId,
|
||||
env = "development",
|
||||
pageRange,
|
||||
dateRange,
|
||||
category,
|
||||
settings,
|
||||
queueManager,
|
||||
}: ScheduleAsyncCrawlOptions): Promise<string> => {
|
||||
const payload = ListingTaskPayloadSchema.parse({
|
||||
source_id: sourceId,
|
||||
env,
|
||||
page_range: pageRange ?? undefined,
|
||||
date_range: dateRange ?? undefined,
|
||||
category: category ?? undefined,
|
||||
});
|
||||
|
||||
const manager = queueManager ?? createQueueManager({ settings });
|
||||
logger.debug(
|
||||
{
|
||||
sourceId,
|
||||
env: payload.env,
|
||||
pageRange: payload.page_range,
|
||||
dateRange: payload.date_range,
|
||||
category: payload.category,
|
||||
},
|
||||
"Scheduling listing collection job",
|
||||
);
|
||||
try {
|
||||
const job = await manager.enqueueListing(payload);
|
||||
logger.info(
|
||||
{ jobId: job.id, sourceId, env: payload.env },
|
||||
"Scheduled listing collection job",
|
||||
);
|
||||
return job.id;
|
||||
} finally {
|
||||
if (!queueManager) {
|
||||
await manager.close();
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const collectListing = async (payload: unknown): Promise<number> => {
|
||||
const data = ListingTaskPayloadSchema.parse(payload);
|
||||
logger.debug(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
pageRange: data.page_range,
|
||||
dateRange: data.date_range,
|
||||
category: data.category,
|
||||
},
|
||||
"Collecting listing",
|
||||
);
|
||||
|
||||
const result = await handlers.collectListing(data);
|
||||
const count = typeof result === "number" ? result : 0;
|
||||
|
||||
logger.info(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
queuedArticles: count,
|
||||
},
|
||||
"Listing collection completed",
|
||||
);
|
||||
|
||||
return count;
|
||||
};
|
||||
|
||||
export const collectArticle = async (payload: unknown): Promise<unknown> => {
|
||||
const data = ArticleTaskPayloadSchema.parse(payload);
|
||||
logger.debug(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
url: data.url,
|
||||
page: data.page,
|
||||
},
|
||||
"Collecting article",
|
||||
);
|
||||
|
||||
const result = await handlers.collectArticle(data);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
url: data.url,
|
||||
},
|
||||
"Article collection completed",
|
||||
);
|
||||
|
||||
return result;
|
||||
};
|
||||
|
||||
export const forwardForProcessing = async (
|
||||
payload: unknown,
|
||||
): Promise<unknown> => {
|
||||
const data = ProcessedTaskPayloadSchema.parse(payload);
|
||||
logger.debug(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
},
|
||||
"Forwarding article for processing",
|
||||
);
|
||||
|
||||
const result = await handlers.forwardForProcessing(data);
|
||||
|
||||
logger.info(
|
||||
{
|
||||
sourceId: data.source_id,
|
||||
env: data.env,
|
||||
},
|
||||
"Article forwarded for processing",
|
||||
);
|
||||
|
||||
return result;
|
||||
};
|
||||
@@ -1,87 +0,0 @@
|
||||
import IORedis from "ioredis";
|
||||
import { Worker, QueueEvents } from "bullmq";
|
||||
|
||||
import {
|
||||
createQueueManager,
|
||||
QueueFactory,
|
||||
QueueManager,
|
||||
QueueSettings,
|
||||
QueueSettingsInput,
|
||||
} from "./queue";
|
||||
import { collectArticle, collectListing, forwardForProcessing } from "./tasks";
|
||||
|
||||
export interface WorkerOptions {
|
||||
queueNames?: string[];
|
||||
settings?: QueueSettings | QueueSettingsInput;
|
||||
connection?: IORedis;
|
||||
queueFactory?: QueueFactory;
|
||||
concurrency?: number;
|
||||
onError?: (error: Error) => void;
|
||||
queueManager?: QueueManager;
|
||||
}
|
||||
|
||||
export interface WorkerHandle {
|
||||
readonly workers: Worker[];
|
||||
readonly events: QueueEvents[];
|
||||
close: () => Promise<void>;
|
||||
}
|
||||
|
||||
export const startWorker = (options: WorkerOptions = {}): WorkerHandle => {
|
||||
const manager =
|
||||
options.queueManager ??
|
||||
createQueueManager({
|
||||
settings: options.settings,
|
||||
connection: options.connection,
|
||||
queueFactory: options.queueFactory,
|
||||
});
|
||||
|
||||
const queueNames = options.queueNames ?? manager.iterQueueNames();
|
||||
const workers: Worker[] = [];
|
||||
const events: QueueEvents[] = [];
|
||||
|
||||
const connection = manager.connection;
|
||||
|
||||
for (const queueName of queueNames) {
|
||||
const worker = new Worker(
|
||||
queueName,
|
||||
async (job) => {
|
||||
switch (job.name) {
|
||||
case "collect_listing":
|
||||
return collectListing(job.data);
|
||||
case "collect_article":
|
||||
return collectArticle(job.data);
|
||||
case "forward_for_processing":
|
||||
return forwardForProcessing(job.data);
|
||||
default:
|
||||
throw new Error(`Unknown job name: ${job.name}`);
|
||||
}
|
||||
},
|
||||
{
|
||||
connection,
|
||||
concurrency: options.concurrency ?? 5,
|
||||
},
|
||||
);
|
||||
|
||||
if (options.onError) {
|
||||
worker.on("failed", (_, err) => options.onError?.(err as Error));
|
||||
worker.on("error", (err) => options.onError?.(err as Error));
|
||||
}
|
||||
|
||||
const queueEvents = new QueueEvents(queueName, { connection });
|
||||
|
||||
workers.push(worker);
|
||||
events.push(queueEvents);
|
||||
}
|
||||
|
||||
return {
|
||||
workers,
|
||||
events,
|
||||
close: async () => {
|
||||
await Promise.all(workers.map((worker) => worker.close()));
|
||||
await Promise.all(events.map((event) => event.close()));
|
||||
if (!options.queueManager) {
|
||||
await manager.close();
|
||||
}
|
||||
},
|
||||
};
|
||||
};
|
||||
@@ -1,3 +0,0 @@
|
||||
export * from "./async/queue";
|
||||
export * from "./async/tasks";
|
||||
export * from "./async/worker";
|
||||
@@ -6,33 +6,33 @@ import { get_encoding } from "tiktoken";
|
||||
import type { ProjectPaths } from "@crawler/schema";
|
||||
|
||||
export const ensureDirectories = (paths: ProjectPaths): void => {
|
||||
for (const dir of [paths.data, paths.logs, paths.configs]) {
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
for (const dir of [paths.data, paths.logs, paths.configs]) {
|
||||
if (!fs.existsSync(dir)) {
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
}
|
||||
}
|
||||
};
|
||||
|
||||
export const parseRedisUrl = (url: string): RedisOptions => {
|
||||
if (!url.startsWith("redis://")) {
|
||||
return {};
|
||||
}
|
||||
const parsed = new URL(url);
|
||||
return {
|
||||
host: parsed.hostname,
|
||||
port: Number(parsed.port || 6379),
|
||||
password: parsed.password || undefined,
|
||||
db: Number(parsed.pathname?.replace("/", "") || 0),
|
||||
};
|
||||
if (!url.startsWith("redis://")) {
|
||||
return {};
|
||||
}
|
||||
const parsed = new URL(url);
|
||||
return {
|
||||
host: parsed.hostname,
|
||||
port: Number(parsed.port || 6379),
|
||||
password: parsed.password || undefined,
|
||||
db: Number(parsed.pathname?.replace("/", "") || 0),
|
||||
};
|
||||
};
|
||||
|
||||
export const countTokens = (text: string, encoding = "cl100k_base"): number => {
|
||||
try {
|
||||
const encoder = get_encoding(encoding);
|
||||
const tokens = encoder.encode(text);
|
||||
encoder.free();
|
||||
return tokens.length;
|
||||
} catch {
|
||||
return text.length;
|
||||
}
|
||||
try {
|
||||
const encoder = get_encoding(encoding);
|
||||
const tokens = encoder.encode(text);
|
||||
encoder.free();
|
||||
return tokens.length;
|
||||
} catch {
|
||||
return text.length;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist",
|
||||
"paths": {
|
||||
"@crawler": ["./src/index.ts"],
|
||||
"@crawler/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": []
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"compilerOptions": {
|
||||
"rootDir": "src",
|
||||
"outDir": "dist",
|
||||
"paths": {
|
||||
"@crawler": ["./src/index.ts"],
|
||||
"@crawler/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"include": ["src"],
|
||||
"references": []
|
||||
}
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import { defineConfig } from "vitest/config";
|
||||
|
||||
export default defineConfig({
|
||||
test: {
|
||||
environment: "node",
|
||||
globals: true,
|
||||
include: ["src/**/*.test.ts"],
|
||||
},
|
||||
test: {
|
||||
environment: "node",
|
||||
globals: true,
|
||||
include: ["src/**/*.test.ts"],
|
||||
},
|
||||
});
|
||||
|
||||
+32
-32
@@ -1,34 +1,34 @@
|
||||
{
|
||||
"$schema": "https://biomejs.dev/schemas/2.3.1/schema.json",
|
||||
"vcs": {
|
||||
"enabled": true,
|
||||
"clientKind": "git",
|
||||
"useIgnoreFile": true
|
||||
},
|
||||
"files": {
|
||||
"ignoreUnknown": false
|
||||
},
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
"indentStyle": "space"
|
||||
},
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true
|
||||
}
|
||||
},
|
||||
"javascript": {
|
||||
"formatter": {
|
||||
"quoteStyle": "double"
|
||||
}
|
||||
},
|
||||
"assist": {
|
||||
"enabled": true,
|
||||
"actions": {
|
||||
"source": {
|
||||
"organizeImports": "on"
|
||||
}
|
||||
}
|
||||
}
|
||||
"$schema": "https://biomejs.dev/schemas/2.3.1/schema.json",
|
||||
"vcs": {
|
||||
"enabled": true,
|
||||
"clientKind": "git",
|
||||
"useIgnoreFile": true
|
||||
},
|
||||
"files": {
|
||||
"ignoreUnknown": false
|
||||
},
|
||||
"formatter": {
|
||||
"enabled": true,
|
||||
"indentStyle": "space"
|
||||
},
|
||||
"linter": {
|
||||
"enabled": true,
|
||||
"rules": {
|
||||
"recommended": true
|
||||
}
|
||||
},
|
||||
"javascript": {
|
||||
"formatter": {
|
||||
"quoteStyle": "double"
|
||||
}
|
||||
},
|
||||
"assist": {
|
||||
"enabled": true,
|
||||
"actions": {
|
||||
"source": {
|
||||
"organizeImports": "on"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
+30
-7
@@ -7,13 +7,14 @@
|
||||
"@biomejs/biome": "^2.3.1",
|
||||
"@manypkg/cli": "^0.25.1",
|
||||
"turbo": "^2.5.8",
|
||||
"typescript": "5.9.2",
|
||||
"typescript": "catalog:",
|
||||
},
|
||||
},
|
||||
"apps/crawler": {
|
||||
"name": "@basango/crawler",
|
||||
"version": "0.1.0",
|
||||
"dependencies": {
|
||||
"@basango/logger": "workspace:*",
|
||||
"bullmq": "^4.17.0",
|
||||
"date-fns": "^3.6.0",
|
||||
"ioredis": "^5.3.2",
|
||||
@@ -25,24 +26,28 @@
|
||||
"name": "@basango/db",
|
||||
"version": "1.0.0",
|
||||
"dependencies": {
|
||||
"@basango/logger": "workspace:*",
|
||||
"@date-fns/utc": "^2.1.1",
|
||||
"drizzle-orm": "^0.44.7",
|
||||
"pg": "^8.16.3",
|
||||
"snakecase-keys": "^9.0.2",
|
||||
},
|
||||
"devDependencies": {
|
||||
"@types/bun": "^1.3.1",
|
||||
"@types/pg": "^8.15.6",
|
||||
"drizzle-kit": "^0.31.6",
|
||||
"typescript": "catalog:",
|
||||
},
|
||||
},
|
||||
"packages/logger": {
|
||||
"name": "@midday/logger",
|
||||
"version": "0.0.0",
|
||||
"name": "@basango/logger",
|
||||
"version": "0.0.1",
|
||||
"dependencies": {
|
||||
"pino": "^10.1.0",
|
||||
"pino-pretty": "^13.1.2",
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.9.2",
|
||||
"typescript": "catalog:",
|
||||
},
|
||||
},
|
||||
"packages/tsconfig": {
|
||||
@@ -50,11 +55,17 @@
|
||||
"version": "0.0.0",
|
||||
},
|
||||
},
|
||||
"catalog": {
|
||||
"@types/bun": "^1.3.1",
|
||||
"typescript": "^5.9.3",
|
||||
},
|
||||
"packages": {
|
||||
"@basango/crawler": ["@basango/crawler@workspace:apps/crawler"],
|
||||
|
||||
"@basango/db": ["@basango/db@workspace:packages/db"],
|
||||
|
||||
"@basango/logger": ["@basango/logger@workspace:packages/logger"],
|
||||
|
||||
"@basango/tsconfig": ["@basango/tsconfig@workspace:packages/tsconfig"],
|
||||
|
||||
"@biomejs/biome": ["@biomejs/biome@2.3.1", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.1", "@biomejs/cli-darwin-x64": "2.3.1", "@biomejs/cli-linux-arm64": "2.3.1", "@biomejs/cli-linux-arm64-musl": "2.3.1", "@biomejs/cli-linux-x64": "2.3.1", "@biomejs/cli-linux-x64-musl": "2.3.1", "@biomejs/cli-win32-arm64": "2.3.1", "@biomejs/cli-win32-x64": "2.3.1" }, "bin": { "biome": "bin/biome" } }, "sha512-A29evf1R72V5bo4o2EPxYMm5mtyGvzp2g+biZvRFx29nWebGyyeOSsDWGx3tuNNMFRepGwxmA9ZQ15mzfabK2w=="],
|
||||
@@ -145,8 +156,6 @@
|
||||
|
||||
"@manypkg/tools": ["@manypkg/tools@2.1.0", "", { "dependencies": { "jju": "^1.4.0", "js-yaml": "^4.1.0", "tinyglobby": "^0.2.13" } }, "sha512-0FOIepYR4ugPYaHwK7hDeHDkfPOBVvayt9QpvRbi2LT/h2b0GaE/gM9Gag7fsnyYyNaTZ2IGyOuVg07IYepvYQ=="],
|
||||
|
||||
"@midday/logger": ["@midday/logger@workspace:packages/logger"],
|
||||
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="],
|
||||
|
||||
"@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="],
|
||||
@@ -167,6 +176,14 @@
|
||||
|
||||
"@pnpm/npm-conf": ["@pnpm/npm-conf@2.3.1", "", { "dependencies": { "@pnpm/config.env-replace": "^1.1.0", "@pnpm/network.ca-file": "^1.0.1", "config-chain": "^1.1.11" } }, "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw=="],
|
||||
|
||||
"@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="],
|
||||
|
||||
"@types/node": ["@types/node@24.9.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA=="],
|
||||
|
||||
"@types/pg": ["@types/pg@8.15.6", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ=="],
|
||||
|
||||
"@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="],
|
||||
|
||||
"argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
|
||||
|
||||
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
|
||||
@@ -179,6 +196,8 @@
|
||||
|
||||
"bullmq": ["bullmq@4.18.3", "", { "dependencies": { "cron-parser": "^4.6.0", "glob": "^8.0.3", "ioredis": "^5.3.2", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-H8t9vhfHEbJDaXp7aalSTe+Do+tR1nvr+lsT+jQxLhy+FFfFj/0p4aYJzADTNLdEqltuxneLVxCGVg92GkQx4w=="],
|
||||
|
||||
"bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="],
|
||||
|
||||
"change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="],
|
||||
|
||||
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
|
||||
@@ -189,6 +208,8 @@
|
||||
|
||||
"cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="],
|
||||
|
||||
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
|
||||
|
||||
"date-fns": ["date-fns@3.6.0", "", {}, "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww=="],
|
||||
|
||||
"dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="],
|
||||
@@ -387,7 +408,9 @@
|
||||
|
||||
"type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="],
|
||||
|
||||
"typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="],
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||
|
||||
"uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
|
||||
|
||||
|
||||
@@ -0,0 +1,105 @@
|
||||
Specifying packages in a monorepo
|
||||
Declaring directories for packages
|
||||
First, your package manager needs to describe the locations of your packages. We recommend starting with splitting your packages into apps/ for applications and services and packages/ for everything else, like libraries and tooling.
|
||||
|
||||
pnpm
|
||||
yarn
|
||||
npm
|
||||
bun
|
||||
./package.json
|
||||
|
||||
{
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
"packages/*"
|
||||
]
|
||||
}
|
||||
→
|
||||
bun workspace documentation
|
||||
Using this configuration, every directory with a package.json in the apps or packages directories will be considered a package.
|
||||
|
||||
Turborepo does not support nested packages like apps/** or packages/** due to ambiguous behavior among package managers in the JavaScript ecosystem. Using a structure that would put a package at apps/a and another at apps/a/b will result in an error.If you'd like to group packages by directory, you can do this using globs like packages/* and packages/group/* and not creating a packages/group/package.json file.
|
||||
package.json in each package
|
||||
In the directory of the package, there must be a package.json to make the package discoverable to your package manager and turbo. The requirements for the package.json of a package are below.
|
||||
|
||||
Root package.json
|
||||
The root package.json is the base for your workspace. Below is a common example of what you would find in a root package.json:
|
||||
|
||||
pnpm
|
||||
yarn
|
||||
npm
|
||||
bun
|
||||
./package.json
|
||||
|
||||
{
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "turbo run build",
|
||||
"dev": "turbo run dev",
|
||||
"lint": "turbo run lint"
|
||||
},
|
||||
"devDependencies": {
|
||||
"turbo": "latest"
|
||||
},
|
||||
"packageManager": "bun@1.2.0",
|
||||
"workspaces": ["apps/*", "packages/*"]
|
||||
}
|
||||
Root turbo.json
|
||||
turbo.json is used to configure the behavior of turbo. To learn more about how to configure your tasks, visit the Configuring tasks page.
|
||||
|
||||
Package manager lockfile
|
||||
A lockfile is key to reproducible behavior for both your package manager and turbo. Additionally, Turborepo uses the lockfile to understand the dependencies between your Internal Packages within your Workspace.
|
||||
|
||||
If you do not have a lockfile present when you run turbo, you may see unpredictable behavior.
|
||||
Anatomy of a package
|
||||
It's often best to start thinking about designing a package as its own unit within the Workspace. At a high-level, each package is almost like its own small "project", with its own package.json, tooling configuration, and source code. There are limits to this idea—but its a good mental model to start from.
|
||||
|
||||
Additionally, a package has specific entrypoints that other packages in your Workspace can use to access the package, specified by exports.
|
||||
|
||||
package.json for a package
|
||||
name
|
||||
The name field is used to identify the package. It should be unique within your workspace.
|
||||
|
||||
It's best practice to use a namespace prefix for your Internal Packages to avoid conflicts with other packages on the npm registry. For example, if your organization is named acme, you might name your packages @acme/package-name.We use @repo in our docs and examples because it is an unused, unclaimable namespace on the npm registry. You can choose to keep it or use your own prefix.
|
||||
scripts
|
||||
The scripts field is used to define scripts that can be run in the package's context. Turborepo will use the name of these scripts to identify what scripts to run (if any) in a package. We talk more about these scripts on the Running Tasks page.
|
||||
|
||||
exports
|
||||
The exports field is used to specify the entrypoints for other packages that want to use the package. When you want to use code from one package in another package, you'll import from that entrypoint.
|
||||
|
||||
For example, if you had a @repo/math package, you might have the following exports field:
|
||||
|
||||
./packages/math/package.json
|
||||
|
||||
{
|
||||
"exports": {
|
||||
".": "./src/constants.ts",
|
||||
"./add": "./src/add.ts",
|
||||
"./subtract": "./src/subtract.ts"
|
||||
}
|
||||
}
|
||||
Note that this example uses the Just-in-Time Package pattern for simplicity. It exports TypeScript directly, but you might choose to use the Compiled Package pattern instead.
|
||||
|
||||
The exports field in this example requires modern versions of Node.js and TypeScript.
|
||||
This would allow you to import add and subtract functions from the @repo/math package like so:
|
||||
|
||||
./apps/my-app/src/index.ts
|
||||
|
||||
import { GRAVITATIONAL_CONSTANT, SPEED_OF_LIGHT } from '@repo/math';
|
||||
import { add } from '@repo/math/add';
|
||||
import { subtract } from '@repo/math/subtract';
|
||||
Using exports this way provides three major benefits:
|
||||
|
||||
Avoiding barrel files: Barrel files are files that re-export other files in the same package, creating one entrypoint for the entire package. While they might appear convenient, they're difficult for compilers and bundlers to handle and can quickly lead to performance problems.
|
||||
More powerful features: exports also has other powerful features compared to the main field like Conditional Exports. In general, we recommend using exports over main whenever possible as it is the more modern option.
|
||||
IDE autocompletion: By specifying the entrypoints for your package using exports, you can ensure that your code editor can provide auto-completion for the package's exports.
|
||||
imports (optional)
|
||||
The imports field gives you a way to create subpaths to other modules within your package. You can think of these like "shortcuts" to write simpler import paths that are more resilient to refactors that move files. To learn how, visit the TypeScript page.
|
||||
|
||||
You may be more familiar with TypeScript's compilerOptions#paths option, which accomplishes a similar goal. As of TypeScript 5.4, TypeScript can infer subpaths from imports, making it a better option since you'll be working with Node.js conventions. For more information, visit our TypeScript guide.
|
||||
Source code
|
||||
Of course, you'll want some source code in your package. Packages commonly use an src directory to store their source code and compile to a dist directory (that should also be located within the package), although this is not a requirement.
|
||||
|
||||
Common pitfalls
|
||||
If you're using TypeScript, you likely don't need a tsconfig.json in the root of your workspace. Packages should independently specify their own configurations, usually building off of a shared tsconfig.json from a separate package in the workspace. For more information, visit the TypeScript guide.
|
||||
You want to avoid accessing files across package boundaries as much as possible. If you ever find yourself writing ../ to get from one package to another, you likely have an opportunity to re-think your approach by installing the package where it's needed and importing it into your code.
|
||||
+31
-26
@@ -1,28 +1,33 @@
|
||||
{
|
||||
"name": "basango",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "turbo run build",
|
||||
"clean": "git clean -xdf node_modules",
|
||||
"clean:workspaces": "turbo run clean",
|
||||
"dev": "turbo run dev --parallel",
|
||||
"test": "turbo run test --parallel",
|
||||
"lint": "turbo run lint && manypkg check",
|
||||
"format": "biome format --write .",
|
||||
"typecheck": "turbo run typecheck"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.3.1",
|
||||
"@manypkg/cli": "^0.25.1",
|
||||
"turbo": "^2.5.8",
|
||||
"typescript": "5.9.2"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"packageManager": "bun@1.2.8",
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
"packages/*"
|
||||
]
|
||||
"name": "basango",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "turbo run build",
|
||||
"clean": "git clean -xdf node_modules",
|
||||
"clean:workspaces": "turbo run clean",
|
||||
"dev": "turbo run dev --parallel",
|
||||
"test": "turbo run test --parallel",
|
||||
"lint": "turbo run lint && manypkg check",
|
||||
"format": "biome format --write .",
|
||||
"typecheck": "turbo run typecheck"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@biomejs/biome": "^2.3.1",
|
||||
"@manypkg/cli": "^0.25.1",
|
||||
"turbo": "^2.5.8",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=22"
|
||||
},
|
||||
"packageManager": "bun@1.3.1",
|
||||
"workspaces": [
|
||||
"apps/*",
|
||||
"packages/*"
|
||||
],
|
||||
"catalog": {
|
||||
"typescript": "^5.9.3",
|
||||
"@types/bun": "^1.3.1",
|
||||
"zod": "^4.0.0"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -5,6 +5,6 @@ export default {
|
||||
out: "./migrations",
|
||||
dialect: "postgresql",
|
||||
dbCredentials: {
|
||||
url: process.env.DATABASE_SESSION_POOLER!,
|
||||
url: process.env.DATABASE_URL!,
|
||||
},
|
||||
} satisfies Config;
|
||||
} satisfies Config;
|
||||
|
||||
@@ -1,17 +1,23 @@
|
||||
{
|
||||
"name": "@basango/db",
|
||||
"version": "1.0.0",
|
||||
"main": "index.ts",
|
||||
"author": "",
|
||||
"license": "ISC",
|
||||
"description": "",
|
||||
"private": true,
|
||||
"exports": {
|
||||
"./client": "./src/client.ts",
|
||||
"./schema": "./src/schema.ts",
|
||||
"./utils": "./src/utils/index.ts",
|
||||
"./queries": "./src/queries/index.ts"
|
||||
},
|
||||
"dependencies": {
|
||||
"@basango/logger": "workspace:*",
|
||||
"@date-fns/utc": "^2.1.1",
|
||||
"drizzle-orm": "^0.44.7",
|
||||
"pg": "^8.16.3",
|
||||
"snakecase-keys": "^9.0.2"
|
||||
},
|
||||
"devDependencies": {
|
||||
"drizzle-kit": "^0.31.6"
|
||||
"@types/bun": "^1.3.1",
|
||||
"@types/pg": "^8.15.6",
|
||||
"drizzle-kit": "^0.31.6",
|
||||
"typescript": "catalog:"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { drizzle } from "drizzle-orm/node-postgres";
|
||||
import { Pool } from "pg";
|
||||
import * as schema from "@basango/db/schema";
|
||||
import * as schema from "@db/schema";
|
||||
|
||||
const isDevelopment = process.env.NODE_ENV === "development";
|
||||
|
||||
|
||||
@@ -0,0 +1,2 @@
|
||||
export const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
||||
export const PUBLICATION_GRAPH_DAYS = 180;
|
||||
@@ -1,84 +0,0 @@
|
||||
import type { SQL } from "drizzle-orm";
|
||||
import { and, desc, eq, sql } from "drizzle-orm";
|
||||
|
||||
import type { Database } from "@db/client";
|
||||
import { articles, sources } from "@db/schema";
|
||||
|
||||
export interface ArticleExportRow {
|
||||
articleId: string;
|
||||
articleTitle: string;
|
||||
articleLink: string;
|
||||
articleCategories: string | null;
|
||||
articleBody: string;
|
||||
articleSource: string;
|
||||
articleHash: string;
|
||||
articlePublishedAt: string;
|
||||
articleCrawledAt: string;
|
||||
}
|
||||
|
||||
export interface ArticleExportParams {
|
||||
source?: string | null;
|
||||
dateRange?: { start: number; end: number } | null;
|
||||
batchSize?: number;
|
||||
}
|
||||
|
||||
export async function* getArticlesForExport(
|
||||
db: Database,
|
||||
params: ArticleExportParams = {},
|
||||
): AsyncGenerator<ArticleExportRow> {
|
||||
const batchSize = params.batchSize && params.batchSize > 0
|
||||
? params.batchSize
|
||||
: 1000;
|
||||
|
||||
const filters: SQL[] = [];
|
||||
|
||||
if (params.source) {
|
||||
filters.push(eq(sources.name, params.source));
|
||||
}
|
||||
|
||||
if (params.dateRange) {
|
||||
filters.push(
|
||||
sql`${articles.publishedAt} BETWEEN to_timestamp(${params.dateRange.start}) AND to_timestamp(${params.dateRange.end})`,
|
||||
);
|
||||
}
|
||||
|
||||
let query = db
|
||||
.select({
|
||||
articleId: articles.id,
|
||||
articleTitle: articles.title,
|
||||
articleLink: articles.link,
|
||||
articleCategories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
||||
articleBody: articles.body,
|
||||
articleSource: sources.name,
|
||||
articleHash: articles.hash,
|
||||
articlePublishedAt: articles.publishedAt,
|
||||
articleCrawledAt: articles.crawledAt,
|
||||
})
|
||||
.from(articles)
|
||||
.innerJoin(sources, eq(articles.sourceId, sources.id));
|
||||
|
||||
if (filters.length === 1) {
|
||||
query = query.where(filters[0]);
|
||||
} else if (filters.length > 1) {
|
||||
query = query.where(and(...filters));
|
||||
}
|
||||
|
||||
query = query.orderBy(desc(articles.publishedAt), desc(articles.id));
|
||||
|
||||
let offset = 0;
|
||||
while (true) {
|
||||
const rows = await query.limit(batchSize).offset(offset);
|
||||
if (rows.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
yield {
|
||||
...row,
|
||||
articleCategories: row.articleCategories ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
offset += batchSize;
|
||||
}
|
||||
}
|
||||
@@ -1,87 +0,0 @@
|
||||
import { and, eq, sql } from "drizzle-orm";
|
||||
|
||||
import type { Database } from "@db/client";
|
||||
import { articles, sources } from "@db/schema";
|
||||
|
||||
export interface SourceStatisticsRow {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
sourceCrawledAt: string | null;
|
||||
articlesCount: number;
|
||||
articleMetadataAvailable: number;
|
||||
}
|
||||
|
||||
export async function getSourceStatisticsList(
|
||||
db: Database,
|
||||
): Promise<SourceStatisticsRow[]> {
|
||||
const rows = await db
|
||||
.select({
|
||||
sourceId: sources.id,
|
||||
sourceName: sources.name,
|
||||
sourceCrawledAt: sql<string | null>`max(${articles.crawledAt})`,
|
||||
articlesCount: sql<number>`count(${articles.id})`,
|
||||
articleMetadataAvailable: sql<number>`sum(CASE WHEN ${articles.metadata} IS NOT NULL THEN 1 ELSE 0 END)`,
|
||||
})
|
||||
.from(sources)
|
||||
.leftJoin(articles, eq(articles.sourceId, sources.id))
|
||||
.groupBy(sources.id, sources.name)
|
||||
.orderBy(sources.name.asc());
|
||||
|
||||
return rows.map((row) => ({
|
||||
sourceId: row.sourceId,
|
||||
sourceName: row.sourceName,
|
||||
sourceCrawledAt: row.sourceCrawledAt,
|
||||
articlesCount: Number(row.articlesCount ?? 0),
|
||||
articleMetadataAvailable: Number(row.articleMetadataAvailable ?? 0),
|
||||
}));
|
||||
}
|
||||
|
||||
export interface PublicationDateParams {
|
||||
source: string;
|
||||
category?: string | null;
|
||||
}
|
||||
|
||||
async function selectPublicationBoundary(
|
||||
db: Database,
|
||||
fn: "min" | "max",
|
||||
params: PublicationDateParams,
|
||||
): Promise<string> {
|
||||
const conditions = [eq(sources.name, params.source)];
|
||||
|
||||
if (params.category) {
|
||||
conditions.push(
|
||||
sql`${params.category} = ANY(${articles.categories})`,
|
||||
);
|
||||
}
|
||||
|
||||
const whereClause = conditions.length > 1
|
||||
? and(...conditions)
|
||||
: conditions[0];
|
||||
|
||||
const [result] = await db
|
||||
.select({
|
||||
boundary:
|
||||
fn === "min"
|
||||
? sql<string | null>`min(${articles.publishedAt})`
|
||||
: sql<string | null>`max(${articles.publishedAt})`,
|
||||
})
|
||||
.from(articles)
|
||||
.innerJoin(sources, eq(articles.sourceId, sources.id))
|
||||
.where(whereClause);
|
||||
|
||||
return result?.boundary ?? new Date().toISOString();
|
||||
}
|
||||
|
||||
export async function getEarliestPublicationDate(
|
||||
db: Database,
|
||||
params: PublicationDateParams,
|
||||
): Promise<string> {
|
||||
return selectPublicationBoundary(db, "min", params);
|
||||
}
|
||||
|
||||
export async function getLatestPublicationDate(
|
||||
db: Database,
|
||||
params: PublicationDateParams,
|
||||
): Promise<string> {
|
||||
return selectPublicationBoundary(db, "max", params);
|
||||
}
|
||||
+104
-32
@@ -1,18 +1,9 @@
|
||||
import type { SQL, AnyColumn } from "drizzle-orm";
|
||||
import {
|
||||
and,
|
||||
asc,
|
||||
desc,
|
||||
eq,
|
||||
gt,
|
||||
lt,
|
||||
or,
|
||||
sql,
|
||||
} from "drizzle-orm";
|
||||
import type { AnyColumn, SQL } from "drizzle-orm";
|
||||
import { and, asc, desc, eq, gt, lt, or, sql } from "drizzle-orm";
|
||||
|
||||
import type { Database } from "@db/client";
|
||||
import {
|
||||
appUsers,
|
||||
users,
|
||||
articles,
|
||||
bookmarkArticles,
|
||||
bookmarks,
|
||||
@@ -104,6 +95,86 @@ interface NormalizedArticleFilters {
|
||||
sortDirection: SortDirection;
|
||||
}
|
||||
|
||||
export interface ArticleExportRow {
|
||||
articleId: string;
|
||||
articleTitle: string;
|
||||
articleLink: string;
|
||||
articleCategories: string | null;
|
||||
articleBody: string;
|
||||
articleSource: string;
|
||||
articleHash: string;
|
||||
articlePublishedAt: string;
|
||||
articleCrawledAt: string;
|
||||
}
|
||||
|
||||
export interface ArticleExportParams {
|
||||
source?: string | null;
|
||||
dateRange?: { start: number; end: number } | null;
|
||||
batchSize?: number;
|
||||
}
|
||||
|
||||
export async function* getArticlesForExport(
|
||||
db: Database,
|
||||
params: ArticleExportParams = {},
|
||||
): AsyncGenerator<ArticleExportRow> {
|
||||
const batchSize =
|
||||
params.batchSize && params.batchSize > 0 ? params.batchSize : 1000;
|
||||
|
||||
const filters: SQL[] = [];
|
||||
|
||||
if (params.source) {
|
||||
filters.push(eq(sources.name, params.source));
|
||||
}
|
||||
|
||||
if (params.dateRange) {
|
||||
filters.push(
|
||||
sql`${articles.publishedAt} BETWEEN to_timestamp(${params.dateRange.start}) AND to_timestamp(${params.dateRange.end})`,
|
||||
);
|
||||
}
|
||||
|
||||
let query = db
|
||||
.select({
|
||||
articleId: articles.id,
|
||||
articleTitle: articles.title,
|
||||
articleLink: articles.link,
|
||||
articleCategories: sql<
|
||||
string | null
|
||||
>`array_to_string(${articles.categories}, ',')`,
|
||||
articleBody: articles.body,
|
||||
articleSource: sources.name,
|
||||
articleHash: articles.hash,
|
||||
articlePublishedAt: articles.publishedAt,
|
||||
articleCrawledAt: articles.crawledAt,
|
||||
})
|
||||
.from(articles)
|
||||
.innerJoin(sources, eq(articles.sourceId, sources.id));
|
||||
|
||||
if (filters.length === 1) {
|
||||
query = query.where(filters[0]);
|
||||
} else if (filters.length > 1) {
|
||||
query = query.where(and(...filters));
|
||||
}
|
||||
|
||||
query = query.orderBy(desc(articles.publishedAt), desc(articles.id));
|
||||
|
||||
let offset = 0;
|
||||
while (true) {
|
||||
const rows = await query.limit(batchSize).offset(offset);
|
||||
if (rows.length === 0) {
|
||||
break;
|
||||
}
|
||||
|
||||
for (const row of rows) {
|
||||
yield {
|
||||
...row,
|
||||
articleCategories: row.articleCategories ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
offset += batchSize;
|
||||
}
|
||||
}
|
||||
|
||||
const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
||||
|
||||
function normalizeArticleFilters(
|
||||
@@ -113,7 +184,8 @@ function normalizeArticleFilters(
|
||||
const trimmedCategory = filters?.category?.trim();
|
||||
|
||||
return {
|
||||
search: trimmedSearch && trimmedSearch.length > 0 ? trimmedSearch : undefined,
|
||||
search:
|
||||
trimmedSearch && trimmedSearch.length > 0 ? trimmedSearch : undefined,
|
||||
category:
|
||||
trimmedCategory && trimmedCategory.length > 0
|
||||
? trimmedCategory
|
||||
@@ -123,9 +195,10 @@ function normalizeArticleFilters(
|
||||
};
|
||||
}
|
||||
|
||||
function buildArticleFilterConditions(
|
||||
filters: NormalizedArticleFilters,
|
||||
): { conditions: SQL[]; searchQuery?: string } {
|
||||
function buildArticleFilterConditions(filters: NormalizedArticleFilters): {
|
||||
conditions: SQL[];
|
||||
searchQuery?: string;
|
||||
} {
|
||||
const conditions: SQL[] = [];
|
||||
let searchQuery: string | undefined;
|
||||
|
||||
@@ -181,7 +254,9 @@ async function fetchArticleOverview(
|
||||
article_id: articles.id,
|
||||
article_title: articles.title,
|
||||
article_link: articles.link,
|
||||
article_categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
||||
article_categories: sql<
|
||||
string | null
|
||||
>`array_to_string(${articles.categories}, ',')`,
|
||||
article_excerpt: articles.excerpt,
|
||||
article_published_at: articles.publishedAt,
|
||||
article_image: articles.image,
|
||||
@@ -242,9 +317,7 @@ async function fetchArticleOverview(
|
||||
orderings.push(desc(articles.publishedAt), desc(articles.id));
|
||||
}
|
||||
|
||||
const rows = await query
|
||||
.orderBy(...orderings)
|
||||
.limit(options.page.limit + 1);
|
||||
const rows = await query.orderBy(...orderings).limit(options.page.limit + 1);
|
||||
|
||||
return buildPaginationResult(rows, options.page, {
|
||||
id: "article_id",
|
||||
@@ -314,7 +387,9 @@ export async function getBookmarkedArticleList(
|
||||
article_id: articles.id,
|
||||
article_title: articles.title,
|
||||
article_link: articles.link,
|
||||
article_categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
||||
article_categories: sql<
|
||||
string | null
|
||||
>`array_to_string(${articles.categories}, ',')`,
|
||||
article_excerpt: articles.excerpt,
|
||||
article_published_at: articles.publishedAt,
|
||||
article_image: articles.image,
|
||||
@@ -377,9 +452,7 @@ export async function getBookmarkedArticleList(
|
||||
orderings.push(desc(articles.publishedAt), desc(articles.id));
|
||||
}
|
||||
|
||||
const rows = await query
|
||||
.orderBy(...orderings)
|
||||
.limit(page.limit + 1);
|
||||
const rows = await query.orderBy(...orderings).limit(page.limit + 1);
|
||||
|
||||
return buildPaginationResult(rows, page, {
|
||||
id: "article_id",
|
||||
@@ -398,7 +471,9 @@ export async function getArticleDetails(
|
||||
article_id: articles.id,
|
||||
article_title: articles.title,
|
||||
article_link: articles.link,
|
||||
article_categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
||||
article_categories: sql<
|
||||
string | null
|
||||
>`array_to_string(${articles.categories}, ',')`,
|
||||
article_body: articles.body,
|
||||
article_hash: articles.hash,
|
||||
article_published_at: articles.publishedAt,
|
||||
@@ -442,10 +517,7 @@ export async function getArticleCommentList(
|
||||
whereConditions.push(
|
||||
or(
|
||||
lt(comments.createdAt, cursor.date),
|
||||
and(
|
||||
eq(comments.createdAt, cursor.date),
|
||||
lt(comments.id, cursor.id),
|
||||
),
|
||||
and(eq(comments.createdAt, cursor.date), lt(comments.id, cursor.id)),
|
||||
),
|
||||
);
|
||||
}
|
||||
@@ -456,11 +528,11 @@ export async function getArticleCommentList(
|
||||
comment_content: comments.content,
|
||||
comment_sentiment: comments.sentiment,
|
||||
comment_created_at: comments.createdAt,
|
||||
user_id: appUsers.id,
|
||||
user_name: appUsers.name,
|
||||
user_id: users.id,
|
||||
user_name: users.name,
|
||||
})
|
||||
.from(comments)
|
||||
.innerJoin(appUsers, eq(comments.userId, appUsers.id));
|
||||
.innerJoin(users, eq(comments.userId, users.id));
|
||||
|
||||
if (whereConditions.length === 1) {
|
||||
query = query.where(whereConditions[0]);
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./articles";
|
||||
export * from "./bookmarks";
|
||||
export * from "./sources";
|
||||
export * from "./users";
|
||||
+125
-19
@@ -9,11 +9,8 @@ import {
|
||||
decodeCursor,
|
||||
type PageRequest,
|
||||
type PaginationMeta,
|
||||
type PageState,
|
||||
} from "@db/utils/pagination";
|
||||
|
||||
const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
||||
const PUBLICATION_GRAPH_DAYS = 180;
|
||||
import { PUBLICATION_GRAPH_DAYS, SOURCE_IMAGE_BASE } from "@db/constant";
|
||||
|
||||
export interface SourceOverviewRow {
|
||||
source_id: string;
|
||||
@@ -62,12 +59,97 @@ export interface SourceDetailsResult {
|
||||
categoryShares: CategoryShare[];
|
||||
}
|
||||
|
||||
export interface SourceStatisticsRow {
|
||||
sourceId: string;
|
||||
sourceName: string;
|
||||
sourceCrawledAt: string | null;
|
||||
articlesCount: number;
|
||||
articleMetadataAvailable: number;
|
||||
}
|
||||
|
||||
export async function getSourceStatisticsList(
|
||||
db: Database,
|
||||
): Promise<SourceStatisticsRow[]> {
|
||||
const rows = await db
|
||||
.select({
|
||||
sourceId: sources.id,
|
||||
sourceName: sources.name,
|
||||
sourceCrawledAt: sql<string | null>`max
|
||||
(${articles.crawledAt})`,
|
||||
articlesCount: sql<number>`count
|
||||
(${articles.id})`,
|
||||
articleMetadataAvailable: sql<number>`sum
|
||||
(CASE WHEN ${articles.metadata} IS NOT NULL THEN 1 ELSE 0 END)`,
|
||||
})
|
||||
.from(sources)
|
||||
.leftJoin(articles, eq(articles.sourceId, sources.id))
|
||||
.groupBy(sources.id, sources.name)
|
||||
.orderBy(sources.name.asc());
|
||||
|
||||
return rows.map((row) => ({
|
||||
sourceId: row.sourceId,
|
||||
sourceName: row.sourceName,
|
||||
sourceCrawledAt: row.sourceCrawledAt,
|
||||
articlesCount: Number(row.articlesCount ?? 0),
|
||||
articleMetadataAvailable: Number(row.articleMetadataAvailable ?? 0),
|
||||
}));
|
||||
}
|
||||
|
||||
export interface PublicationDateParams {
|
||||
source: string;
|
||||
category?: string | null;
|
||||
}
|
||||
|
||||
async function selectPublicationBoundary(
|
||||
db: Database,
|
||||
fn: "min" | "max",
|
||||
params: PublicationDateParams,
|
||||
): Promise<string> {
|
||||
const conditions: SQL[] = [eq(sources.name, params.source)];
|
||||
|
||||
if (params.category) {
|
||||
conditions.push(sql`${params.category} = ANY(${articles.categories})`);
|
||||
}
|
||||
|
||||
const whereClause =
|
||||
conditions.length > 1 ? and(...conditions) : conditions[0];
|
||||
|
||||
const [result] = await db
|
||||
.select({
|
||||
boundary:
|
||||
fn === "min"
|
||||
? sql<string | null>`min
|
||||
(${articles.publishedAt})`
|
||||
: sql<string | null>`max
|
||||
(${articles.publishedAt})`,
|
||||
})
|
||||
.from(articles)
|
||||
.innerJoin(sources, eq(articles.sourceId, sources.id))
|
||||
.where(whereClause);
|
||||
|
||||
return result?.boundary ?? new Date().toISOString();
|
||||
}
|
||||
|
||||
export async function getEarliestPublicationDate(
|
||||
db: Database,
|
||||
params: PublicationDateParams,
|
||||
): Promise<string> {
|
||||
return selectPublicationBoundary(db, "min", params);
|
||||
}
|
||||
|
||||
export async function getLatestPublicationDate(
|
||||
db: Database,
|
||||
params: PublicationDateParams,
|
||||
): Promise<string> {
|
||||
return selectPublicationBoundary(db, "max", params);
|
||||
}
|
||||
|
||||
function buildFollowExistsExpression(userId: string): SQL<boolean> {
|
||||
return sql`EXISTS (
|
||||
SELECT 1
|
||||
FROM ${followedSources} f
|
||||
WHERE f.source_id = ${sources.id} AND f.follower_id = ${userId}
|
||||
)`;
|
||||
return sql`EXISTS
|
||||
(SELECT 1
|
||||
FROM ${followedSources} f
|
||||
WHERE f.source_id = ${sources.id}
|
||||
AND f.follower_id = ${userId})`;
|
||||
}
|
||||
|
||||
export async function getSourceOverviewList(
|
||||
@@ -126,16 +208,27 @@ async function fetchPublicationGraph(
|
||||
|
||||
const rows = await db
|
||||
.select({
|
||||
day: sql<string>`date(${articles.publishedAt})`,
|
||||
count: sql<number>`count(${articles.id})`,
|
||||
day: sql<string>`date
|
||||
(${articles.publishedAt})`,
|
||||
count: sql<number>`count
|
||||
(${articles.id})`,
|
||||
})
|
||||
.from(articles)
|
||||
.where(eq(articles.sourceId, sourceId))
|
||||
.where(
|
||||
sql`${articles.publishedAt} BETWEEN to_timestamp(${range.start}) AND to_timestamp(${range.end})`,
|
||||
sql`${articles.publishedAt} BETWEEN to_timestamp(
|
||||
${range.start}
|
||||
)
|
||||
AND
|
||||
to_timestamp
|
||||
(
|
||||
${range.end}
|
||||
)`,
|
||||
)
|
||||
.groupBy(sql`date(${articles.publishedAt})`)
|
||||
.orderBy(sql`date(${articles.publishedAt})`);
|
||||
.groupBy(sql`date
|
||||
(${articles.publishedAt})`)
|
||||
.orderBy(sql`date
|
||||
(${articles.publishedAt})`);
|
||||
|
||||
const counts = new Map<string, number>();
|
||||
for (const row of rows) {
|
||||
@@ -164,7 +257,8 @@ async function fetchCategoryShares(
|
||||
): Promise<CategoryShare[]> {
|
||||
const rows = await db
|
||||
.select({
|
||||
categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
||||
categories: sql<string | null>`array_to_string
|
||||
(${articles.categories}, ',')`,
|
||||
})
|
||||
.from(articles)
|
||||
.where(eq(articles.sourceId, sourceId));
|
||||
@@ -179,7 +273,10 @@ async function fetchCategoryShares(
|
||||
}
|
||||
}
|
||||
|
||||
const total = Array.from(counts.values()).reduce((acc, value) => acc + value, 0);
|
||||
const total = Array.from(counts.values()).reduce(
|
||||
(acc, value) => acc + value,
|
||||
0,
|
||||
);
|
||||
|
||||
const shares: CategoryShare[] = Array.from(counts.entries()).map(
|
||||
([category, count]) => ({
|
||||
@@ -211,9 +308,18 @@ export async function getSourceDetails(
|
||||
source_reliability: sources.reliability,
|
||||
source_transparency: sources.transparency,
|
||||
source_image: sql<string>`('${SOURCE_IMAGE_BASE}' || ${sources.name} || '.png')`,
|
||||
articles_count: sql<number>`count(${articles.id})`,
|
||||
source_crawled_at: sql<string | null>`max(${articles.crawledAt})`,
|
||||
articles_metadata_available: sql<number>`count(*) FILTER (WHERE ${articles.metadata} IS NOT NULL)`,
|
||||
articles_count: sql<number>`count
|
||||
(${articles.id})`,
|
||||
source_crawled_at: sql<string | null>`max
|
||||
(${articles.crawledAt})`,
|
||||
articles_metadata_available: sql<number>`count
|
||||
(*)
|
||||
FILTER (WHERE
|
||||
${articles.metadata}
|
||||
IS
|
||||
NOT
|
||||
NULL
|
||||
)`,
|
||||
source_is_followed: followExpression,
|
||||
})
|
||||
.from(sources)
|
||||
+8
-8
@@ -1,7 +1,7 @@
|
||||
import { eq } from "drizzle-orm";
|
||||
|
||||
import type { Database } from "@db/client";
|
||||
import { appUsers } from "@db/schema";
|
||||
import { users } from "@db/schema";
|
||||
|
||||
export interface UserProfileRow {
|
||||
user_id: string;
|
||||
@@ -17,14 +17,14 @@ export async function getUserProfile(
|
||||
): Promise<UserProfileRow | null> {
|
||||
const [row] = await db
|
||||
.select({
|
||||
user_id: appUsers.id,
|
||||
user_name: appUsers.name,
|
||||
user_email: appUsers.email,
|
||||
user_created_at: appUsers.createdAt,
|
||||
user_updated_at: appUsers.updatedAt,
|
||||
user_id: users.id,
|
||||
user_name: users.name,
|
||||
user_email: users.email,
|
||||
user_created_at: users.createdAt,
|
||||
user_updated_at: users.updatedAt,
|
||||
})
|
||||
.from(appUsers)
|
||||
.where(eq(appUsers.id, params.userId))
|
||||
.from(users)
|
||||
.where(eq(users.id, params.userId))
|
||||
.limit(1);
|
||||
|
||||
return row ?? null;
|
||||
+49
-3288
File diff suppressed because it is too large
Load Diff
@@ -7,14 +7,14 @@ import { randomBytes } from "node:crypto";
|
||||
export function generateApiKey(): string {
|
||||
// Generate 32 random bytes and convert to hex
|
||||
const randomString = randomBytes(32).toString("hex");
|
||||
return `mid_${randomString}`;
|
||||
return `basango_${randomString}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Validates if a string is a valid API key format
|
||||
* @param key The key to validate
|
||||
* @returns True if the key starts with 'mid-' and has the correct length
|
||||
* @returns True if the key starts with 'basango_' and has the correct length
|
||||
*/
|
||||
export function isValidApiKeyFormat(key: string): boolean {
|
||||
return key.startsWith("mid_") && key.length === 68; // mid_ (4) + 64 hex chars
|
||||
return key.startsWith("basango_") && key.length === 68; // basango_ (8) + 64 hex chars
|
||||
}
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
import { sql } from "drizzle-orm";
|
||||
import { db } from "@db/client";
|
||||
|
||||
export async function checkHealth() {
|
||||
await db.execute(sql`SELECT 1`);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
export * from "./api-keys";
|
||||
export * from "./health";
|
||||
export * from "./pagination";
|
||||
export * from "./search-query";
|
||||
@@ -32,13 +32,15 @@ const DEFAULT_LIMIT = 5;
|
||||
const MAX_LIMIT = 100;
|
||||
|
||||
export function createPageState(request: PageRequest = {}): PageState {
|
||||
const page = Number.isFinite(request.page) && (request.page ?? 0) > 0
|
||||
? Math.trunc(request.page!)
|
||||
: DEFAULT_PAGE;
|
||||
const page =
|
||||
Number.isFinite(request.page) && (request.page ?? 0) > 0
|
||||
? Math.trunc(request.page!)
|
||||
: DEFAULT_PAGE;
|
||||
|
||||
let limit = Number.isFinite(request.limit) && (request.limit ?? 0) > 0
|
||||
? Math.trunc(request.limit!)
|
||||
: DEFAULT_LIMIT;
|
||||
let limit =
|
||||
Number.isFinite(request.limit) && (request.limit ?? 0) > 0
|
||||
? Math.trunc(request.limit!)
|
||||
: DEFAULT_LIMIT;
|
||||
|
||||
if (limit < DEFAULT_LIMIT) {
|
||||
limit = DEFAULT_LIMIT;
|
||||
|
||||
@@ -1,11 +1,11 @@
|
||||
{
|
||||
"extends": "@midday/tsconfig/base.json",
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"include": ["src"],
|
||||
"exclude": ["node_modules"],
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@/db": ["./src/*"]
|
||||
"@db/*": ["./src/*"]
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,18 +1,18 @@
|
||||
{
|
||||
"name": "@basango/logger",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "^5.9.2"
|
||||
},
|
||||
"dependencies": {
|
||||
"pino": "^10.1.0",
|
||||
"pino-pretty": "^13.1.2"
|
||||
}
|
||||
"name": "@basango/logger",
|
||||
"version": "0.0.1",
|
||||
"private": true,
|
||||
"type": "module",
|
||||
"main": "src/index.ts",
|
||||
"types": "src/index.ts",
|
||||
"scripts": {
|
||||
"typecheck": "tsc --noEmit"
|
||||
},
|
||||
"devDependencies": {
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"dependencies": {
|
||||
"pino": "^10.1.0",
|
||||
"pino-pretty": "^13.1.2"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,20 +1,20 @@
|
||||
import pino from "pino";
|
||||
|
||||
export const logger = pino({
|
||||
level: process.env.LOG_LEVEL || "info",
|
||||
// Use pretty printing in development, structured JSON in production
|
||||
...(process.env.NODE_ENV === "development" && {
|
||||
transport: {
|
||||
target: "pino-pretty",
|
||||
options: {
|
||||
colorize: true,
|
||||
translateTime: "HH:MM:ss",
|
||||
ignore: "pid,hostname",
|
||||
messageFormat: true,
|
||||
hideObject: false,
|
||||
},
|
||||
},
|
||||
}),
|
||||
level: process.env.LOG_LEVEL || "info",
|
||||
// Use pretty printing in development, structured JSON in production
|
||||
...(process.env.NODE_ENV === "development" && {
|
||||
transport: {
|
||||
target: "pino-pretty",
|
||||
options: {
|
||||
colorize: true,
|
||||
translateTime: "HH:MM:ss",
|
||||
ignore: "pid,hostname",
|
||||
messageFormat: true,
|
||||
hideObject: false,
|
||||
},
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
export default logger;
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
{
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules"],
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"include": ["src/**/*"],
|
||||
"exclude": ["node_modules"]
|
||||
}
|
||||
|
||||
@@ -1,19 +1,19 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"esModuleInterop": true,
|
||||
"incremental": false,
|
||||
"isolatedModules": true,
|
||||
"lib": ["es2022", "DOM", "DOM.Iterable"],
|
||||
"module": "NodeNext",
|
||||
"moduleDetection": "force",
|
||||
"moduleResolution": "NodeNext",
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"target": "ES2022"
|
||||
}
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"compilerOptions": {
|
||||
"declaration": true,
|
||||
"declarationMap": true,
|
||||
"esModuleInterop": true,
|
||||
"incremental": false,
|
||||
"isolatedModules": true,
|
||||
"lib": ["es2022", "DOM", "DOM.Iterable"],
|
||||
"module": "NodeNext",
|
||||
"moduleDetection": "force",
|
||||
"moduleResolution": "NodeNext",
|
||||
"noUncheckedIndexedAccess": true,
|
||||
"resolveJsonModule": true,
|
||||
"skipLibCheck": true,
|
||||
"strict": true,
|
||||
"target": "ES2022"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "./base.json",
|
||||
"compilerOptions": {
|
||||
"plugins": [{ "name": "next" }],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"allowJs": true,
|
||||
"jsx": "preserve",
|
||||
"noEmit": true
|
||||
}
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "./base.json",
|
||||
"compilerOptions": {
|
||||
"plugins": [{ "name": "next" }],
|
||||
"module": "ESNext",
|
||||
"moduleResolution": "Bundler",
|
||||
"allowJs": true,
|
||||
"jsx": "preserve",
|
||||
"noEmit": true
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@basango/tsconfig",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"base.json"
|
||||
]
|
||||
"name": "@basango/tsconfig",
|
||||
"version": "0.0.0",
|
||||
"private": true,
|
||||
"license": "MIT",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"files": [
|
||||
"base.json"
|
||||
]
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "./base.json",
|
||||
"compilerOptions": {
|
||||
"jsx": "react-jsx"
|
||||
}
|
||||
"$schema": "https://json.schemastore.org/tsconfig",
|
||||
"extends": "./base.json",
|
||||
"compilerOptions": {
|
||||
"jsx": "react-jsx"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,3 +1,3 @@
|
||||
{
|
||||
"extends": "@basango/tsconfig/base.json"
|
||||
"extends": "@basango/tsconfig/base.json"
|
||||
}
|
||||
|
||||
+41
-41
@@ -1,43 +1,43 @@
|
||||
{
|
||||
"$schema": "https://turborepo.com/schema.json",
|
||||
"globalDependencies": ["**/.env"],
|
||||
"ui": "tui",
|
||||
"tasks": {
|
||||
"topo": {
|
||||
"dependsOn": ["^topo"]
|
||||
},
|
||||
"build": {
|
||||
"dependsOn": ["^build"],
|
||||
"inputs": ["$TURBO_DEFAULT$", ".env*"],
|
||||
"outputs": [
|
||||
".next/**",
|
||||
"!.next/cache/**",
|
||||
"next-env.d.ts",
|
||||
".expo/**",
|
||||
"dist/**",
|
||||
"build/**",
|
||||
"lib/**"
|
||||
],
|
||||
"passThroughEnv": []
|
||||
},
|
||||
"start": {
|
||||
"cache": false
|
||||
},
|
||||
"test": {
|
||||
"cache": false
|
||||
},
|
||||
"dev": {
|
||||
"inputs": ["$TURBO_DEFAULT$", ".env"],
|
||||
"cache": false,
|
||||
"persistent": true
|
||||
},
|
||||
"format": {},
|
||||
"lint": {
|
||||
"dependsOn": ["^topo"]
|
||||
},
|
||||
"typecheck": {
|
||||
"dependsOn": ["^topo"],
|
||||
"outputs": []
|
||||
}
|
||||
}
|
||||
"$schema": "https://turborepo.com/schema.json",
|
||||
"globalDependencies": ["**/.env"],
|
||||
"ui": "tui",
|
||||
"tasks": {
|
||||
"topo": {
|
||||
"dependsOn": ["^topo"]
|
||||
},
|
||||
"build": {
|
||||
"dependsOn": ["^build"],
|
||||
"inputs": ["$TURBO_DEFAULT$", ".env*"],
|
||||
"outputs": [
|
||||
".next/**",
|
||||
"!.next/cache/**",
|
||||
"next-env.d.ts",
|
||||
".expo/**",
|
||||
"dist/**",
|
||||
"build/**",
|
||||
"lib/**"
|
||||
],
|
||||
"passThroughEnv": []
|
||||
},
|
||||
"start": {
|
||||
"cache": false
|
||||
},
|
||||
"test": {
|
||||
"cache": false
|
||||
},
|
||||
"dev": {
|
||||
"inputs": ["$TURBO_DEFAULT$", ".env"],
|
||||
"cache": false,
|
||||
"persistent": true
|
||||
},
|
||||
"format": {},
|
||||
"lint": {
|
||||
"dependsOn": ["^topo"]
|
||||
},
|
||||
"typecheck": {
|
||||
"dependsOn": ["^topo"],
|
||||
"outputs": []
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
@@ -0,0 +1,258 @@
|
||||
{
|
||||
"lockfileVersion": 1,
|
||||
"workspaces": {
|
||||
"": {
|
||||
"name": "basango-monorepo",
|
||||
"devDependencies": {
|
||||
"@types/node": "^20.11.30",
|
||||
"typescript": "^5.4.0",
|
||||
"vitest": "^1.6.0",
|
||||
},
|
||||
},
|
||||
},
|
||||
"packages": {
|
||||
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="],
|
||||
|
||||
"@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="],
|
||||
|
||||
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="],
|
||||
|
||||
"@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="],
|
||||
|
||||
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="],
|
||||
|
||||
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="],
|
||||
|
||||
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="],
|
||||
|
||||
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="],
|
||||
|
||||
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="],
|
||||
|
||||
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="],
|
||||
|
||||
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="],
|
||||
|
||||
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="],
|
||||
|
||||
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="],
|
||||
|
||||
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="],
|
||||
|
||||
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="],
|
||||
|
||||
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="],
|
||||
|
||||
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="],
|
||||
|
||||
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="],
|
||||
|
||||
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="],
|
||||
|
||||
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="],
|
||||
|
||||
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="],
|
||||
|
||||
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="],
|
||||
|
||||
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="],
|
||||
|
||||
"@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="],
|
||||
|
||||
"@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="],
|
||||
|
||||
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.52.5", "", { "os": "android", "cpu": "arm" }, "sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ=="],
|
||||
|
||||
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.52.5", "", { "os": "android", "cpu": "arm64" }, "sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA=="],
|
||||
|
||||
"@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.52.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA=="],
|
||||
|
||||
"@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.52.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA=="],
|
||||
|
||||
"@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.52.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA=="],
|
||||
|
||||
"@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.52.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ=="],
|
||||
|
||||
"@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ=="],
|
||||
|
||||
"@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ=="],
|
||||
|
||||
"@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg=="],
|
||||
|
||||
"@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q=="],
|
||||
|
||||
"@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA=="],
|
||||
|
||||
"@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.52.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw=="],
|
||||
|
||||
"@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw=="],
|
||||
|
||||
"@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg=="],
|
||||
|
||||
"@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.52.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ=="],
|
||||
|
||||
"@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q=="],
|
||||
|
||||
"@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg=="],
|
||||
|
||||
"@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.52.5", "", { "os": "none", "cpu": "arm64" }, "sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw=="],
|
||||
|
||||
"@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.52.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w=="],
|
||||
|
||||
"@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.52.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg=="],
|
||||
|
||||
"@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ=="],
|
||||
|
||||
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg=="],
|
||||
|
||||
"@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="],
|
||||
|
||||
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
|
||||
|
||||
"@types/node": ["@types/node@20.19.24", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA=="],
|
||||
|
||||
"@vitest/expect": ["@vitest/expect@1.6.1", "", { "dependencies": { "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "chai": "^4.3.10" } }, "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog=="],
|
||||
|
||||
"@vitest/runner": ["@vitest/runner@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "p-limit": "^5.0.0", "pathe": "^1.1.1" } }, "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA=="],
|
||||
|
||||
"@vitest/snapshot": ["@vitest/snapshot@1.6.1", "", { "dependencies": { "magic-string": "^0.30.5", "pathe": "^1.1.1", "pretty-format": "^29.7.0" } }, "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ=="],
|
||||
|
||||
"@vitest/spy": ["@vitest/spy@1.6.1", "", { "dependencies": { "tinyspy": "^2.2.0" } }, "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw=="],
|
||||
|
||||
"@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="],
|
||||
|
||||
"acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="],
|
||||
|
||||
"acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="],
|
||||
|
||||
"ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="],
|
||||
|
||||
"assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="],
|
||||
|
||||
"cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="],
|
||||
|
||||
"chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="],
|
||||
|
||||
"check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="],
|
||||
|
||||
"confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="],
|
||||
|
||||
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
||||
|
||||
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||
|
||||
"deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="],
|
||||
|
||||
"diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="],
|
||||
|
||||
"esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="],
|
||||
|
||||
"estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="],
|
||||
|
||||
"execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="],
|
||||
|
||||
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||
|
||||
"get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="],
|
||||
|
||||
"get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="],
|
||||
|
||||
"human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="],
|
||||
|
||||
"is-stream": ["is-stream@3.0.0", "", {}, "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA=="],
|
||||
|
||||
"isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
|
||||
|
||||
"js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="],
|
||||
|
||||
"local-pkg": ["local-pkg@0.5.1", "", { "dependencies": { "mlly": "^1.7.3", "pkg-types": "^1.2.1" } }, "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ=="],
|
||||
|
||||
"loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="],
|
||||
|
||||
"magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="],
|
||||
|
||||
"merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="],
|
||||
|
||||
"mimic-fn": ["mimic-fn@4.0.0", "", {}, "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw=="],
|
||||
|
||||
"mlly": ["mlly@1.8.0", "", { "dependencies": { "acorn": "^8.15.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.1" } }, "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g=="],
|
||||
|
||||
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||
|
||||
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||
|
||||
"npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="],
|
||||
|
||||
"onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="],
|
||||
|
||||
"p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="],
|
||||
|
||||
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
|
||||
|
||||
"pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="],
|
||||
|
||||
"pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="],
|
||||
|
||||
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
|
||||
|
||||
"pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="],
|
||||
|
||||
"postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="],
|
||||
|
||||
"pretty-format": ["pretty-format@29.7.0", "", { "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } }, "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ=="],
|
||||
|
||||
"react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="],
|
||||
|
||||
"rollup": ["rollup@4.52.5", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.52.5", "@rollup/rollup-android-arm64": "4.52.5", "@rollup/rollup-darwin-arm64": "4.52.5", "@rollup/rollup-darwin-x64": "4.52.5", "@rollup/rollup-freebsd-arm64": "4.52.5", "@rollup/rollup-freebsd-x64": "4.52.5", "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", "@rollup/rollup-linux-arm-musleabihf": "4.52.5", "@rollup/rollup-linux-arm64-gnu": "4.52.5", "@rollup/rollup-linux-arm64-musl": "4.52.5", "@rollup/rollup-linux-loong64-gnu": "4.52.5", "@rollup/rollup-linux-ppc64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-musl": "4.52.5", "@rollup/rollup-linux-s390x-gnu": "4.52.5", "@rollup/rollup-linux-x64-gnu": "4.52.5", "@rollup/rollup-linux-x64-musl": "4.52.5", "@rollup/rollup-openharmony-arm64": "4.52.5", "@rollup/rollup-win32-arm64-msvc": "4.52.5", "@rollup/rollup-win32-ia32-msvc": "4.52.5", "@rollup/rollup-win32-x64-gnu": "4.52.5", "@rollup/rollup-win32-x64-msvc": "4.52.5", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw=="],
|
||||
|
||||
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
||||
|
||||
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
||||
|
||||
"siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="],
|
||||
|
||||
"signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
|
||||
|
||||
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||
|
||||
"stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="],
|
||||
|
||||
"std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="],
|
||||
|
||||
"strip-final-newline": ["strip-final-newline@3.0.0", "", {}, "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw=="],
|
||||
|
||||
"strip-literal": ["strip-literal@2.1.1", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q=="],
|
||||
|
||||
"tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="],
|
||||
|
||||
"tinypool": ["tinypool@0.8.4", "", {}, "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ=="],
|
||||
|
||||
"tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="],
|
||||
|
||||
"type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="],
|
||||
|
||||
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||
|
||||
"ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],
|
||||
|
||||
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
|
||||
|
||||
"vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="],
|
||||
|
||||
"vite-node": ["vite-node@1.6.1", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.3.4", "pathe": "^1.1.1", "picocolors": "^1.0.0", "vite": "^5.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA=="],
|
||||
|
||||
"vitest": ["vitest@1.6.1", "", { "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", "@vitest/snapshot": "1.6.1", "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "acorn-walk": "^8.3.2", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", "local-pkg": "^0.5.0", "magic-string": "^0.30.5", "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", "strip-literal": "^2.0.0", "tinybench": "^2.5.1", "tinypool": "^0.8.3", "vite": "^5.0.0", "vite-node": "1.6.1", "why-is-node-running": "^2.2.2" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", "@vitest/browser": "1.6.1", "@vitest/ui": "1.6.1", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag=="],
|
||||
|
||||
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
||||
|
||||
"why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="],
|
||||
|
||||
"yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="],
|
||||
|
||||
"mlly/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
|
||||
|
||||
"npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="],
|
||||
|
||||
"pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user