[crawler]: fix conflit
This commit is contained in:
@@ -0,0 +1,7 @@
|
|||||||
|
# porting to Typescript (./basango)
|
||||||
|
- when working on a porting feature (working directory is basango, refer to docs/ for more instructions)
|
||||||
|
- make sure to use `bun` 1.3 as runtime not `node`
|
||||||
|
- prefer `const fn = () => {}` instead of `function`
|
||||||
|
|
||||||
|
# legacy (./projects)
|
||||||
|
- when working on a legacy feature (working director is projects)
|
||||||
@@ -57,13 +57,7 @@
|
|||||||
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
||||||
"replacement": "$3-$2-$1 $4"
|
"replacement": "$3-$2-$1 $4"
|
||||||
},
|
},
|
||||||
"categories": [
|
"categories": ["politique", "economie", "culture", "sport", "societe"],
|
||||||
"politique",
|
|
||||||
"economie",
|
|
||||||
"culture",
|
|
||||||
"sport",
|
|
||||||
"societe"
|
|
||||||
],
|
|
||||||
"source_selectors": {
|
"source_selectors": {
|
||||||
"articles": ".view-content > .row.views-row",
|
"articles": ".view-content > .row.views-row",
|
||||||
"article_title": ".views-field-title a",
|
"article_title": ".views-field-title a",
|
||||||
@@ -119,31 +113,80 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"wordpress": [
|
"wordpress": [
|
||||||
{ "source_id": "beto.cd", "source_url": "https://beto.cd", "requires_rate_limit": true },
|
{
|
||||||
|
"source_id": "beto.cd",
|
||||||
|
"source_url": "https://beto.cd",
|
||||||
|
"requires_rate_limit": true
|
||||||
|
},
|
||||||
{ "source_id": "newscd.net", "source_url": "https://newscd.net" },
|
{ "source_id": "newscd.net", "source_url": "https://newscd.net" },
|
||||||
{ "source_id": "africanewsrdc.net", "source_url": "https://www.africanewsrdc.net" },
|
{
|
||||||
{ "source_id": "angazainstitute.ac.cd", "source_url": "https://angazainstitute.ac.cd" },
|
"source_id": "africanewsrdc.net",
|
||||||
|
"source_url": "https://www.africanewsrdc.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "angazainstitute.ac.cd",
|
||||||
|
"source_url": "https://angazainstitute.ac.cd"
|
||||||
|
},
|
||||||
{ "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" },
|
{ "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" },
|
||||||
{ "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" },
|
{ "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" },
|
||||||
{ "source_id": "changement7.net", "source_url": "https://changement7.net" },
|
{
|
||||||
|
"source_id": "changement7.net",
|
||||||
|
"source_url": "https://changement7.net"
|
||||||
|
},
|
||||||
{ "source_id": "congoactu.net", "source_url": "https://congoactu.net" },
|
{ "source_id": "congoactu.net", "source_url": "https://congoactu.net" },
|
||||||
{ "source_id": "congoindependant.com", "source_url": "https://www.congoindependant.com" },
|
{
|
||||||
{ "source_id": "congoquotidien.com", "source_url": "https://www.congoquotidien.com" },
|
"source_id": "congoindependant.com",
|
||||||
|
"source_url": "https://www.congoindependant.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "congoquotidien.com",
|
||||||
|
"source_url": "https://www.congoquotidien.com"
|
||||||
|
},
|
||||||
{ "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" },
|
{ "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" },
|
||||||
{ "source_id": "environews-rdc.net", "source_url": "https://environews-rdc.net" },
|
{
|
||||||
{ "source_id": "freemediardc.info", "source_url": "https://www.freemediardc.info" },
|
"source_id": "environews-rdc.net",
|
||||||
{ "source_id": "geopolismagazine.org", "source_url": "https://geopolismagazine.org" },
|
"source_url": "https://environews-rdc.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "freemediardc.info",
|
||||||
|
"source_url": "https://www.freemediardc.info"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "geopolismagazine.org",
|
||||||
|
"source_url": "https://geopolismagazine.org"
|
||||||
|
},
|
||||||
{ "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" },
|
{ "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" },
|
||||||
{ "source_id": "infordc.com", "source_url": "https://infordc.com" },
|
{ "source_id": "infordc.com", "source_url": "https://infordc.com" },
|
||||||
{ "source_id": "kilalopress.net", "source_url": "https://kilalopress.net" },
|
{
|
||||||
{ "source_id": "laprosperiteonline.net", "source_url": "https://laprosperiteonline.net" },
|
"source_id": "kilalopress.net",
|
||||||
{ "source_id": "laprunellerdc.cd", "source_url": "https://laprunellerdc.cd" },
|
"source_url": "https://kilalopress.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "laprosperiteonline.net",
|
||||||
|
"source_url": "https://laprosperiteonline.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "laprunellerdc.cd",
|
||||||
|
"source_url": "https://laprunellerdc.cd"
|
||||||
|
},
|
||||||
{ "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" },
|
{ "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" },
|
||||||
{ "source_id": "lesvolcansnews.net", "source_url": "https://lesvolcansnews.net" },
|
{
|
||||||
{ "source_id": "netic-news.net", "source_url": "https://www.netic-news.net" },
|
"source_id": "lesvolcansnews.net",
|
||||||
{ "source_id": "objectif-infos.cd", "source_url": "https://objectif-infos.cd" },
|
"source_url": "https://lesvolcansnews.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "netic-news.net",
|
||||||
|
"source_url": "https://www.netic-news.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "objectif-infos.cd",
|
||||||
|
"source_url": "https://objectif-infos.cd"
|
||||||
|
},
|
||||||
{ "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" },
|
{ "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" },
|
||||||
{ "source_id": "journaldekinshasa.com", "source_url": "https://www.journaldekinshasa.com" },
|
{
|
||||||
|
"source_id": "journaldekinshasa.com",
|
||||||
|
"source_url": "https://www.journaldekinshasa.com"
|
||||||
|
},
|
||||||
{ "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" },
|
{ "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" },
|
||||||
{ "source_id": "acturdc.com", "source_url": "https://acturdc.com" },
|
{ "source_id": "acturdc.com", "source_url": "https://acturdc.com" },
|
||||||
{ "source_id": "matininfos.net", "source_url": "https://matininfos.net" }
|
{ "source_id": "matininfos.net", "source_url": "https://matininfos.net" }
|
||||||
|
|||||||
@@ -57,13 +57,7 @@
|
|||||||
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
"pattern": "/\\w{3} (\\d{2})/(\\d{2})/(\\d{4}) - (\\d{2}:\\d{2})/",
|
||||||
"replacement": "$3-$2-$1 $4"
|
"replacement": "$3-$2-$1 $4"
|
||||||
},
|
},
|
||||||
"categories": [
|
"categories": ["politique", "economie", "culture", "sport", "societe"],
|
||||||
"politique",
|
|
||||||
"economie",
|
|
||||||
"culture",
|
|
||||||
"sport",
|
|
||||||
"societe"
|
|
||||||
],
|
|
||||||
"source_selectors": {
|
"source_selectors": {
|
||||||
"articles": ".view-content > .row.views-row",
|
"articles": ".view-content > .row.views-row",
|
||||||
"article_title": ".views-field-title a",
|
"article_title": ".views-field-title a",
|
||||||
@@ -119,31 +113,80 @@
|
|||||||
}
|
}
|
||||||
],
|
],
|
||||||
"wordpress": [
|
"wordpress": [
|
||||||
{ "source_id": "beto.cd", "source_url": "https://beto.cd", "requires_rate_limit": true },
|
{
|
||||||
|
"source_id": "beto.cd",
|
||||||
|
"source_url": "https://beto.cd",
|
||||||
|
"requires_rate_limit": true
|
||||||
|
},
|
||||||
{ "source_id": "newscd.net", "source_url": "https://newscd.net" },
|
{ "source_id": "newscd.net", "source_url": "https://newscd.net" },
|
||||||
{ "source_id": "africanewsrdc.net", "source_url": "https://www.africanewsrdc.net" },
|
{
|
||||||
{ "source_id": "angazainstitute.ac.cd", "source_url": "https://angazainstitute.ac.cd" },
|
"source_id": "africanewsrdc.net",
|
||||||
|
"source_url": "https://www.africanewsrdc.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "angazainstitute.ac.cd",
|
||||||
|
"source_url": "https://angazainstitute.ac.cd"
|
||||||
|
},
|
||||||
{ "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" },
|
{ "source_id": "b-onetv.cd", "source_url": "https://b-onetv.cd" },
|
||||||
{ "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" },
|
{ "source_id": "bukavufm.com", "source_url": "https://bukavufm.com" },
|
||||||
{ "source_id": "changement7.net", "source_url": "https://changement7.net" },
|
{
|
||||||
|
"source_id": "changement7.net",
|
||||||
|
"source_url": "https://changement7.net"
|
||||||
|
},
|
||||||
{ "source_id": "congoactu.net", "source_url": "https://congoactu.net" },
|
{ "source_id": "congoactu.net", "source_url": "https://congoactu.net" },
|
||||||
{ "source_id": "congoindependant.com", "source_url": "https://www.congoindependant.com" },
|
{
|
||||||
{ "source_id": "congoquotidien.com", "source_url": "https://www.congoquotidien.com" },
|
"source_id": "congoindependant.com",
|
||||||
|
"source_url": "https://www.congoindependant.com"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "congoquotidien.com",
|
||||||
|
"source_url": "https://www.congoquotidien.com"
|
||||||
|
},
|
||||||
{ "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" },
|
{ "source_id": "cumulard.cd", "source_url": "https://www.cumulard.cd" },
|
||||||
{ "source_id": "environews-rdc.net", "source_url": "https://environews-rdc.net" },
|
{
|
||||||
{ "source_id": "freemediardc.info", "source_url": "https://www.freemediardc.info" },
|
"source_id": "environews-rdc.net",
|
||||||
{ "source_id": "geopolismagazine.org", "source_url": "https://geopolismagazine.org" },
|
"source_url": "https://environews-rdc.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "freemediardc.info",
|
||||||
|
"source_url": "https://www.freemediardc.info"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "geopolismagazine.org",
|
||||||
|
"source_url": "https://geopolismagazine.org"
|
||||||
|
},
|
||||||
{ "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" },
|
{ "source_id": "habarirdc.net", "source_url": "https://habarirdc.net" },
|
||||||
{ "source_id": "infordc.com", "source_url": "https://infordc.com" },
|
{ "source_id": "infordc.com", "source_url": "https://infordc.com" },
|
||||||
{ "source_id": "kilalopress.net", "source_url": "https://kilalopress.net" },
|
{
|
||||||
{ "source_id": "laprosperiteonline.net", "source_url": "https://laprosperiteonline.net" },
|
"source_id": "kilalopress.net",
|
||||||
{ "source_id": "laprunellerdc.cd", "source_url": "https://laprunellerdc.cd" },
|
"source_url": "https://kilalopress.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "laprosperiteonline.net",
|
||||||
|
"source_url": "https://laprosperiteonline.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "laprunellerdc.cd",
|
||||||
|
"source_url": "https://laprunellerdc.cd"
|
||||||
|
},
|
||||||
{ "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" },
|
{ "source_id": "lesmedias.net", "source_url": "https://lesmedias.net" },
|
||||||
{ "source_id": "lesvolcansnews.net", "source_url": "https://lesvolcansnews.net" },
|
{
|
||||||
{ "source_id": "netic-news.net", "source_url": "https://www.netic-news.net" },
|
"source_id": "lesvolcansnews.net",
|
||||||
{ "source_id": "objectif-infos.cd", "source_url": "https://objectif-infos.cd" },
|
"source_url": "https://lesvolcansnews.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "netic-news.net",
|
||||||
|
"source_url": "https://www.netic-news.net"
|
||||||
|
},
|
||||||
|
{
|
||||||
|
"source_id": "objectif-infos.cd",
|
||||||
|
"source_url": "https://objectif-infos.cd"
|
||||||
|
},
|
||||||
{ "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" },
|
{ "source_id": "scooprdc.net", "source_url": "https://scooprdc.net" },
|
||||||
{ "source_id": "journaldekinshasa.com", "source_url": "https://www.journaldekinshasa.com" },
|
{
|
||||||
|
"source_id": "journaldekinshasa.com",
|
||||||
|
"source_url": "https://www.journaldekinshasa.com"
|
||||||
|
},
|
||||||
{ "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" },
|
{ "source_id": "lepotentiel.cd", "source_url": "https://lepotentiel.cd" },
|
||||||
{ "source_id": "acturdc.com", "source_url": "https://acturdc.com" },
|
{ "source_id": "acturdc.com", "source_url": "https://acturdc.com" },
|
||||||
{ "source_id": "matininfos.net", "source_url": "https://matininfos.net" }
|
{ "source_id": "matininfos.net", "source_url": "https://matininfos.net" }
|
||||||
|
|||||||
@@ -1,22 +1,22 @@
|
|||||||
{
|
{
|
||||||
"name": "@basango/crawler",
|
"name": "@basango/crawler",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "dist/index.js",
|
"main": "dist/index.js",
|
||||||
"types": "dist/index.d.ts",
|
"types": "dist/index.d.ts",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "tsc -b",
|
"build": "tsc -b",
|
||||||
"test": "vitest --run",
|
"test": "vitest --run",
|
||||||
"queue": "bun run src/scripts/queue.ts",
|
"queue": "bun run src/scripts/queue.ts",
|
||||||
"worker": "bun run src/scripts/worker.ts"
|
"worker": "bun run src/scripts/worker.ts"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"@basango/logger": "workspace:*",
|
"@basango/logger": "workspace:*",
|
||||||
"bullmq": "^4.17.0",
|
"bullmq": "^4.17.0",
|
||||||
"date-fns": "^3.6.0",
|
"date-fns": "^3.6.0",
|
||||||
"ioredis": "^5.3.2",
|
"ioredis": "^5.3.2",
|
||||||
"tiktoken": "^1.0.14",
|
"tiktoken": "^1.0.14",
|
||||||
"zod": "^4.0.0"
|
"zod": "catalog:"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -8,74 +8,74 @@ import { loadConfig } from "./config";
|
|||||||
import { resolveConfigPath } from "./schema";
|
import { resolveConfigPath } from "./schema";
|
||||||
|
|
||||||
describe("loadConfig", () => {
|
describe("loadConfig", () => {
|
||||||
it("parses json configuration and ensures directories", () => {
|
it("parses json configuration and ensures directories", () => {
|
||||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
||||||
const paths = {
|
const paths = {
|
||||||
root: tempDir,
|
root: tempDir,
|
||||||
data: path.join(tempDir, "data"),
|
data: path.join(tempDir, "data"),
|
||||||
logs: path.join(tempDir, "logs"),
|
logs: path.join(tempDir, "logs"),
|
||||||
configs: path.join(tempDir, "configs"),
|
configs: path.join(tempDir, "configs"),
|
||||||
};
|
};
|
||||||
|
|
||||||
const configPath = path.join(tempDir, "pipeline.json");
|
const configPath = path.join(tempDir, "pipeline.json");
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
configPath,
|
configPath,
|
||||||
JSON.stringify(
|
JSON.stringify(
|
||||||
{
|
{
|
||||||
paths,
|
paths,
|
||||||
fetch: {
|
fetch: {
|
||||||
client: { timeout: 10 },
|
client: { timeout: 10 },
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
null,
|
null,
|
||||||
2,
|
2,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
const config = loadConfig({ configPath });
|
const config = loadConfig({ configPath });
|
||||||
|
|
||||||
expect(config.fetch.client.timeout).toBe(10);
|
expect(config.fetch.client.timeout).toBe(10);
|
||||||
expect(fs.existsSync(paths.data)).toBe(true);
|
expect(fs.existsSync(paths.data)).toBe(true);
|
||||||
expect(fs.existsSync(paths.logs)).toBe(true);
|
expect(fs.existsSync(paths.logs)).toBe(true);
|
||||||
expect(fs.existsSync(paths.configs)).toBe(true);
|
expect(fs.existsSync(paths.configs)).toBe(true);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("merges environment override if available", () => {
|
it("merges environment override if available", () => {
|
||||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), "crawler-config-"));
|
||||||
const paths = {
|
const paths = {
|
||||||
root: tempDir,
|
root: tempDir,
|
||||||
data: path.join(tempDir, "data"),
|
data: path.join(tempDir, "data"),
|
||||||
logs: path.join(tempDir, "logs"),
|
logs: path.join(tempDir, "logs"),
|
||||||
configs: path.join(tempDir, "configs"),
|
configs: path.join(tempDir, "configs"),
|
||||||
};
|
};
|
||||||
|
|
||||||
const basePath = path.join(tempDir, "pipeline.json");
|
const basePath = path.join(tempDir, "pipeline.json");
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
basePath,
|
basePath,
|
||||||
JSON.stringify(
|
JSON.stringify(
|
||||||
{
|
{
|
||||||
paths,
|
paths,
|
||||||
logging: { level: "INFO" },
|
logging: { level: "INFO" },
|
||||||
},
|
},
|
||||||
null,
|
null,
|
||||||
2,
|
2,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
const overridePath = resolveConfigPath(basePath, "production");
|
const overridePath = resolveConfigPath(basePath, "production");
|
||||||
fs.writeFileSync(
|
fs.writeFileSync(
|
||||||
overridePath,
|
overridePath,
|
||||||
JSON.stringify(
|
JSON.stringify(
|
||||||
{
|
{
|
||||||
logging: { level: "DEBUG" },
|
logging: { level: "DEBUG" },
|
||||||
},
|
},
|
||||||
null,
|
null,
|
||||||
2,
|
2,
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
|
|
||||||
const config = loadConfig({ configPath: basePath, env: "production" });
|
const config = loadConfig({ configPath: basePath, env: "production" });
|
||||||
|
|
||||||
expect(config.logging.level).toBe("DEBUG");
|
expect(config.logging.level).toBe("DEBUG");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -3,56 +3,56 @@ import { describe, expect, it } from "vitest";
|
|||||||
import { createQueueManager, createQueueSettings } from "./queue";
|
import { createQueueManager, createQueueSettings } from "./queue";
|
||||||
|
|
||||||
class InMemoryQueue {
|
class InMemoryQueue {
|
||||||
public jobs: Array<{ name: string; data: unknown }> = [];
|
public jobs: Array<{ name: string; data: unknown }> = [];
|
||||||
|
|
||||||
async add(name: string, data: unknown) {
|
async add(name: string, data: unknown) {
|
||||||
this.jobs.push({ name, data });
|
this.jobs.push({ name, data });
|
||||||
return { id: `${name}-${this.jobs.length}` };
|
return { id: `${name}-${this.jobs.length}` };
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
describe("createQueueManager", () => {
|
describe("createQueueManager", () => {
|
||||||
it("prefixes queue names", () => {
|
it("prefixes queue names", () => {
|
||||||
const manager = createQueueManager({
|
const manager = createQueueManager({
|
||||||
settings: createQueueSettings({ prefix: "test" }),
|
settings: createQueueSettings({ prefix: "test" }),
|
||||||
queueFactory: (queueName) => {
|
queueFactory: (queueName) => {
|
||||||
expect(queueName).toBe("listing");
|
expect(queueName).toBe("listing");
|
||||||
return new InMemoryQueue();
|
return new InMemoryQueue();
|
||||||
},
|
},
|
||||||
connection: {
|
connection: {
|
||||||
quit: async () => undefined,
|
quit: async () => undefined,
|
||||||
} as any,
|
} as any,
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(manager.iterQueueNames()).toEqual([
|
expect(manager.iterQueueNames()).toEqual([
|
||||||
"test:listing",
|
"test:listing",
|
||||||
"test:articles",
|
"test:articles",
|
||||||
"test:processed",
|
"test:processed",
|
||||||
]);
|
]);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("enqueues listing job with validated payload", async () => {
|
it("enqueues listing job with validated payload", async () => {
|
||||||
const queue = new InMemoryQueue();
|
const queue = new InMemoryQueue();
|
||||||
const manager = createQueueManager({
|
const manager = createQueueManager({
|
||||||
queueFactory: () => queue,
|
queueFactory: () => queue,
|
||||||
connection: { quit: async () => undefined } as any,
|
connection: { quit: async () => undefined } as any,
|
||||||
});
|
});
|
||||||
|
|
||||||
const job = await manager.enqueueListing({
|
const job = await manager.enqueueListing({
|
||||||
source_id: "radiookapi",
|
source_id: "radiookapi",
|
||||||
env: "test",
|
env: "test",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(job.id).toBe("collect_listing-1");
|
expect(job.id).toBe("collect_listing-1");
|
||||||
expect(queue.jobs[0]).toEqual({
|
expect(queue.jobs[0]).toEqual({
|
||||||
name: "collect_listing",
|
name: "collect_listing",
|
||||||
data: {
|
data: {
|
||||||
source_id: "radiookapi",
|
source_id: "radiookapi",
|
||||||
env: "test",
|
env: "test",
|
||||||
page_range: undefined,
|
page_range: undefined,
|
||||||
date_range: undefined,
|
date_range: undefined,
|
||||||
category: undefined,
|
category: undefined,
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
export * from "../services/crawler/async/queue";
|
export * from "@basango/crawler/services/async/queue";
|
||||||
|
|||||||
@@ -1,35 +1,35 @@
|
|||||||
import { describe, expect, it } from "vitest";
|
import { describe, expect, it } from "vitest";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
PipelineConfigSchema,
|
PipelineConfigSchema,
|
||||||
createDateRange,
|
createDateRange,
|
||||||
formatDateRange,
|
formatDateRange,
|
||||||
isTimestampInRange,
|
isTimestampInRange,
|
||||||
PageRangeSpecSchema,
|
PageRangeSpecSchema,
|
||||||
PageRangeSchema,
|
PageRangeSchema,
|
||||||
schemaToJSON,
|
schemaToJSON,
|
||||||
} from "./schema";
|
} from "./schema";
|
||||||
|
|
||||||
describe("schema helpers", () => {
|
describe("schema helpers", () => {
|
||||||
it("creates date range from spec", () => {
|
it("creates date range from spec", () => {
|
||||||
const range = createDateRange("2024-01-01:2024-01-31");
|
const range = createDateRange("2024-01-01:2024-01-31");
|
||||||
expect(range.start).toBeLessThan(range.end);
|
expect(range.start).toBeLessThan(range.end);
|
||||||
expect(formatDateRange(range)).toBe("2024-01-01:2024-01-31");
|
expect(formatDateRange(range)).toBe("2024-01-01:2024-01-31");
|
||||||
});
|
});
|
||||||
|
|
||||||
it("checks membership", () => {
|
it("checks membership", () => {
|
||||||
const range = createDateRange("2024-01-01:2024-01-02");
|
const range = createDateRange("2024-01-01:2024-01-02");
|
||||||
expect(isTimestampInRange(range, range.start)).toBe(true);
|
expect(isTimestampInRange(range, range.start)).toBe(true);
|
||||||
expect(isTimestampInRange(range, range.start - 1)).toBe(false);
|
expect(isTimestampInRange(range, range.start - 1)).toBe(false);
|
||||||
});
|
});
|
||||||
|
|
||||||
it("parses page range spec", () => {
|
it("parses page range spec", () => {
|
||||||
const range = PageRangeSchema.parse(PageRangeSpecSchema.parse("1:10"));
|
const range = PageRangeSchema.parse(PageRangeSpecSchema.parse("1:10"));
|
||||||
expect(range).toEqual({ start: 1, end: 10 });
|
expect(range).toEqual({ start: 1, end: 10 });
|
||||||
});
|
});
|
||||||
|
|
||||||
it("produces json schema", () => {
|
it("produces json schema", () => {
|
||||||
const json = schemaToJSON(PipelineConfigSchema);
|
const json = schemaToJSON(PipelineConfigSchema);
|
||||||
expect(json.type).toBe("object");
|
expect(json.type).toBe("object");
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1,50 +1,50 @@
|
|||||||
import { describe, expect, it, vi } from "vitest";
|
import { describe, expect, it, vi } from "vitest";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
scheduleAsyncCrawl,
|
scheduleAsyncCrawl,
|
||||||
registerCrawlerTaskHandlers,
|
registerCrawlerTaskHandlers,
|
||||||
collectListing,
|
collectListing,
|
||||||
} from "./tasks";
|
} from "./tasks";
|
||||||
import { QueueManager } from "./queue";
|
import { QueueManager } from "./queue";
|
||||||
|
|
||||||
describe("Async tasks", () => {
|
describe("Async tasks", () => {
|
||||||
it("schedules crawl with provided manager", async () => {
|
it("schedules crawl with provided manager", async () => {
|
||||||
const enqueueListing = vi.fn().mockResolvedValue({ id: "job-1" });
|
const enqueueListing = vi.fn().mockResolvedValue({ id: "job-1" });
|
||||||
const manager = {
|
const manager = {
|
||||||
enqueueListing,
|
enqueueListing,
|
||||||
} as unknown as QueueManager;
|
} as unknown as QueueManager;
|
||||||
|
|
||||||
const jobId = await scheduleAsyncCrawl({
|
const jobId = await scheduleAsyncCrawl({
|
||||||
sourceId: "radiookapi",
|
sourceId: "radiookapi",
|
||||||
queueManager: manager,
|
queueManager: manager,
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(jobId).toBe("job-1");
|
expect(jobId).toBe("job-1");
|
||||||
expect(enqueueListing).toHaveBeenCalledWith({
|
expect(enqueueListing).toHaveBeenCalledWith({
|
||||||
source_id: "radiookapi",
|
source_id: "radiookapi",
|
||||||
env: "development",
|
env: "development",
|
||||||
page_range: undefined,
|
page_range: undefined,
|
||||||
date_range: undefined,
|
date_range: undefined,
|
||||||
category: undefined,
|
category: undefined,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|
||||||
it("delegates listing collection to registered handler", async () => {
|
it("delegates listing collection to registered handler", async () => {
|
||||||
const handler = vi.fn().mockResolvedValue(5);
|
const handler = vi.fn().mockResolvedValue(5);
|
||||||
registerCrawlerTaskHandlers({ collectListing: handler });
|
registerCrawlerTaskHandlers({ collectListing: handler });
|
||||||
|
|
||||||
const count = await collectListing({
|
const count = await collectListing({
|
||||||
source_id: "radiookapi",
|
source_id: "radiookapi",
|
||||||
env: "development",
|
env: "development",
|
||||||
});
|
});
|
||||||
|
|
||||||
expect(count).toBe(5);
|
expect(count).toBe(5);
|
||||||
expect(handler).toHaveBeenCalledWith({
|
expect(handler).toHaveBeenCalledWith({
|
||||||
source_id: "radiookapi",
|
source_id: "radiookapi",
|
||||||
env: "development",
|
env: "development",
|
||||||
page_range: undefined,
|
page_range: undefined,
|
||||||
date_range: undefined,
|
date_range: undefined,
|
||||||
category: undefined,
|
category: undefined,
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
});
|
});
|
||||||
|
|||||||
@@ -1 +1 @@
|
|||||||
export * from "../services/crawler/async/tasks";
|
export * from "@basango/crawler/services/async/tasks";
|
||||||
|
|||||||
+125
-129
@@ -1,186 +1,182 @@
|
|||||||
import fs from "node:fs";
|
import * as fs from "node:fs";
|
||||||
import path from "node:path";
|
import * as path from "node:path";
|
||||||
|
|
||||||
import { logger } from "@basango/logger";
|
import {logger} from "@basango/logger";
|
||||||
|
|
||||||
import {
|
import {
|
||||||
PipelineConfig,
|
mergePipelineConfig,
|
||||||
PipelineConfigSchema,
|
PipelineConfig,
|
||||||
mergePipelineConfig,
|
PipelineConfigSchema,
|
||||||
resolveConfigPath,
|
resolveConfigPath,
|
||||||
resolveProjectPaths,
|
resolveProjectPaths,
|
||||||
} from "./schema";
|
} from "./schema";
|
||||||
import { ensureDirectories } from "./utils";
|
import {ensureDirectories} from "./utils";
|
||||||
|
|
||||||
export interface LoadConfigOptions {
|
export interface LoadConfigOptions {
|
||||||
configPath?: string;
|
configPath?: string;
|
||||||
env?: string;
|
env?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
const DEFAULT_CONFIG_FILES = [
|
const DEFAULT_CONFIG_FILES = [
|
||||||
path.join(process.cwd(), "config", "pipeline.json"),
|
path.join(process.cwd(), "config", "pipeline.json"),
|
||||||
path.join(process.cwd(), "pipeline.json"),
|
path.join(process.cwd(), "pipeline.json"),
|
||||||
];
|
];
|
||||||
|
|
||||||
const readJsonFile = (filePath: string): unknown => {
|
const readJsonFile = (filePath: string): unknown => {
|
||||||
const contents = fs.readFileSync(filePath, "utf-8");
|
const contents = fs.readFileSync(filePath, "utf-8");
|
||||||
return contents.trim() === "" ? {} : JSON.parse(contents);
|
return contents.trim() === "" ? {} : JSON.parse(contents);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const locateConfigFile = (explicit?: string): string => {
|
const locateConfigFile = (explicit?: string): string => {
|
||||||
if (explicit && fs.existsSync(explicit)) {
|
if (explicit && fs.existsSync(explicit)) {
|
||||||
return explicit;
|
return explicit;
|
||||||
}
|
}
|
||||||
|
|
||||||
for (const candidate of DEFAULT_CONFIG_FILES) {
|
for (const candidate of DEFAULT_CONFIG_FILES) {
|
||||||
if (fs.existsSync(candidate)) {
|
if (fs.existsSync(candidate)) {
|
||||||
return candidate;
|
return candidate;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
return DEFAULT_CONFIG_FILES[0];
|
return DEFAULT_CONFIG_FILES[0];
|
||||||
};
|
};
|
||||||
|
|
||||||
const readPipelineConfig = (configPath: string): PipelineConfig => {
|
const readPipelineConfig = (configPath: string): PipelineConfig => {
|
||||||
if (!fs.existsSync(configPath)) {
|
if (!fs.existsSync(configPath)) {
|
||||||
return PipelineConfigSchema.parse({
|
return PipelineConfigSchema.parse({
|
||||||
paths: resolveProjectPaths(path.resolve(".")),
|
paths: resolveProjectPaths(path.resolve(".")),
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
const raw = readJsonFile(configPath);
|
const raw = readJsonFile(configPath);
|
||||||
return PipelineConfigSchema.parse(raw);
|
return PipelineConfigSchema.parse(raw);
|
||||||
};
|
};
|
||||||
|
|
||||||
const applyEnvironmentOverride = (
|
const applyEnvironmentOverride = (
|
||||||
baseConfig: PipelineConfig,
|
baseConfig: PipelineConfig,
|
||||||
basePath: string,
|
basePath: string,
|
||||||
env?: string,
|
env?: string,
|
||||||
): PipelineConfig => {
|
): PipelineConfig => {
|
||||||
if (!env || env === "development") {
|
if (!env || env === "development") {
|
||||||
return baseConfig;
|
return baseConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
const overridePath = resolveConfigPath(basePath, env);
|
const overridePath = resolveConfigPath(basePath, env);
|
||||||
if (!fs.existsSync(overridePath)) {
|
if (!fs.existsSync(overridePath)) {
|
||||||
return baseConfig;
|
return baseConfig;
|
||||||
}
|
}
|
||||||
|
|
||||||
const overrides = PipelineConfigSchema.parse(readJsonFile(overridePath));
|
const overrides = PipelineConfigSchema.parse(readJsonFile(overridePath));
|
||||||
return mergePipelineConfig(baseConfig, overrides);
|
return mergePipelineConfig(baseConfig, overrides);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const loadConfig = (options: LoadConfigOptions = {}): PipelineConfig => {
|
export const loadConfig = (options: LoadConfigOptions = {}): PipelineConfig => {
|
||||||
const basePath = locateConfigFile(options.configPath);
|
const basePath = locateConfigFile(options.configPath);
|
||||||
const config = applyEnvironmentOverride(
|
const config = applyEnvironmentOverride(
|
||||||
readPipelineConfig(basePath),
|
readPipelineConfig(basePath),
|
||||||
basePath,
|
basePath,
|
||||||
options.env,
|
options.env,
|
||||||
);
|
);
|
||||||
|
|
||||||
ensureDirectories(config.paths);
|
ensureDirectories(config.paths);
|
||||||
return config;
|
return config;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const dumpConfig = (
|
export const dumpConfig = (
|
||||||
config: PipelineConfig,
|
config: PipelineConfig,
|
||||||
targetPath?: string,
|
targetPath?: string,
|
||||||
): void => {
|
): void => {
|
||||||
const destination = targetPath ?? locateConfigFile();
|
const destination = targetPath ?? locateConfigFile();
|
||||||
const normalized = PipelineConfigSchema.parse(config);
|
const normalized = PipelineConfigSchema.parse(config);
|
||||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
fs.mkdirSync(path.dirname(destination), {recursive: true});
|
||||||
fs.writeFileSync(destination, JSON.stringify(normalized, null, 2));
|
fs.writeFileSync(destination, JSON.stringify(normalized, null, 2));
|
||||||
};
|
};
|
||||||
|
|
||||||
export interface PipelineConfigManagerOptions {
|
export interface PipelineConfigManagerOptions {
|
||||||
configPath?: string;
|
configPath?: string;
|
||||||
env?: string;
|
env?: string;
|
||||||
autoLoad?: boolean;
|
autoLoad?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
export class PipelineConfigManager {
|
export class PipelineConfigManager {
|
||||||
private readonly explicitPath?: string;
|
private readonly explicitPath?: string;
|
||||||
|
|
||||||
private readonly defaultEnv: string;
|
private readonly defaultEnv: string;
|
||||||
|
|
||||||
private cache?: PipelineConfig;
|
private cache?: PipelineConfig;
|
||||||
|
|
||||||
constructor(options: PipelineConfigManagerOptions = {}) {
|
constructor(options: PipelineConfigManagerOptions = {}) {
|
||||||
this.explicitPath = options.configPath;
|
this.explicitPath = options.configPath;
|
||||||
this.defaultEnv = options.env ?? "development";
|
this.defaultEnv = options.env ?? "development";
|
||||||
|
|
||||||
if (options.autoLoad !== false) {
|
if (options.autoLoad !== false) {
|
||||||
this.cache = loadConfig({
|
this.cache = loadConfig({
|
||||||
configPath: this.explicitPath,
|
configPath: this.explicitPath,
|
||||||
env: this.defaultEnv,
|
env: this.defaultEnv,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
get(env?: string): PipelineConfig {
|
get(env?: string): PipelineConfig {
|
||||||
const resolvedEnv = env ?? this.defaultEnv;
|
const resolvedEnv = env ?? this.defaultEnv;
|
||||||
|
|
||||||
if (resolvedEnv !== this.defaultEnv) {
|
if (resolvedEnv !== this.defaultEnv) {
|
||||||
return loadConfig({
|
return loadConfig({
|
||||||
configPath: this.explicitPath,
|
configPath: this.explicitPath,
|
||||||
env: resolvedEnv,
|
env: resolvedEnv,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
if (!this.cache) {
|
if (!this.cache) {
|
||||||
this.cache = loadConfig({
|
this.cache = loadConfig({
|
||||||
configPath: this.explicitPath,
|
configPath: this.explicitPath,
|
||||||
env: resolvedEnv,
|
env: resolvedEnv,
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
return this.cache;
|
return this.cache;
|
||||||
}
|
}
|
||||||
|
|
||||||
reload(env?: string): PipelineConfig {
|
reload(env?: string): PipelineConfig {
|
||||||
const resolvedEnv = env ?? this.defaultEnv;
|
const resolvedEnv = env ?? this.defaultEnv;
|
||||||
const config = loadConfig({
|
const config = loadConfig({
|
||||||
configPath: this.explicitPath,
|
configPath: this.explicitPath,
|
||||||
env: resolvedEnv,
|
env: resolvedEnv,
|
||||||
});
|
});
|
||||||
|
|
||||||
if (resolvedEnv === this.defaultEnv) {
|
if (resolvedEnv === this.defaultEnv) {
|
||||||
this.cache = config;
|
this.cache = config;
|
||||||
}
|
}
|
||||||
|
|
||||||
return config;
|
return config;
|
||||||
}
|
}
|
||||||
|
|
||||||
ensureDirectories(config?: PipelineConfig): PipelineConfig {
|
ensureDirectories(config?: PipelineConfig): PipelineConfig {
|
||||||
const pipeline = config ?? this.get();
|
const pipeline = config ?? this.get();
|
||||||
ensureDirectories(pipeline.paths);
|
ensureDirectories(pipeline.paths);
|
||||||
return pipeline;
|
return pipeline;
|
||||||
}
|
}
|
||||||
|
|
||||||
setupLogging(config?: PipelineConfig): void {
|
setupLogging(config?: PipelineConfig): void {
|
||||||
const pipeline = config ?? this.get();
|
const pipeline = config ?? this.get();
|
||||||
this.ensureDirectories(pipeline);
|
this.ensureDirectories(pipeline);
|
||||||
|
|
||||||
const level = pipeline.logging.level.toLowerCase();
|
const level = pipeline.logging.level.toLowerCase();
|
||||||
process.env.LOG_LEVEL = level;
|
process.env.LOG_LEVEL = level;
|
||||||
const normalizedLevel = level as typeof logger.level;
|
logger.level = level as typeof logger.level;
|
||||||
logger.level = normalizedLevel;
|
|
||||||
|
|
||||||
if (pipeline.logging.file_logging) {
|
if (pipeline.logging.file_logging) {
|
||||||
const logDir = pipeline.paths.logs;
|
const logDir = pipeline.paths.logs;
|
||||||
const destination = path.join(
|
const destination = path.join(logDir, pipeline.logging.log_file);
|
||||||
logDir,
|
fs.mkdirSync(path.dirname(destination), {recursive: true});
|
||||||
pipeline.logging.log_file,
|
if (!fs.existsSync(destination)) {
|
||||||
);
|
fs.writeFileSync(destination, "");
|
||||||
fs.mkdirSync(path.dirname(destination), { recursive: true });
|
}
|
||||||
if (!fs.existsSync(destination)) {
|
}
|
||||||
fs.writeFileSync(destination, "");
|
}
|
||||||
}
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
resolveConfigPath(env?: string): string {
|
resolveConfigPath(env?: string): string {
|
||||||
const base = locateConfigFile(this.explicitPath);
|
const base = locateConfigFile(this.explicitPath);
|
||||||
return resolveConfigPath(base, env ?? this.defaultEnv);
|
return resolveConfigPath(base, env ?? this.defaultEnv);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+215
-220
@@ -1,7 +1,7 @@
|
|||||||
import path from "node:path";
|
import * as path from "node:path";
|
||||||
|
|
||||||
import { getUnixTime, parse, isMatch, format as formatDate } from "date-fns";
|
import {format as formatDate, getUnixTime, isMatch, parse} from "date-fns";
|
||||||
import { z } from "zod";
|
import {z} from "zod";
|
||||||
|
|
||||||
export const UpdateDirectionSchema = z.enum(["forward", "backward"]);
|
export const UpdateDirectionSchema = z.enum(["forward", "backward"]);
|
||||||
export type UpdateDirection = z.infer<typeof UpdateDirectionSchema>;
|
export type UpdateDirection = z.infer<typeof UpdateDirectionSchema>;
|
||||||
@@ -10,47 +10,47 @@ export const SourceKindSchema = z.enum(["wordpress", "html"]);
|
|||||||
export type SourceKind = z.infer<typeof SourceKindSchema>;
|
export type SourceKind = z.infer<typeof SourceKindSchema>;
|
||||||
|
|
||||||
export const SourceDateSchema = z.object({
|
export const SourceDateSchema = z.object({
|
||||||
format: z.string().default("yyyy-LL-dd HH:mm"),
|
format: z.string().default("yyyy-LL-dd HH:mm"),
|
||||||
pattern: z.string().nullable().optional(),
|
pattern: z.string().nullable().optional(),
|
||||||
replacement: z.string().nullable().optional(),
|
replacement: z.string().nullable().optional(),
|
||||||
});
|
});
|
||||||
export type SourceDate = z.infer<typeof SourceDateSchema>;
|
export type SourceDate = z.infer<typeof SourceDateSchema>;
|
||||||
|
|
||||||
export const SourceSelectorsSchema = z.object({
|
export const SourceSelectorsSchema = z.object({
|
||||||
articles: z.string().optional().nullable(),
|
articles: z.string().optional().nullable(),
|
||||||
article_title: z.string().optional().nullable(),
|
article_title: z.string().optional().nullable(),
|
||||||
article_link: z.string().optional().nullable(),
|
article_link: z.string().optional().nullable(),
|
||||||
article_body: z.string().optional().nullable(),
|
article_body: z.string().optional().nullable(),
|
||||||
article_date: z.string().optional().nullable(),
|
article_date: z.string().optional().nullable(),
|
||||||
article_categories: z.string().optional().nullable(),
|
article_categories: z.string().optional().nullable(),
|
||||||
pagination: z.string().default("ul.pagination > li a"),
|
pagination: z.string().default("ul.pagination > li a"),
|
||||||
});
|
});
|
||||||
export type SourceSelectors = z.infer<typeof SourceSelectorsSchema>;
|
export type SourceSelectors = z.infer<typeof SourceSelectorsSchema>;
|
||||||
|
|
||||||
const BaseSourceSchema = z.object({
|
const BaseSourceSchema = z.object({
|
||||||
source_id: z.string(),
|
source_id: z.string(),
|
||||||
source_url: z.string().url(),
|
source_url: z.url(),
|
||||||
source_date: SourceDateSchema.default(SourceDateSchema.parse({})),
|
source_date: SourceDateSchema.default(SourceDateSchema.parse({})),
|
||||||
source_kind: SourceKindSchema,
|
source_kind: SourceKindSchema,
|
||||||
categories: z.array(z.string()).default([]),
|
categories: z.array(z.string()).default([]),
|
||||||
supports_categories: z.boolean().default(false),
|
supports_categories: z.boolean().default(false),
|
||||||
requires_details: z.boolean().default(false),
|
requires_details: z.boolean().default(false),
|
||||||
requires_rate_limit: z.boolean().default(false),
|
requires_rate_limit: z.boolean().default(false),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const HtmlSourceConfigSchema = BaseSourceSchema.extend({
|
export const HtmlSourceConfigSchema = BaseSourceSchema.extend({
|
||||||
source_kind: z.literal("html"),
|
source_kind: z.literal("html"),
|
||||||
source_selectors: SourceSelectorsSchema.default(
|
source_selectors: SourceSelectorsSchema.default(
|
||||||
SourceSelectorsSchema.parse({}),
|
SourceSelectorsSchema.parse({}),
|
||||||
),
|
),
|
||||||
pagination_template: z.string(),
|
pagination_template: z.string(),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const WordPressSourceConfigSchema = BaseSourceSchema.extend({
|
export const WordPressSourceConfigSchema = BaseSourceSchema.extend({
|
||||||
source_kind: z.literal("wordpress"),
|
source_kind: z.literal("wordpress"),
|
||||||
source_date: SourceDateSchema.default(
|
source_date: SourceDateSchema.default(
|
||||||
SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" }),
|
SourceDateSchema.parse({format: "yyyy-LL-dd'T'HH:mm:ss"}),
|
||||||
),
|
),
|
||||||
});
|
});
|
||||||
|
|
||||||
export type HtmlSourceConfig = z.infer<typeof HtmlSourceConfigSchema>;
|
export type HtmlSourceConfig = z.infer<typeof HtmlSourceConfigSchema>;
|
||||||
@@ -58,279 +58,274 @@ export type WordPressSourceConfig = z.infer<typeof WordPressSourceConfigSchema>;
|
|||||||
export type AnySourceConfig = HtmlSourceConfig | WordPressSourceConfig;
|
export type AnySourceConfig = HtmlSourceConfig | WordPressSourceConfig;
|
||||||
|
|
||||||
export const DateRangeSchema = z
|
export const DateRangeSchema = z
|
||||||
.object({
|
.object({
|
||||||
start: z.number().int(),
|
start: z.number().int(),
|
||||||
end: z.number().int(),
|
end: z.number().int(),
|
||||||
})
|
})
|
||||||
.superRefine((value, ctx) => {
|
.superRefine((value, ctx) => {
|
||||||
if (value.start === 0 || value.end === 0) {
|
if (value.start === 0 || value.end === 0) {
|
||||||
ctx.addIssue({
|
ctx.addIssue({
|
||||||
code: z.ZodIssueCode.custom,
|
code: "custom",
|
||||||
message: "Timestamp cannot be zero",
|
message: "Timestamp cannot be zero",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
if (value.end < value.start) {
|
if (value.end < value.start) {
|
||||||
ctx.addIssue({
|
ctx.addIssue({
|
||||||
code: z.ZodIssueCode.custom,
|
code: "custom",
|
||||||
message: "End timestamp must be greater than or equal to start",
|
message: "End timestamp must be greater than or equal to start",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
export type DateRange = z.infer<typeof DateRangeSchema>;
|
export type DateRange = z.infer<typeof DateRangeSchema>;
|
||||||
|
|
||||||
export const PageRangeSchema = z
|
export const PageRangeSchema = z
|
||||||
.object({
|
.object({
|
||||||
start: z.number().int().min(0),
|
start: z.number().int().min(0),
|
||||||
end: z.number().int().min(0),
|
end: z.number().int().min(0),
|
||||||
})
|
})
|
||||||
.superRefine((value, ctx) => {
|
.superRefine((value, ctx) => {
|
||||||
if (value.end < value.start) {
|
if (value.end < value.start) {
|
||||||
ctx.addIssue({
|
ctx.addIssue({
|
||||||
code: z.ZodIssueCode.custom,
|
code: "custom",
|
||||||
message: "End page must be greater than or equal to start page",
|
message: "End page must be greater than or equal to start page",
|
||||||
});
|
});
|
||||||
}
|
}
|
||||||
});
|
});
|
||||||
|
|
||||||
export type PageRange = z.infer<typeof PageRangeSchema>;
|
export type PageRange = z.infer<typeof PageRangeSchema>;
|
||||||
|
|
||||||
export const PageRangeSpecSchema = z
|
export const PageRangeSpecSchema = z
|
||||||
.string()
|
.string()
|
||||||
.regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end")
|
.regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end")
|
||||||
.transform((spec) => {
|
.transform((spec) => {
|
||||||
const [startText, endText] = spec.split(":");
|
const [startText, endText] = spec.split(":");
|
||||||
return {
|
return {
|
||||||
start: Number.parseInt(startText, 10),
|
start: Number.parseInt(startText, 10),
|
||||||
end: Number.parseInt(endText, 10),
|
end: Number.parseInt(endText, 10),
|
||||||
};
|
};
|
||||||
});
|
});
|
||||||
|
|
||||||
const defaultDateFormat = "yyyy-LL-dd";
|
const defaultDateFormat = "yyyy-LL-dd";
|
||||||
|
|
||||||
export const DateRangeSpecSchema = z
|
export const DateRangeSpecSchema = z
|
||||||
.string()
|
.string()
|
||||||
.regex(/.+:.+/, "Expected start:end format")
|
.regex(/.+:.+/, "Expected start:end format")
|
||||||
.transform((spec) => {
|
.transform((spec) => {
|
||||||
const [startRaw, endRaw] = spec.split(":");
|
const [startRaw, endRaw] = spec.split(":");
|
||||||
return { startRaw, endRaw };
|
return {startRaw, endRaw};
|
||||||
});
|
});
|
||||||
|
|
||||||
const parseDate = (value: string, format: string): Date => {
|
const parseDate = (value: string, format: string): Date => {
|
||||||
if (!isMatch(value, format)) {
|
if (!isMatch(value, format)) {
|
||||||
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
||||||
}
|
}
|
||||||
const parsed = parse(value, format, new Date());
|
const parsed = parse(value, format, new Date());
|
||||||
if (Number.isNaN(parsed.getTime())) {
|
if (Number.isNaN(parsed.getTime())) {
|
||||||
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
throw new Error(`Invalid date '${value}' for format '${format}'`);
|
||||||
}
|
}
|
||||||
return parsed;
|
return parsed;
|
||||||
};
|
};
|
||||||
|
|
||||||
export interface CreateDateRangeOptions {
|
export interface CreateDateRangeOptions {
|
||||||
format?: string;
|
format?: string;
|
||||||
separator?: string;
|
separator?: string;
|
||||||
}
|
}
|
||||||
|
|
||||||
export const createDateRange = (
|
export const createDateRange = (
|
||||||
spec: string,
|
spec: string,
|
||||||
options: CreateDateRangeOptions = {},
|
options: CreateDateRangeOptions = {},
|
||||||
): DateRange => {
|
): DateRange => {
|
||||||
const { format = defaultDateFormat, separator = ":" } = options;
|
const {format = defaultDateFormat, separator = ":"} = options;
|
||||||
if (!separator) {
|
if (!separator) {
|
||||||
throw new Error("Separator cannot be empty");
|
throw new Error("Separator cannot be empty");
|
||||||
}
|
}
|
||||||
|
|
||||||
const normalized = spec.replace(separator, ":");
|
const normalized = spec.replace(separator, ":");
|
||||||
const parsedSpec = DateRangeSpecSchema.parse(normalized);
|
const parsedSpec = DateRangeSpecSchema.parse(normalized);
|
||||||
|
|
||||||
const startDate = parseDate(parsedSpec.startRaw, format);
|
const startDate = parseDate(parsedSpec.startRaw, format);
|
||||||
const endDate = parseDate(parsedSpec.endRaw, format);
|
const endDate = parseDate(parsedSpec.endRaw, format);
|
||||||
|
|
||||||
const range = {
|
const range = {
|
||||||
start: getUnixTime(startDate),
|
start: getUnixTime(startDate),
|
||||||
end: getUnixTime(endDate),
|
end: getUnixTime(endDate),
|
||||||
};
|
};
|
||||||
|
|
||||||
return DateRangeSchema.parse(range);
|
return DateRangeSchema.parse(range);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const formatDateRange = (
|
export const formatDateRange = (
|
||||||
range: DateRange,
|
range: DateRange,
|
||||||
fmt = defaultDateFormat,
|
fmt = defaultDateFormat,
|
||||||
): string => {
|
): string => {
|
||||||
const start = formatDate(new Date(range.start * 1000), fmt);
|
const start = formatDate(new Date(range.start * 1000), fmt);
|
||||||
const end = formatDate(new Date(range.end * 1000), fmt);
|
const end = formatDate(new Date(range.end * 1000), fmt);
|
||||||
return `${start}:${end}`;
|
return `${start}:${end}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const isTimestampInRange = (
|
export const isTimestampInRange = (
|
||||||
range: DateRange,
|
range: DateRange,
|
||||||
timestamp: number,
|
timestamp: number,
|
||||||
): boolean => {
|
): boolean => {
|
||||||
return range.start <= timestamp && timestamp <= range.end;
|
return range.start <= timestamp && timestamp <= range.end;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const ProjectPathsSchema = z.object({
|
export const ProjectPathsSchema = z.object({
|
||||||
root: z.string(),
|
root: z.string(),
|
||||||
data: z.string(),
|
data: z.string(),
|
||||||
logs: z.string(),
|
logs: z.string(),
|
||||||
configs: z.string(),
|
configs: z.string(),
|
||||||
});
|
});
|
||||||
export type ProjectPaths = z.infer<typeof ProjectPathsSchema>;
|
export type ProjectPaths = z.infer<typeof ProjectPathsSchema>;
|
||||||
|
|
||||||
export const resolveProjectPaths = (rootDir: string): ProjectPaths => {
|
export const resolveProjectPaths = (rootDir: string): ProjectPaths => {
|
||||||
return ProjectPathsSchema.parse({
|
return ProjectPathsSchema.parse({
|
||||||
root: rootDir,
|
root: rootDir,
|
||||||
data: path.join(rootDir, "data", "dataset"),
|
data: path.join(rootDir, "data", "dataset"),
|
||||||
logs: path.join(rootDir, "data", "logs"),
|
logs: path.join(rootDir, "data", "logs"),
|
||||||
configs: path.join(rootDir, "config"),
|
configs: path.join(rootDir, "config"),
|
||||||
});
|
});
|
||||||
};
|
};
|
||||||
|
|
||||||
export const LoggingConfigSchema = z.object({
|
export const LoggingConfigSchema = z.object({
|
||||||
level: z.string().default("INFO"),
|
level: z.string().default("INFO"),
|
||||||
format: z
|
format: z
|
||||||
.string()
|
.string()
|
||||||
.default("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
|
.default("%(asctime)s - %(name)s - %(levelname)s - %(message)s"),
|
||||||
console_logging: z.boolean().default(true),
|
console_logging: z.boolean().default(true),
|
||||||
file_logging: z.boolean().default(false),
|
file_logging: z.boolean().default(false),
|
||||||
log_file: z.string().default("crawler.log"),
|
log_file: z.string().default("crawler.log"),
|
||||||
max_log_size: z
|
max_log_size: z
|
||||||
.number()
|
.number()
|
||||||
.int()
|
.int()
|
||||||
.positive()
|
.positive()
|
||||||
.default(10 * 1024 * 1024),
|
.default(10 * 1024 * 1024),
|
||||||
backup_count: z.number().int().nonnegative().default(5),
|
backup_count: z.number().int().nonnegative().default(5),
|
||||||
});
|
});
|
||||||
export type LoggingConfig = z.infer<typeof LoggingConfigSchema>;
|
export type LoggingConfig = z.infer<typeof LoggingConfigSchema>;
|
||||||
|
|
||||||
export const ClientConfigSchema = z.object({
|
export const ClientConfigSchema = z.object({
|
||||||
timeout: z.number().positive().default(20),
|
timeout: z.number().positive().default(20),
|
||||||
user_agent: z
|
user_agent: z
|
||||||
.string()
|
.string()
|
||||||
.default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
|
.default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
|
||||||
follow_redirects: z.boolean().default(true),
|
follow_redirects: z.boolean().default(true),
|
||||||
verify_ssl: z.boolean().default(true),
|
verify_ssl: z.boolean().default(true),
|
||||||
rotate: z.boolean().default(true),
|
rotate: z.boolean().default(true),
|
||||||
max_retries: z.number().int().nonnegative().default(3),
|
max_retries: z.number().int().nonnegative().default(3),
|
||||||
backoff_initial: z.number().nonnegative().default(1),
|
backoff_initial: z.number().nonnegative().default(1),
|
||||||
backoff_multiplier: z.number().positive().default(2),
|
backoff_multiplier: z.number().positive().default(2),
|
||||||
backoff_max: z.number().nonnegative().default(30),
|
backoff_max: z.number().nonnegative().default(30),
|
||||||
respect_retry_after: z.boolean().default(true),
|
respect_retry_after: z.boolean().default(true),
|
||||||
});
|
});
|
||||||
|
|
||||||
export const CrawlerConfigSchema = z.object({
|
export const CrawlerConfigSchema = z.object({
|
||||||
source: z
|
source: z
|
||||||
.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema])
|
.union([HtmlSourceConfigSchema, WordPressSourceConfigSchema])
|
||||||
.optional(),
|
.optional(),
|
||||||
page_range: PageRangeSchema.optional(),
|
page_range: PageRangeSchema.optional(),
|
||||||
date_range: DateRangeSchema.optional(),
|
date_range: DateRangeSchema.optional(),
|
||||||
category: z.string().optional(),
|
category: z.string().optional(),
|
||||||
notify: z.boolean().default(false),
|
notify: z.boolean().default(false),
|
||||||
is_update: z.boolean().default(false),
|
is_update: z.boolean().default(false),
|
||||||
use_multi_threading: z.boolean().default(false),
|
use_multi_threading: z.boolean().default(false),
|
||||||
max_workers: z.number().int().positive().default(5),
|
max_workers: z.number().int().positive().default(5),
|
||||||
direction: UpdateDirectionSchema.default("forward"),
|
direction: UpdateDirectionSchema.default("forward"),
|
||||||
});
|
});
|
||||||
|
|
||||||
export type ClientConfig = z.infer<typeof ClientConfigSchema>;
|
export type ClientConfig = z.infer<typeof ClientConfigSchema>;
|
||||||
export type CrawlerConfig = z.infer<typeof CrawlerConfigSchema> & {
|
export type CrawlerConfig = z.infer<typeof CrawlerConfigSchema> & {
|
||||||
source?: AnySourceConfig;
|
source?: AnySourceConfig;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const FetchConfigSchema = z.object({
|
export const FetchConfigSchema = z.object({
|
||||||
client: ClientConfigSchema.default(ClientConfigSchema.parse({})),
|
client: ClientConfigSchema.default(ClientConfigSchema.parse({})),
|
||||||
crawler: CrawlerConfigSchema.default(CrawlerConfigSchema.parse({})),
|
crawler: CrawlerConfigSchema.default(CrawlerConfigSchema.parse({})),
|
||||||
});
|
});
|
||||||
export type FetchConfig = z.infer<typeof FetchConfigSchema>;
|
export type FetchConfig = z.infer<typeof FetchConfigSchema>;
|
||||||
|
|
||||||
const SourcesConfigSchema = z.object({
|
const SourcesConfigSchema = z.object({
|
||||||
html: z.array(HtmlSourceConfigSchema).default([]),
|
html: z.array(HtmlSourceConfigSchema).default([]),
|
||||||
wordpress: z.array(WordPressSourceConfigSchema).default([]),
|
wordpress: z.array(WordPressSourceConfigSchema).default([]),
|
||||||
});
|
});
|
||||||
|
|
||||||
export type SourcesConfig = z.infer<typeof SourcesConfigSchema> & {
|
export type SourcesConfig = z.infer<typeof SourcesConfigSchema> & {
|
||||||
find: (sourceId: string) => AnySourceConfig | undefined;
|
find: (sourceId: string) => AnySourceConfig | undefined;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const createSourcesConfig = (input: unknown): SourcesConfig => {
|
export const createSourcesConfig = (input: unknown): SourcesConfig => {
|
||||||
const parsed = SourcesConfigSchema.parse(input);
|
const parsed = SourcesConfigSchema.parse(input);
|
||||||
const resolver = (sourceId: string) =>
|
const resolver = (sourceId: string) =>
|
||||||
[...parsed.html, ...parsed.wordpress].find(
|
[...parsed.html, ...parsed.wordpress].find(
|
||||||
(source) => source.source_id === sourceId,
|
(source) => source.source_id === sourceId,
|
||||||
);
|
);
|
||||||
return Object.assign({ find: resolver }, parsed);
|
return Object.assign({find: resolver}, parsed);
|
||||||
};
|
};
|
||||||
|
|
||||||
export const PipelineConfigSchema = z.object({
|
export const PipelineConfigSchema = z.object({
|
||||||
paths: ProjectPathsSchema.default(resolveProjectPaths(process.cwd())),
|
paths: ProjectPathsSchema.default(resolveProjectPaths(process.cwd())),
|
||||||
logging: LoggingConfigSchema.default(LoggingConfigSchema.parse({})),
|
logging: LoggingConfigSchema.default(LoggingConfigSchema.parse({})),
|
||||||
fetch: FetchConfigSchema.default(FetchConfigSchema.parse({})),
|
fetch: FetchConfigSchema.default(FetchConfigSchema.parse({})),
|
||||||
sources: z
|
sources: z
|
||||||
.union([SourcesConfigSchema, z.undefined()])
|
.union([SourcesConfigSchema, z.undefined()])
|
||||||
.transform((value) => createSourcesConfig(value ?? {})),
|
.transform((value) => createSourcesConfig(value ?? {})),
|
||||||
});
|
});
|
||||||
|
export type PipelineConfig = z.infer<typeof PipelineConfigSchema>
|
||||||
export type PipelineConfig = z.infer<typeof PipelineConfigSchema> & {
|
|
||||||
sources: SourcesConfig;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const mergePipelineConfig = (
|
export const mergePipelineConfig = (
|
||||||
base: PipelineConfig,
|
base: PipelineConfig,
|
||||||
overrides: Partial<PipelineConfig>,
|
overrides: Partial<PipelineConfig>,
|
||||||
): PipelineConfig => {
|
): PipelineConfig => {
|
||||||
const paths = overrides.paths ?? base.paths;
|
const paths = overrides.paths ?? base.paths;
|
||||||
const logging = { ...base.logging, ...(overrides.logging ?? {}) };
|
const logging = {...base.logging, ...(overrides.logging ?? {})};
|
||||||
const fetch = {
|
const fetch = {
|
||||||
client: { ...base.fetch.client, ...(overrides.fetch?.client ?? {}) },
|
client: {...base.fetch.client, ...(overrides.fetch?.client ?? {})},
|
||||||
crawler: { ...base.fetch.crawler, ...(overrides.fetch?.crawler ?? {}) },
|
crawler: {...base.fetch.crawler, ...(overrides.fetch?.crawler ?? {})},
|
||||||
};
|
};
|
||||||
|
|
||||||
const sources = createSourcesConfig({
|
const sources = createSourcesConfig({
|
||||||
html: overrides.sources?.html ?? base.sources.html,
|
html: overrides.sources?.html ?? base.sources.html,
|
||||||
wordpress: overrides.sources?.wordpress ?? base.sources.wordpress,
|
wordpress: overrides.sources?.wordpress ?? base.sources.wordpress,
|
||||||
});
|
});
|
||||||
|
|
||||||
return {
|
return {
|
||||||
paths,
|
paths,
|
||||||
logging,
|
logging,
|
||||||
fetch,
|
fetch,
|
||||||
sources,
|
sources,
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export const resolveConfigPath = (basePath: string, env?: string): string => {
|
export const resolveConfigPath = (basePath: string, env?: string): string => {
|
||||||
if (!env || env === "development") {
|
if (!env || env === "development") {
|
||||||
return basePath;
|
return basePath;
|
||||||
}
|
}
|
||||||
|
|
||||||
const ext = path.extname(basePath);
|
const ext = path.extname(basePath);
|
||||||
const withoutExt = basePath.slice(0, basePath.length - ext.length);
|
const withoutExt = basePath.slice(0, basePath.length - ext.length);
|
||||||
return `${withoutExt}.${env}${ext}`;
|
return `${withoutExt}.${env}${ext}`;
|
||||||
};
|
};
|
||||||
|
|
||||||
export const schemaToJSON = <T extends z.ZodTypeAny>(schema: T): unknown => {
|
export const schemaToJSON = <T extends z.ZodTypeAny>(schema: T): unknown => {
|
||||||
const candidate = schema as unknown as { toJSON?: () => unknown };
|
const toJSONSchema = (z as any).toJSONSchema as
|
||||||
if (typeof candidate.toJSON === "function") {
|
| ((s: z.ZodTypeAny, opts?: Record<string, unknown>) => unknown)
|
||||||
return candidate.toJSON();
|
| undefined;
|
||||||
}
|
|
||||||
|
|
||||||
const typeName = (schema as { _def?: { typeName?: z.ZodFirstPartyTypeKind } })._def
|
if (typeof toJSONSchema === "function") {
|
||||||
?.typeName;
|
try {
|
||||||
|
// target can be "draft-2020-12" | "draft-7" | "draft-4" | "openapi-3.0"
|
||||||
|
return toJSONSchema(schema, {target: "draft-2020-12", unrepresentable: "any"});
|
||||||
|
} catch {
|
||||||
|
// fall through to minimal mapping
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
switch (typeName) {
|
if (schema instanceof z.ZodObject) return {type: "object"};
|
||||||
case z.ZodFirstPartyTypeKind.ZodObject:
|
if (schema instanceof z.ZodArray) return {type: "array"};
|
||||||
return { type: "object" };
|
if (schema instanceof z.ZodString) return {type: "string"};
|
||||||
case z.ZodFirstPartyTypeKind.ZodArray:
|
if (schema instanceof z.ZodNumber) return {type: "number"};
|
||||||
return { type: "array" };
|
if (schema instanceof z.ZodBoolean) return {type: "boolean"};
|
||||||
case z.ZodFirstPartyTypeKind.ZodString:
|
|
||||||
return { type: "string" };
|
return {type: "unknown"};
|
||||||
case z.ZodFirstPartyTypeKind.ZodNumber:
|
|
||||||
return { type: "number" };
|
|
||||||
case z.ZodFirstPartyTypeKind.ZodBoolean:
|
|
||||||
return { type: "boolean" };
|
|
||||||
default:
|
|
||||||
return { type: "unknown" };
|
|
||||||
}
|
|
||||||
};
|
};
|
||||||
@@ -1,88 +1,88 @@
|
|||||||
import { parseArgs } from "node:util";
|
import {parseArgs} from "node:util";
|
||||||
|
|
||||||
import { logger } from "@basango/logger";
|
import {logger} from "@basango/logger";
|
||||||
|
|
||||||
import { PipelineConfigManager } from "@crawler/config";
|
import {PipelineConfigManager} from "@crawler/config";
|
||||||
import { scheduleAsyncCrawl } from "@crawler/services/crawler";
|
import {createQueueSettings} from "@crawler/services/async/queue";
|
||||||
import { createQueueSettings } from "@crawler/services/crawler/async/queue";
|
import {scheduleAsyncCrawl} from "@crawler/services/async/tasks";
|
||||||
|
|
||||||
interface QueueCliOptions {
|
interface QueueCliOptions {
|
||||||
"source-id"?: string;
|
"source-id"?: string;
|
||||||
env: string;
|
env: string;
|
||||||
"page-range"?: string;
|
"page-range"?: string;
|
||||||
"date-range"?: string;
|
"date-range"?: string;
|
||||||
category?: string;
|
category?: string;
|
||||||
"redis-url"?: string;
|
"redis-url"?: string;
|
||||||
help?: boolean;
|
help?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const usage = `Usage: bun run src/scripts/queue.ts -- --source-id <id> [options]\n\nOptions:\n --env <env> Environment to load (default: development)\n --page-range <range> Optional page range filter (e.g. 1:5)\n --date-range <range> Optional date range filter (e.g. 2024-01-01:2024-01-31)\n --category <slug> Optional category to crawl\n --redis-url <url> Override Redis connection URL\n -h, --help Show this message`;
|
const usage = `Usage: bun run src/scripts/queue.ts -- --source-id <id> [options]\n\nOptions:\n --env <env> Environment to load (default: development)\n --page-range <range> Optional page range filter (e.g. 1:5)\n --date-range <range> Optional date range filter (e.g. 2024-01-01:2024-01-31)\n --category <slug> Optional category to crawl\n --redis-url <url> Override Redis connection URL\n -h, --help Show this message`;
|
||||||
|
|
||||||
const parseCliArgs = (): QueueCliOptions => {
|
const parseCliArgs = (): QueueCliOptions => {
|
||||||
const { values } = parseArgs({
|
const {values} = parseArgs({
|
||||||
options: {
|
options: {
|
||||||
"source-id": { type: "string" },
|
"source-id": {type: "string"},
|
||||||
env: { type: "string", default: "development" },
|
env: {type: "string", default: "development"},
|
||||||
"page-range": { type: "string" },
|
"page-range": {type: "string"},
|
||||||
"date-range": { type: "string" },
|
"date-range": {type: "string"},
|
||||||
category: { type: "string" },
|
category: {type: "string"},
|
||||||
"redis-url": { type: "string" },
|
"redis-url": {type: "string"},
|
||||||
help: { type: "boolean", short: "h" },
|
help: {type: "boolean", short: "h"},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return values as QueueCliOptions;
|
return values as QueueCliOptions;
|
||||||
};
|
};
|
||||||
|
|
||||||
const main = async (): Promise<void> => {
|
const main = async (): Promise<void> => {
|
||||||
const options = parseCliArgs();
|
const options = parseCliArgs();
|
||||||
|
|
||||||
if (options.help || !options["source-id"]) {
|
if (options.help || !options["source-id"]) {
|
||||||
console.log(usage);
|
console.log(usage);
|
||||||
if (!options["source-id"]) {
|
if (!options["source-id"]) {
|
||||||
process.exitCode = 1;
|
process.exitCode = 1;
|
||||||
}
|
}
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const env = options.env ?? "development";
|
const env = options.env ?? "development";
|
||||||
const manager = new PipelineConfigManager({ env });
|
const manager = new PipelineConfigManager({env});
|
||||||
const config = manager.ensureDirectories();
|
const config = manager.ensureDirectories();
|
||||||
manager.setupLogging(config);
|
manager.setupLogging(config);
|
||||||
|
|
||||||
const settings = options["redis-url"]
|
const settings = options["redis-url"]
|
||||||
? createQueueSettings({ redis_url: options["redis-url"] })
|
? createQueueSettings({redis_url: options["redis-url"]})
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
try {
|
try {
|
||||||
const jobId = await scheduleAsyncCrawl({
|
const jobId = await scheduleAsyncCrawl({
|
||||||
sourceId: options["source-id"],
|
sourceId: options["source-id"],
|
||||||
env,
|
env,
|
||||||
pageRange: options["page-range"] ?? null,
|
pageRange: options["page-range"] ?? null,
|
||||||
dateRange: options["date-range"] ?? null,
|
dateRange: options["date-range"] ?? null,
|
||||||
category: options.category ?? null,
|
category: options.category ?? null,
|
||||||
settings,
|
settings,
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
jobId,
|
jobId,
|
||||||
sourceId: options["source-id"],
|
sourceId: options["source-id"],
|
||||||
env,
|
env,
|
||||||
},
|
},
|
||||||
"Scheduled asynchronous crawl job",
|
"Scheduled asynchronous crawl job",
|
||||||
);
|
);
|
||||||
console.log(
|
console.log(
|
||||||
`Scheduled async crawl job ${jobId} for source '${options["source-id"]}' (env=${env})`,
|
`Scheduled async crawl job ${jobId} for source '${options["source-id"]}' (env=${env})`,
|
||||||
);
|
);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(
|
logger.error(
|
||||||
error instanceof Error ? error : { error },
|
error instanceof Error ? error : {error},
|
||||||
"Failed to schedule crawl job",
|
"Failed to schedule crawl job",
|
||||||
);
|
);
|
||||||
console.error(`Failed to schedule crawl job: ${(error as Error).message}`);
|
console.error(`Failed to schedule crawl job: ${(error as Error).message}`);
|
||||||
process.exitCode = 1;
|
process.exitCode = 1;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
void main();
|
void main();
|
||||||
|
|||||||
@@ -1,112 +1,112 @@
|
|||||||
import { parseArgs } from "node:util";
|
import {parseArgs} from "node:util";
|
||||||
|
|
||||||
import { logger } from "@basango/logger";
|
import {logger} from "@basango/logger";
|
||||||
|
|
||||||
import { PipelineConfigManager } from "@crawler/config";
|
import {PipelineConfigManager} from "@crawler/config";
|
||||||
import { createQueueManager, createQueueSettings } from "@crawler/services/crawler/async/queue";
|
import {createQueueManager, createQueueSettings,} from "@crawler/services/async/queue";
|
||||||
import { startWorker } from "@crawler/services/crawler/async/worker";
|
import {startWorker} from "@crawler/services/async/worker";
|
||||||
|
|
||||||
interface WorkerCliOptions {
|
interface WorkerCliOptions {
|
||||||
env: string;
|
env: string;
|
||||||
queue?: string[];
|
queue?: string[];
|
||||||
concurrency?: string;
|
concurrency?: string;
|
||||||
"redis-url"?: string;
|
"redis-url"?: string;
|
||||||
help?: boolean;
|
help?: boolean;
|
||||||
}
|
}
|
||||||
|
|
||||||
const usage = `Usage: bun run src/scripts/worker.ts [options]\n\nOptions:\n --env <env> Environment to load (default: development)\n -q, --queue <name> Queue name to listen on (repeatable)\n --concurrency <number> Number of concurrent jobs per worker\n --redis-url <url> Override Redis connection URL\n -h, --help Show this message`;
|
const usage = `Usage: bun run src/scripts/worker.ts [options]\n\nOptions:\n --env <env> Environment to load (default: development)\n -q, --queue <name> Queue name to listen on (repeatable)\n --concurrency <number> Number of concurrent jobs per worker\n --redis-url <url> Override Redis connection URL\n -h, --help Show this message`;
|
||||||
|
|
||||||
const parseCliArgs = (): WorkerCliOptions => {
|
const parseCliArgs = (): WorkerCliOptions => {
|
||||||
const { values } = parseArgs({
|
const {values} = parseArgs({
|
||||||
options: {
|
options: {
|
||||||
env: { type: "string", default: "development" },
|
env: {type: "string", default: "development"},
|
||||||
queue: { type: "string", multiple: true, short: "q" },
|
queue: {type: "string", multiple: true, short: "q"},
|
||||||
concurrency: { type: "string" },
|
concurrency: {type: "string"},
|
||||||
"redis-url": { type: "string" },
|
"redis-url": {type: "string"},
|
||||||
help: { type: "boolean", short: "h" },
|
help: {type: "boolean", short: "h"},
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|
||||||
return values as WorkerCliOptions;
|
return values as WorkerCliOptions;
|
||||||
};
|
};
|
||||||
|
|
||||||
const parseConcurrency = (value?: string): number | undefined => {
|
const parseConcurrency = (value?: string): number | undefined => {
|
||||||
if (!value) {
|
if (!value) {
|
||||||
return undefined;
|
return undefined;
|
||||||
}
|
}
|
||||||
|
|
||||||
const parsed = Number.parseInt(value, 10);
|
const parsed = Number.parseInt(value, 10);
|
||||||
if (Number.isNaN(parsed) || parsed <= 0) {
|
if (Number.isNaN(parsed) || parsed <= 0) {
|
||||||
throw new Error(`Invalid concurrency value: ${value}`);
|
throw new Error(`Invalid concurrency value: ${value}`);
|
||||||
}
|
}
|
||||||
|
|
||||||
return parsed;
|
return parsed;
|
||||||
};
|
};
|
||||||
|
|
||||||
const main = async (): Promise<void> => {
|
const main = async (): Promise<void> => {
|
||||||
const options = parseCliArgs();
|
const options = parseCliArgs();
|
||||||
|
|
||||||
if (options.help) {
|
if (options.help) {
|
||||||
console.log(usage);
|
console.log(usage);
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
|
|
||||||
const env = options.env ?? "development";
|
const env = options.env ?? "development";
|
||||||
const manager = new PipelineConfigManager({ env });
|
const manager = new PipelineConfigManager({env});
|
||||||
const config = manager.ensureDirectories();
|
const config = manager.ensureDirectories();
|
||||||
manager.setupLogging(config);
|
manager.setupLogging(config);
|
||||||
|
|
||||||
let concurrency: number | undefined;
|
let concurrency: number | undefined;
|
||||||
try {
|
try {
|
||||||
concurrency = parseConcurrency(options.concurrency);
|
concurrency = parseConcurrency(options.concurrency);
|
||||||
} catch (error) {
|
} catch (error) {
|
||||||
logger.error(
|
logger.error(
|
||||||
error instanceof Error ? error : { error },
|
error instanceof Error ? error : {error},
|
||||||
"Invalid concurrency value provided",
|
"Invalid concurrency value provided",
|
||||||
);
|
);
|
||||||
process.exitCode = 1;
|
process.exitCode = 1;
|
||||||
return;
|
return;
|
||||||
}
|
}
|
||||||
const settings = options["redis-url"]
|
const settings = options["redis-url"]
|
||||||
? createQueueSettings({ redis_url: options["redis-url"] })
|
? createQueueSettings({redis_url: options["redis-url"]})
|
||||||
: undefined;
|
: undefined;
|
||||||
const queueManager = createQueueManager({ settings });
|
const queueManager = createQueueManager({settings});
|
||||||
|
|
||||||
const queueNames = options.queue?.length
|
const queueNames = options.queue?.length
|
||||||
? options.queue.map((name) => queueManager.queueName(name))
|
? options.queue.map((name) => queueManager.queueName(name))
|
||||||
: undefined;
|
: undefined;
|
||||||
|
|
||||||
const handle = startWorker({
|
const handle = startWorker({
|
||||||
queueManager,
|
queueManager,
|
||||||
queueNames,
|
queueNames,
|
||||||
concurrency,
|
concurrency,
|
||||||
});
|
});
|
||||||
|
|
||||||
const shutdown = async (signal: NodeJS.Signals) => {
|
const shutdown = async (signal: NodeJS.Signals) => {
|
||||||
logger.info({ signal }, "Received shutdown signal, draining workers");
|
logger.info({signal}, "Received shutdown signal, draining workers");
|
||||||
try {
|
try {
|
||||||
await handle.close();
|
await handle.close();
|
||||||
} finally {
|
} finally {
|
||||||
await queueManager.close();
|
await queueManager.close();
|
||||||
process.exit(0);
|
process.exit(0);
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
process.once("SIGINT", (signal) => {
|
process.once("SIGINT", (signal) => {
|
||||||
void shutdown(signal);
|
void shutdown(signal);
|
||||||
});
|
});
|
||||||
process.once("SIGTERM", (signal) => {
|
process.once("SIGTERM", (signal) => {
|
||||||
void shutdown(signal);
|
void shutdown(signal);
|
||||||
});
|
});
|
||||||
|
|
||||||
logger.info(
|
logger.info(
|
||||||
{
|
{
|
||||||
env,
|
env,
|
||||||
queueNames: queueNames ?? queueManager.iterQueueNames(),
|
queueNames: queueNames ?? queueManager.iterQueueNames(),
|
||||||
concurrency: concurrency ?? "default",
|
concurrency: concurrency ?? "default",
|
||||||
},
|
},
|
||||||
"Crawler workers started",
|
"Crawler workers started",
|
||||||
);
|
);
|
||||||
};
|
};
|
||||||
|
|
||||||
void main();
|
void main();
|
||||||
|
|||||||
@@ -0,0 +1,142 @@
|
|||||||
|
import { randomUUID } from "node:crypto";
|
||||||
|
|
||||||
|
import IORedis from "ioredis";
|
||||||
|
import { JobsOptions, Queue, QueueOptions } from "bullmq";
|
||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
import {
|
||||||
|
ArticleTaskPayload,
|
||||||
|
ArticleTaskPayloadSchema,
|
||||||
|
ListingTaskPayload,
|
||||||
|
ListingTaskPayloadSchema,
|
||||||
|
ProcessedTaskPayload,
|
||||||
|
ProcessedTaskPayloadSchema,
|
||||||
|
} from "./schemas";
|
||||||
|
import { parseRedisUrl } from "@crawler/utils";
|
||||||
|
|
||||||
|
const QueueSettingsSchema = z.object({
|
||||||
|
redis_url: z
|
||||||
|
.string()
|
||||||
|
.default(process.env.BASANGO_REDIS_URL ?? "redis://localhost:6379/0"),
|
||||||
|
prefix: z.string().default(process.env.BASANGO_QUEUE_PREFIX ?? "crawler"),
|
||||||
|
default_timeout: z
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.positive()
|
||||||
|
.default(Number(process.env.BASANGO_QUEUE_TIMEOUT ?? 600)),
|
||||||
|
result_ttl: z
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.nonnegative()
|
||||||
|
.default(Number(process.env.BASANGO_QUEUE_RESULT_TTL ?? 3600)),
|
||||||
|
failure_ttl: z
|
||||||
|
.number()
|
||||||
|
.int()
|
||||||
|
.nonnegative()
|
||||||
|
.default(Number(process.env.BASANGO_QUEUE_FAILURE_TTL ?? 3600)),
|
||||||
|
listing_queue: z.string().default("listing"),
|
||||||
|
article_queue: z.string().default("articles"),
|
||||||
|
processed_queue: z.string().default("processed"),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type QueueSettingsInput = z.input<typeof QueueSettingsSchema>;
|
||||||
|
export type QueueSettings = z.output<typeof QueueSettingsSchema>;
|
||||||
|
|
||||||
|
export const createQueueSettings = (
|
||||||
|
input?: QueueSettingsInput,
|
||||||
|
): QueueSettings => QueueSettingsSchema.parse(input ?? {});
|
||||||
|
|
||||||
|
export interface QueueBackend<T = unknown> {
|
||||||
|
add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export type QueueFactory = (
|
||||||
|
queueName: string,
|
||||||
|
settings: QueueSettings,
|
||||||
|
connection?: IORedis,
|
||||||
|
) => QueueBackend;
|
||||||
|
|
||||||
|
const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => {
|
||||||
|
const redisConnection =
|
||||||
|
connection ??
|
||||||
|
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
||||||
|
const options: QueueOptions = {
|
||||||
|
connection: redisConnection,
|
||||||
|
prefix: settings.prefix,
|
||||||
|
};
|
||||||
|
|
||||||
|
const queue = new Queue(queueName, options);
|
||||||
|
return {
|
||||||
|
add: async (name, data, opts) => {
|
||||||
|
const job = await queue.add(name, data, {
|
||||||
|
removeOnComplete: settings.result_ttl === 0 ? true : undefined,
|
||||||
|
removeOnFail: settings.failure_ttl === 0 ? true : undefined,
|
||||||
|
//timeout: settings.default_timeout * 1000,
|
||||||
|
...opts,
|
||||||
|
});
|
||||||
|
return { id: job.id ?? randomUUID() };
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface CreateQueueManagerOptions {
|
||||||
|
settings?: QueueSettings | QueueSettingsInput;
|
||||||
|
queueFactory?: QueueFactory;
|
||||||
|
connection?: IORedis;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface QueueManager {
|
||||||
|
readonly settings: QueueSettings;
|
||||||
|
readonly connection: IORedis;
|
||||||
|
enqueueListing: (payload: ListingTaskPayload) => Promise<{ id: string }>;
|
||||||
|
enqueueArticle: (payload: ArticleTaskPayload) => Promise<{ id: string }>;
|
||||||
|
enqueueProcessed: (payload: ProcessedTaskPayload) => Promise<{ id: string }>;
|
||||||
|
iterQueueNames: () => string[];
|
||||||
|
queueName: (suffix: string) => string;
|
||||||
|
close: () => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const createQueueManager = (
|
||||||
|
options: CreateQueueManagerOptions = {},
|
||||||
|
): QueueManager => {
|
||||||
|
const settings = createQueueSettings(
|
||||||
|
options.settings as QueueSettingsInput | undefined,
|
||||||
|
);
|
||||||
|
|
||||||
|
const connection =
|
||||||
|
options.connection ??
|
||||||
|
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
||||||
|
const factory = options.queueFactory ?? defaultQueueFactory;
|
||||||
|
|
||||||
|
const ensureQueue = (queueName: string) =>
|
||||||
|
factory(queueName, settings, connection);
|
||||||
|
|
||||||
|
return {
|
||||||
|
settings,
|
||||||
|
connection,
|
||||||
|
enqueueListing: (payload) => {
|
||||||
|
const data = ListingTaskPayloadSchema.parse(payload);
|
||||||
|
const queue = ensureQueue(settings.listing_queue);
|
||||||
|
return queue.add("collect_listing", data);
|
||||||
|
},
|
||||||
|
enqueueArticle: (payload) => {
|
||||||
|
const data = ArticleTaskPayloadSchema.parse(payload);
|
||||||
|
const queue = ensureQueue(settings.article_queue);
|
||||||
|
return queue.add("collect_article", data);
|
||||||
|
},
|
||||||
|
enqueueProcessed: (payload) => {
|
||||||
|
const data = ProcessedTaskPayloadSchema.parse(payload);
|
||||||
|
const queue = ensureQueue(settings.processed_queue);
|
||||||
|
return queue.add("forward_for_processing", data);
|
||||||
|
},
|
||||||
|
iterQueueNames: () => [
|
||||||
|
`${settings.prefix}:${settings.listing_queue}`,
|
||||||
|
`${settings.prefix}:${settings.article_queue}`,
|
||||||
|
`${settings.prefix}:${settings.processed_queue}`,
|
||||||
|
],
|
||||||
|
queueName: (suffix: string) => `${settings.prefix}:${suffix}`,
|
||||||
|
close: async () => {
|
||||||
|
await connection.quit();
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -0,0 +1,41 @@
|
|||||||
|
import { z } from "zod";
|
||||||
|
|
||||||
|
import {
|
||||||
|
AnySourceConfig,
|
||||||
|
DateRangeSchema,
|
||||||
|
PageRangeSchema,
|
||||||
|
} from "@crawler/schema";
|
||||||
|
|
||||||
|
export const ListingTaskPayloadSchema = z.object({
|
||||||
|
source_id: z.string(),
|
||||||
|
env: z.string().default("development"),
|
||||||
|
page_range: z.string().optional().nullable(),
|
||||||
|
date_range: z.string().optional().nullable(),
|
||||||
|
category: z.string().optional().nullable(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ListingTaskPayload = z.infer<typeof ListingTaskPayloadSchema>;
|
||||||
|
|
||||||
|
export const ArticleTaskPayloadSchema = z.object({
|
||||||
|
source_id: z.string(),
|
||||||
|
env: z.string().default("development"),
|
||||||
|
url: z.url(),
|
||||||
|
page: z.number().int().nonnegative().optional(),
|
||||||
|
page_range: PageRangeSchema.optional().nullable(),
|
||||||
|
date_range: DateRangeSchema.optional().nullable(),
|
||||||
|
category: z.string().optional().nullable(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ArticleTaskPayload = z.infer<typeof ArticleTaskPayloadSchema>;
|
||||||
|
|
||||||
|
export const ProcessedTaskPayloadSchema = z.object({
|
||||||
|
source_id: z.string(),
|
||||||
|
env: z.string().default("development"),
|
||||||
|
article: z.any(),
|
||||||
|
});
|
||||||
|
|
||||||
|
export type ProcessedTaskPayload = z.infer<typeof ProcessedTaskPayloadSchema>;
|
||||||
|
|
||||||
|
export interface ListingContext {
|
||||||
|
source: AnySourceConfig;
|
||||||
|
}
|
||||||
@@ -0,0 +1,171 @@
|
|||||||
|
import { logger } from "@basango/logger";
|
||||||
|
|
||||||
|
import {
|
||||||
|
ArticleTaskPayload,
|
||||||
|
ArticleTaskPayloadSchema,
|
||||||
|
ListingTaskPayload,
|
||||||
|
ListingTaskPayloadSchema,
|
||||||
|
ProcessedTaskPayload,
|
||||||
|
ProcessedTaskPayloadSchema,
|
||||||
|
} from "./schemas";
|
||||||
|
import {
|
||||||
|
createQueueManager,
|
||||||
|
QueueManager,
|
||||||
|
QueueSettings,
|
||||||
|
QueueSettingsInput,
|
||||||
|
} from "./queue";
|
||||||
|
|
||||||
|
export interface CrawlerTaskHandlers {
|
||||||
|
collectListing: (payload: ListingTaskPayload) => Promise<number> | number;
|
||||||
|
collectArticle: (payload: ArticleTaskPayload) => Promise<unknown> | unknown;
|
||||||
|
forwardForProcessing: (
|
||||||
|
payload: ProcessedTaskPayload,
|
||||||
|
) => Promise<unknown> | unknown;
|
||||||
|
}
|
||||||
|
|
||||||
|
const notImplemented = (name: keyof CrawlerTaskHandlers) => () => {
|
||||||
|
throw new Error(`Crawler task handler '${name}' is not implemented`);
|
||||||
|
};
|
||||||
|
|
||||||
|
let handlers: CrawlerTaskHandlers = {
|
||||||
|
collectListing: notImplemented("collectListing"),
|
||||||
|
collectArticle: notImplemented("collectArticle"),
|
||||||
|
forwardForProcessing: notImplemented("forwardForProcessing"),
|
||||||
|
};
|
||||||
|
|
||||||
|
export const registerCrawlerTaskHandlers = (
|
||||||
|
overrides: Partial<CrawlerTaskHandlers>,
|
||||||
|
): void => {
|
||||||
|
handlers = { ...handlers, ...overrides };
|
||||||
|
};
|
||||||
|
|
||||||
|
export interface ScheduleAsyncCrawlOptions {
|
||||||
|
sourceId: string;
|
||||||
|
env?: string;
|
||||||
|
pageRange?: string | null;
|
||||||
|
dateRange?: string | null;
|
||||||
|
category?: string | null;
|
||||||
|
settings?: QueueSettings | QueueSettingsInput;
|
||||||
|
queueManager?: QueueManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const scheduleAsyncCrawl = async ({
|
||||||
|
sourceId,
|
||||||
|
env = "development",
|
||||||
|
pageRange,
|
||||||
|
dateRange,
|
||||||
|
category,
|
||||||
|
settings,
|
||||||
|
queueManager,
|
||||||
|
}: ScheduleAsyncCrawlOptions): Promise<string> => {
|
||||||
|
const payload = ListingTaskPayloadSchema.parse({
|
||||||
|
source_id: sourceId,
|
||||||
|
env,
|
||||||
|
page_range: pageRange ?? undefined,
|
||||||
|
date_range: dateRange ?? undefined,
|
||||||
|
category: category ?? undefined,
|
||||||
|
});
|
||||||
|
|
||||||
|
const manager = queueManager ?? createQueueManager({ settings });
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
sourceId,
|
||||||
|
env: payload.env,
|
||||||
|
pageRange: payload.page_range,
|
||||||
|
dateRange: payload.date_range,
|
||||||
|
category: payload.category,
|
||||||
|
},
|
||||||
|
"Scheduling listing collection job",
|
||||||
|
);
|
||||||
|
try {
|
||||||
|
const job = await manager.enqueueListing(payload);
|
||||||
|
logger.info(
|
||||||
|
{ jobId: job.id, sourceId, env: payload.env },
|
||||||
|
"Scheduled listing collection job",
|
||||||
|
);
|
||||||
|
return job.id;
|
||||||
|
} finally {
|
||||||
|
if (!queueManager) {
|
||||||
|
await manager.close();
|
||||||
|
}
|
||||||
|
}
|
||||||
|
};
|
||||||
|
|
||||||
|
export const collectListing = async (payload: unknown): Promise<number> => {
|
||||||
|
const data = ListingTaskPayloadSchema.parse(payload);
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
sourceId: data.source_id,
|
||||||
|
env: data.env,
|
||||||
|
pageRange: data.page_range,
|
||||||
|
dateRange: data.date_range,
|
||||||
|
category: data.category,
|
||||||
|
},
|
||||||
|
"Collecting listing",
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await handlers.collectListing(data);
|
||||||
|
const count = typeof result === "number" ? result : 0;
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
sourceId: data.source_id,
|
||||||
|
env: data.env,
|
||||||
|
queuedArticles: count,
|
||||||
|
},
|
||||||
|
"Listing collection completed",
|
||||||
|
);
|
||||||
|
|
||||||
|
return count;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const collectArticle = async (payload: unknown): Promise<unknown> => {
|
||||||
|
const data = ArticleTaskPayloadSchema.parse(payload);
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
sourceId: data.source_id,
|
||||||
|
env: data.env,
|
||||||
|
url: data.url,
|
||||||
|
page: data.page,
|
||||||
|
},
|
||||||
|
"Collecting article",
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await handlers.collectArticle(data);
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
sourceId: data.source_id,
|
||||||
|
env: data.env,
|
||||||
|
url: data.url,
|
||||||
|
},
|
||||||
|
"Article collection completed",
|
||||||
|
);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
|
|
||||||
|
export const forwardForProcessing = async (
|
||||||
|
payload: unknown,
|
||||||
|
): Promise<unknown> => {
|
||||||
|
const data = ProcessedTaskPayloadSchema.parse(payload);
|
||||||
|
logger.debug(
|
||||||
|
{
|
||||||
|
sourceId: data.source_id,
|
||||||
|
env: data.env,
|
||||||
|
},
|
||||||
|
"Forwarding article for processing",
|
||||||
|
);
|
||||||
|
|
||||||
|
const result = await handlers.forwardForProcessing(data);
|
||||||
|
|
||||||
|
logger.info(
|
||||||
|
{
|
||||||
|
sourceId: data.source_id,
|
||||||
|
env: data.env,
|
||||||
|
},
|
||||||
|
"Article forwarded for processing",
|
||||||
|
);
|
||||||
|
|
||||||
|
return result;
|
||||||
|
};
|
||||||
@@ -0,0 +1,87 @@
|
|||||||
|
import IORedis from "ioredis";
|
||||||
|
import { Worker, QueueEvents } from "bullmq";
|
||||||
|
|
||||||
|
import {
|
||||||
|
createQueueManager,
|
||||||
|
QueueFactory,
|
||||||
|
QueueManager,
|
||||||
|
QueueSettings,
|
||||||
|
QueueSettingsInput,
|
||||||
|
} from "./queue";
|
||||||
|
import { collectArticle, collectListing, forwardForProcessing } from "./tasks";
|
||||||
|
|
||||||
|
export interface WorkerOptions {
|
||||||
|
queueNames?: string[];
|
||||||
|
settings?: QueueSettings | QueueSettingsInput;
|
||||||
|
connection?: IORedis;
|
||||||
|
queueFactory?: QueueFactory;
|
||||||
|
concurrency?: number;
|
||||||
|
onError?: (error: Error) => void;
|
||||||
|
queueManager?: QueueManager;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface WorkerHandle {
|
||||||
|
readonly workers: Worker[];
|
||||||
|
readonly events: QueueEvents[];
|
||||||
|
close: () => Promise<void>;
|
||||||
|
}
|
||||||
|
|
||||||
|
export const startWorker = (options: WorkerOptions = {}): WorkerHandle => {
|
||||||
|
const manager =
|
||||||
|
options.queueManager ??
|
||||||
|
createQueueManager({
|
||||||
|
settings: options.settings,
|
||||||
|
connection: options.connection,
|
||||||
|
queueFactory: options.queueFactory,
|
||||||
|
});
|
||||||
|
|
||||||
|
const queueNames = options.queueNames ?? manager.iterQueueNames();
|
||||||
|
const workers: Worker[] = [];
|
||||||
|
const events: QueueEvents[] = [];
|
||||||
|
|
||||||
|
const connection = manager.connection;
|
||||||
|
|
||||||
|
for (const queueName of queueNames) {
|
||||||
|
const worker = new Worker(
|
||||||
|
queueName,
|
||||||
|
async (job) => {
|
||||||
|
switch (job.name) {
|
||||||
|
case "collect_listing":
|
||||||
|
return collectListing(job.data);
|
||||||
|
case "collect_article":
|
||||||
|
return collectArticle(job.data);
|
||||||
|
case "forward_for_processing":
|
||||||
|
return forwardForProcessing(job.data);
|
||||||
|
default:
|
||||||
|
throw new Error(`Unknown job name: ${job.name}`);
|
||||||
|
}
|
||||||
|
},
|
||||||
|
{
|
||||||
|
connection,
|
||||||
|
concurrency: options.concurrency ?? 5,
|
||||||
|
},
|
||||||
|
);
|
||||||
|
|
||||||
|
if (options.onError) {
|
||||||
|
worker.on("failed", (_, err) => options.onError?.(err as Error));
|
||||||
|
worker.on("error", (err) => options.onError?.(err as Error));
|
||||||
|
}
|
||||||
|
|
||||||
|
const queueEvents = new QueueEvents(queueName, { connection });
|
||||||
|
|
||||||
|
workers.push(worker);
|
||||||
|
events.push(queueEvents);
|
||||||
|
}
|
||||||
|
|
||||||
|
return {
|
||||||
|
workers,
|
||||||
|
events,
|
||||||
|
close: async () => {
|
||||||
|
await Promise.all(workers.map((worker) => worker.close()));
|
||||||
|
await Promise.all(events.map((event) => event.close()));
|
||||||
|
if (!options.queueManager) {
|
||||||
|
await manager.close();
|
||||||
|
}
|
||||||
|
},
|
||||||
|
};
|
||||||
|
};
|
||||||
@@ -1,142 +0,0 @@
|
|||||||
import { randomUUID } from "node:crypto";
|
|
||||||
|
|
||||||
import IORedis from "ioredis";
|
|
||||||
import { Queue, JobsOptions, QueueOptions } from "bullmq";
|
|
||||||
import { z } from "zod";
|
|
||||||
|
|
||||||
import {
|
|
||||||
ListingTaskPayload,
|
|
||||||
ArticleTaskPayload,
|
|
||||||
ProcessedTaskPayload,
|
|
||||||
ListingTaskPayloadSchema,
|
|
||||||
ArticleTaskPayloadSchema,
|
|
||||||
ProcessedTaskPayloadSchema,
|
|
||||||
} from "./schemas";
|
|
||||||
import { parseRedisUrl } from "../../../utils";
|
|
||||||
|
|
||||||
const QueueSettingsSchema = z.object({
|
|
||||||
redis_url: z
|
|
||||||
.string()
|
|
||||||
.default(process.env.BASANGO_REDIS_URL ?? "redis://localhost:6379/0"),
|
|
||||||
prefix: z.string().default(process.env.BASANGO_QUEUE_PREFIX ?? "crawler"),
|
|
||||||
default_timeout: z
|
|
||||||
.number()
|
|
||||||
.int()
|
|
||||||
.positive()
|
|
||||||
.default(Number(process.env.BASANGO_QUEUE_TIMEOUT ?? 600)),
|
|
||||||
result_ttl: z
|
|
||||||
.number()
|
|
||||||
.int()
|
|
||||||
.nonnegative()
|
|
||||||
.default(Number(process.env.BASANGO_QUEUE_RESULT_TTL ?? 3600)),
|
|
||||||
failure_ttl: z
|
|
||||||
.number()
|
|
||||||
.int()
|
|
||||||
.nonnegative()
|
|
||||||
.default(Number(process.env.BASANGO_QUEUE_FAILURE_TTL ?? 3600)),
|
|
||||||
listing_queue: z.string().default("listing"),
|
|
||||||
article_queue: z.string().default("articles"),
|
|
||||||
processed_queue: z.string().default("processed"),
|
|
||||||
});
|
|
||||||
|
|
||||||
export type QueueSettingsInput = z.input<typeof QueueSettingsSchema>;
|
|
||||||
export type QueueSettings = z.output<typeof QueueSettingsSchema>;
|
|
||||||
|
|
||||||
export const createQueueSettings = (
|
|
||||||
input?: QueueSettingsInput,
|
|
||||||
): QueueSettings => QueueSettingsSchema.parse(input ?? {});
|
|
||||||
|
|
||||||
export interface QueueBackend<T = unknown> {
|
|
||||||
add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export type QueueFactory = (
|
|
||||||
queueName: string,
|
|
||||||
settings: QueueSettings,
|
|
||||||
connection?: IORedis,
|
|
||||||
) => QueueBackend;
|
|
||||||
|
|
||||||
const defaultQueueFactory: QueueFactory = (queueName, settings, connection) => {
|
|
||||||
const redisConnection =
|
|
||||||
connection ??
|
|
||||||
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
|
||||||
const options: QueueOptions = {
|
|
||||||
connection: redisConnection,
|
|
||||||
prefix: settings.prefix,
|
|
||||||
};
|
|
||||||
|
|
||||||
const queue = new Queue(queueName, options);
|
|
||||||
return {
|
|
||||||
add: async (name, data, opts) => {
|
|
||||||
const job = await queue.add(name, data, {
|
|
||||||
removeOnComplete: settings.result_ttl === 0 ? true : undefined,
|
|
||||||
removeOnFail: settings.failure_ttl === 0 ? true : undefined,
|
|
||||||
timeout: settings.default_timeout * 1000,
|
|
||||||
...opts,
|
|
||||||
});
|
|
||||||
return { id: job.id ?? randomUUID() };
|
|
||||||
},
|
|
||||||
};
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface CreateQueueManagerOptions {
|
|
||||||
settings?: QueueSettings | QueueSettingsInput;
|
|
||||||
queueFactory?: QueueFactory;
|
|
||||||
connection?: IORedis;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface QueueManager {
|
|
||||||
readonly settings: QueueSettings;
|
|
||||||
readonly connection: IORedis;
|
|
||||||
enqueueListing: (payload: ListingTaskPayload) => Promise<{ id: string }>;
|
|
||||||
enqueueArticle: (payload: ArticleTaskPayload) => Promise<{ id: string }>;
|
|
||||||
enqueueProcessed: (payload: ProcessedTaskPayload) => Promise<{ id: string }>;
|
|
||||||
iterQueueNames: () => string[];
|
|
||||||
queueName: (suffix: string) => string;
|
|
||||||
close: () => Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const createQueueManager = (
|
|
||||||
options: CreateQueueManagerOptions = {},
|
|
||||||
): QueueManager => {
|
|
||||||
const settings = createQueueSettings(
|
|
||||||
options.settings as QueueSettingsInput | undefined,
|
|
||||||
);
|
|
||||||
|
|
||||||
const connection =
|
|
||||||
options.connection ??
|
|
||||||
new IORedis(settings.redis_url, parseRedisUrl(settings.redis_url));
|
|
||||||
const factory = options.queueFactory ?? defaultQueueFactory;
|
|
||||||
|
|
||||||
const ensureQueue = (queueName: string) =>
|
|
||||||
factory(queueName, settings, connection);
|
|
||||||
|
|
||||||
return {
|
|
||||||
settings,
|
|
||||||
connection,
|
|
||||||
enqueueListing: (payload) => {
|
|
||||||
const data = ListingTaskPayloadSchema.parse(payload);
|
|
||||||
const queue = ensureQueue(settings.listing_queue);
|
|
||||||
return queue.add("collect_listing", data);
|
|
||||||
},
|
|
||||||
enqueueArticle: (payload) => {
|
|
||||||
const data = ArticleTaskPayloadSchema.parse(payload);
|
|
||||||
const queue = ensureQueue(settings.article_queue);
|
|
||||||
return queue.add("collect_article", data);
|
|
||||||
},
|
|
||||||
enqueueProcessed: (payload) => {
|
|
||||||
const data = ProcessedTaskPayloadSchema.parse(payload);
|
|
||||||
const queue = ensureQueue(settings.processed_queue);
|
|
||||||
return queue.add("forward_for_processing", data);
|
|
||||||
},
|
|
||||||
iterQueueNames: () => [
|
|
||||||
`${settings.prefix}:${settings.listing_queue}`,
|
|
||||||
`${settings.prefix}:${settings.article_queue}`,
|
|
||||||
`${settings.prefix}:${settings.processed_queue}`,
|
|
||||||
],
|
|
||||||
queueName: (suffix: string) => `${settings.prefix}:${suffix}`,
|
|
||||||
close: async () => {
|
|
||||||
await connection.quit();
|
|
||||||
},
|
|
||||||
};
|
|
||||||
};
|
|
||||||
@@ -1,41 +0,0 @@
|
|||||||
import { z } from "zod";
|
|
||||||
|
|
||||||
import {
|
|
||||||
AnySourceConfig,
|
|
||||||
DateRangeSchema,
|
|
||||||
PageRangeSchema,
|
|
||||||
} from "../../../schema";
|
|
||||||
|
|
||||||
export const ListingTaskPayloadSchema = z.object({
|
|
||||||
source_id: z.string(),
|
|
||||||
env: z.string().default("development"),
|
|
||||||
page_range: z.string().optional().nullable(),
|
|
||||||
date_range: z.string().optional().nullable(),
|
|
||||||
category: z.string().optional().nullable(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export type ListingTaskPayload = z.infer<typeof ListingTaskPayloadSchema>;
|
|
||||||
|
|
||||||
export const ArticleTaskPayloadSchema = z.object({
|
|
||||||
source_id: z.string(),
|
|
||||||
env: z.string().default("development"),
|
|
||||||
url: z.string().url(),
|
|
||||||
page: z.number().int().nonnegative().optional(),
|
|
||||||
page_range: PageRangeSchema.optional().nullable(),
|
|
||||||
date_range: DateRangeSchema.optional().nullable(),
|
|
||||||
category: z.string().optional().nullable(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export type ArticleTaskPayload = z.infer<typeof ArticleTaskPayloadSchema>;
|
|
||||||
|
|
||||||
export const ProcessedTaskPayloadSchema = z.object({
|
|
||||||
source_id: z.string(),
|
|
||||||
env: z.string().default("development"),
|
|
||||||
article: z.any(),
|
|
||||||
});
|
|
||||||
|
|
||||||
export type ProcessedTaskPayload = z.infer<typeof ProcessedTaskPayloadSchema>;
|
|
||||||
|
|
||||||
export interface ListingContext {
|
|
||||||
source: AnySourceConfig;
|
|
||||||
}
|
|
||||||
@@ -1,171 +0,0 @@
|
|||||||
import { logger } from "@basango/logger";
|
|
||||||
|
|
||||||
import {
|
|
||||||
ListingTaskPayloadSchema,
|
|
||||||
ArticleTaskPayloadSchema,
|
|
||||||
ProcessedTaskPayloadSchema,
|
|
||||||
ListingTaskPayload,
|
|
||||||
ArticleTaskPayload,
|
|
||||||
ProcessedTaskPayload,
|
|
||||||
} from "./schemas";
|
|
||||||
import {
|
|
||||||
createQueueManager,
|
|
||||||
QueueManager,
|
|
||||||
QueueSettings,
|
|
||||||
QueueSettingsInput,
|
|
||||||
} from "./queue";
|
|
||||||
|
|
||||||
export interface CrawlerTaskHandlers {
|
|
||||||
collectListing: (payload: ListingTaskPayload) => Promise<number> | number;
|
|
||||||
collectArticle: (payload: ArticleTaskPayload) => Promise<unknown> | unknown;
|
|
||||||
forwardForProcessing: (
|
|
||||||
payload: ProcessedTaskPayload,
|
|
||||||
) => Promise<unknown> | unknown;
|
|
||||||
}
|
|
||||||
|
|
||||||
const notImplemented = (name: keyof CrawlerTaskHandlers) => () => {
|
|
||||||
throw new Error(`Crawler task handler '${name}' is not implemented`);
|
|
||||||
};
|
|
||||||
|
|
||||||
let handlers: CrawlerTaskHandlers = {
|
|
||||||
collectListing: notImplemented("collectListing"),
|
|
||||||
collectArticle: notImplemented("collectArticle"),
|
|
||||||
forwardForProcessing: notImplemented("forwardForProcessing"),
|
|
||||||
};
|
|
||||||
|
|
||||||
export const registerCrawlerTaskHandlers = (
|
|
||||||
overrides: Partial<CrawlerTaskHandlers>,
|
|
||||||
): void => {
|
|
||||||
handlers = { ...handlers, ...overrides };
|
|
||||||
};
|
|
||||||
|
|
||||||
export interface ScheduleAsyncCrawlOptions {
|
|
||||||
sourceId: string;
|
|
||||||
env?: string;
|
|
||||||
pageRange?: string | null;
|
|
||||||
dateRange?: string | null;
|
|
||||||
category?: string | null;
|
|
||||||
settings?: QueueSettings | QueueSettingsInput;
|
|
||||||
queueManager?: QueueManager;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const scheduleAsyncCrawl = async ({
|
|
||||||
sourceId,
|
|
||||||
env = "development",
|
|
||||||
pageRange,
|
|
||||||
dateRange,
|
|
||||||
category,
|
|
||||||
settings,
|
|
||||||
queueManager,
|
|
||||||
}: ScheduleAsyncCrawlOptions): Promise<string> => {
|
|
||||||
const payload = ListingTaskPayloadSchema.parse({
|
|
||||||
source_id: sourceId,
|
|
||||||
env,
|
|
||||||
page_range: pageRange ?? undefined,
|
|
||||||
date_range: dateRange ?? undefined,
|
|
||||||
category: category ?? undefined,
|
|
||||||
});
|
|
||||||
|
|
||||||
const manager = queueManager ?? createQueueManager({ settings });
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
sourceId,
|
|
||||||
env: payload.env,
|
|
||||||
pageRange: payload.page_range,
|
|
||||||
dateRange: payload.date_range,
|
|
||||||
category: payload.category,
|
|
||||||
},
|
|
||||||
"Scheduling listing collection job",
|
|
||||||
);
|
|
||||||
try {
|
|
||||||
const job = await manager.enqueueListing(payload);
|
|
||||||
logger.info(
|
|
||||||
{ jobId: job.id, sourceId, env: payload.env },
|
|
||||||
"Scheduled listing collection job",
|
|
||||||
);
|
|
||||||
return job.id;
|
|
||||||
} finally {
|
|
||||||
if (!queueManager) {
|
|
||||||
await manager.close();
|
|
||||||
}
|
|
||||||
}
|
|
||||||
};
|
|
||||||
|
|
||||||
export const collectListing = async (payload: unknown): Promise<number> => {
|
|
||||||
const data = ListingTaskPayloadSchema.parse(payload);
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
sourceId: data.source_id,
|
|
||||||
env: data.env,
|
|
||||||
pageRange: data.page_range,
|
|
||||||
dateRange: data.date_range,
|
|
||||||
category: data.category,
|
|
||||||
},
|
|
||||||
"Collecting listing",
|
|
||||||
);
|
|
||||||
|
|
||||||
const result = await handlers.collectListing(data);
|
|
||||||
const count = typeof result === "number" ? result : 0;
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
sourceId: data.source_id,
|
|
||||||
env: data.env,
|
|
||||||
queuedArticles: count,
|
|
||||||
},
|
|
||||||
"Listing collection completed",
|
|
||||||
);
|
|
||||||
|
|
||||||
return count;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const collectArticle = async (payload: unknown): Promise<unknown> => {
|
|
||||||
const data = ArticleTaskPayloadSchema.parse(payload);
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
sourceId: data.source_id,
|
|
||||||
env: data.env,
|
|
||||||
url: data.url,
|
|
||||||
page: data.page,
|
|
||||||
},
|
|
||||||
"Collecting article",
|
|
||||||
);
|
|
||||||
|
|
||||||
const result = await handlers.collectArticle(data);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
sourceId: data.source_id,
|
|
||||||
env: data.env,
|
|
||||||
url: data.url,
|
|
||||||
},
|
|
||||||
"Article collection completed",
|
|
||||||
);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
|
|
||||||
export const forwardForProcessing = async (
|
|
||||||
payload: unknown,
|
|
||||||
): Promise<unknown> => {
|
|
||||||
const data = ProcessedTaskPayloadSchema.parse(payload);
|
|
||||||
logger.debug(
|
|
||||||
{
|
|
||||||
sourceId: data.source_id,
|
|
||||||
env: data.env,
|
|
||||||
},
|
|
||||||
"Forwarding article for processing",
|
|
||||||
);
|
|
||||||
|
|
||||||
const result = await handlers.forwardForProcessing(data);
|
|
||||||
|
|
||||||
logger.info(
|
|
||||||
{
|
|
||||||
sourceId: data.source_id,
|
|
||||||
env: data.env,
|
|
||||||
},
|
|
||||||
"Article forwarded for processing",
|
|
||||||
);
|
|
||||||
|
|
||||||
return result;
|
|
||||||
};
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
import IORedis from "ioredis";
|
|
||||||
import { Worker, QueueEvents } from "bullmq";
|
|
||||||
|
|
||||||
import {
|
|
||||||
createQueueManager,
|
|
||||||
QueueFactory,
|
|
||||||
QueueManager,
|
|
||||||
QueueSettings,
|
|
||||||
QueueSettingsInput,
|
|
||||||
} from "./queue";
|
|
||||||
import { collectArticle, collectListing, forwardForProcessing } from "./tasks";
|
|
||||||
|
|
||||||
export interface WorkerOptions {
|
|
||||||
queueNames?: string[];
|
|
||||||
settings?: QueueSettings | QueueSettingsInput;
|
|
||||||
connection?: IORedis;
|
|
||||||
queueFactory?: QueueFactory;
|
|
||||||
concurrency?: number;
|
|
||||||
onError?: (error: Error) => void;
|
|
||||||
queueManager?: QueueManager;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface WorkerHandle {
|
|
||||||
readonly workers: Worker[];
|
|
||||||
readonly events: QueueEvents[];
|
|
||||||
close: () => Promise<void>;
|
|
||||||
}
|
|
||||||
|
|
||||||
export const startWorker = (options: WorkerOptions = {}): WorkerHandle => {
|
|
||||||
const manager =
|
|
||||||
options.queueManager ??
|
|
||||||
createQueueManager({
|
|
||||||
settings: options.settings,
|
|
||||||
connection: options.connection,
|
|
||||||
queueFactory: options.queueFactory,
|
|
||||||
});
|
|
||||||
|
|
||||||
const queueNames = options.queueNames ?? manager.iterQueueNames();
|
|
||||||
const workers: Worker[] = [];
|
|
||||||
const events: QueueEvents[] = [];
|
|
||||||
|
|
||||||
const connection = manager.connection;
|
|
||||||
|
|
||||||
for (const queueName of queueNames) {
|
|
||||||
const worker = new Worker(
|
|
||||||
queueName,
|
|
||||||
async (job) => {
|
|
||||||
switch (job.name) {
|
|
||||||
case "collect_listing":
|
|
||||||
return collectListing(job.data);
|
|
||||||
case "collect_article":
|
|
||||||
return collectArticle(job.data);
|
|
||||||
case "forward_for_processing":
|
|
||||||
return forwardForProcessing(job.data);
|
|
||||||
default:
|
|
||||||
throw new Error(`Unknown job name: ${job.name}`);
|
|
||||||
}
|
|
||||||
},
|
|
||||||
{
|
|
||||||
connection,
|
|
||||||
concurrency: options.concurrency ?? 5,
|
|
||||||
},
|
|
||||||
);
|
|
||||||
|
|
||||||
if (options.onError) {
|
|
||||||
worker.on("failed", (_, err) => options.onError?.(err as Error));
|
|
||||||
worker.on("error", (err) => options.onError?.(err as Error));
|
|
||||||
}
|
|
||||||
|
|
||||||
const queueEvents = new QueueEvents(queueName, { connection });
|
|
||||||
|
|
||||||
workers.push(worker);
|
|
||||||
events.push(queueEvents);
|
|
||||||
}
|
|
||||||
|
|
||||||
return {
|
|
||||||
workers,
|
|
||||||
events,
|
|
||||||
close: async () => {
|
|
||||||
await Promise.all(workers.map((worker) => worker.close()));
|
|
||||||
await Promise.all(events.map((event) => event.close()));
|
|
||||||
if (!options.queueManager) {
|
|
||||||
await manager.close();
|
|
||||||
}
|
|
||||||
},
|
|
||||||
};
|
|
||||||
};
|
|
||||||
@@ -1,3 +0,0 @@
|
|||||||
export * from "./async/queue";
|
|
||||||
export * from "./async/tasks";
|
|
||||||
export * from "./async/worker";
|
|
||||||
@@ -6,33 +6,33 @@ import { get_encoding } from "tiktoken";
|
|||||||
import type { ProjectPaths } from "@crawler/schema";
|
import type { ProjectPaths } from "@crawler/schema";
|
||||||
|
|
||||||
export const ensureDirectories = (paths: ProjectPaths): void => {
|
export const ensureDirectories = (paths: ProjectPaths): void => {
|
||||||
for (const dir of [paths.data, paths.logs, paths.configs]) {
|
for (const dir of [paths.data, paths.logs, paths.configs]) {
|
||||||
if (!fs.existsSync(dir)) {
|
if (!fs.existsSync(dir)) {
|
||||||
fs.mkdirSync(dir, { recursive: true });
|
fs.mkdirSync(dir, { recursive: true });
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
export const parseRedisUrl = (url: string): RedisOptions => {
|
export const parseRedisUrl = (url: string): RedisOptions => {
|
||||||
if (!url.startsWith("redis://")) {
|
if (!url.startsWith("redis://")) {
|
||||||
return {};
|
return {};
|
||||||
}
|
}
|
||||||
const parsed = new URL(url);
|
const parsed = new URL(url);
|
||||||
return {
|
return {
|
||||||
host: parsed.hostname,
|
host: parsed.hostname,
|
||||||
port: Number(parsed.port || 6379),
|
port: Number(parsed.port || 6379),
|
||||||
password: parsed.password || undefined,
|
password: parsed.password || undefined,
|
||||||
db: Number(parsed.pathname?.replace("/", "") || 0),
|
db: Number(parsed.pathname?.replace("/", "") || 0),
|
||||||
};
|
};
|
||||||
};
|
};
|
||||||
|
|
||||||
export const countTokens = (text: string, encoding = "cl100k_base"): number => {
|
export const countTokens = (text: string, encoding = "cl100k_base"): number => {
|
||||||
try {
|
try {
|
||||||
const encoder = get_encoding(encoding);
|
const encoder = get_encoding(encoding);
|
||||||
const tokens = encoder.encode(text);
|
const tokens = encoder.encode(text);
|
||||||
encoder.free();
|
encoder.free();
|
||||||
return tokens.length;
|
return tokens.length;
|
||||||
} catch {
|
} catch {
|
||||||
return text.length;
|
return text.length;
|
||||||
}
|
}
|
||||||
};
|
};
|
||||||
|
|||||||
@@ -1,13 +1,13 @@
|
|||||||
{
|
{
|
||||||
"extends": "@basango/tsconfig/base.json",
|
"extends": "@basango/tsconfig/base.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"rootDir": "src",
|
"rootDir": "src",
|
||||||
"outDir": "dist",
|
"outDir": "dist",
|
||||||
"paths": {
|
"paths": {
|
||||||
"@crawler": ["./src/index.ts"],
|
"@crawler": ["./src/index.ts"],
|
||||||
"@crawler/*": ["./src/*"]
|
"@crawler/*": ["./src/*"]
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"include": ["src"],
|
"include": ["src"],
|
||||||
"references": []
|
"references": []
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,9 +1,9 @@
|
|||||||
import { defineConfig } from "vitest/config";
|
import { defineConfig } from "vitest/config";
|
||||||
|
|
||||||
export default defineConfig({
|
export default defineConfig({
|
||||||
test: {
|
test: {
|
||||||
environment: "node",
|
environment: "node",
|
||||||
globals: true,
|
globals: true,
|
||||||
include: ["src/**/*.test.ts"],
|
include: ["src/**/*.test.ts"],
|
||||||
},
|
},
|
||||||
});
|
});
|
||||||
|
|||||||
+32
-32
@@ -1,34 +1,34 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://biomejs.dev/schemas/2.3.1/schema.json",
|
"$schema": "https://biomejs.dev/schemas/2.3.1/schema.json",
|
||||||
"vcs": {
|
"vcs": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"clientKind": "git",
|
"clientKind": "git",
|
||||||
"useIgnoreFile": true
|
"useIgnoreFile": true
|
||||||
},
|
},
|
||||||
"files": {
|
"files": {
|
||||||
"ignoreUnknown": false
|
"ignoreUnknown": false
|
||||||
},
|
},
|
||||||
"formatter": {
|
"formatter": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"indentStyle": "space"
|
"indentStyle": "space"
|
||||||
},
|
},
|
||||||
"linter": {
|
"linter": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"rules": {
|
"rules": {
|
||||||
"recommended": true
|
"recommended": true
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"javascript": {
|
"javascript": {
|
||||||
"formatter": {
|
"formatter": {
|
||||||
"quoteStyle": "double"
|
"quoteStyle": "double"
|
||||||
}
|
}
|
||||||
},
|
},
|
||||||
"assist": {
|
"assist": {
|
||||||
"enabled": true,
|
"enabled": true,
|
||||||
"actions": {
|
"actions": {
|
||||||
"source": {
|
"source": {
|
||||||
"organizeImports": "on"
|
"organizeImports": "on"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
+30
-7
@@ -7,13 +7,14 @@
|
|||||||
"@biomejs/biome": "^2.3.1",
|
"@biomejs/biome": "^2.3.1",
|
||||||
"@manypkg/cli": "^0.25.1",
|
"@manypkg/cli": "^0.25.1",
|
||||||
"turbo": "^2.5.8",
|
"turbo": "^2.5.8",
|
||||||
"typescript": "5.9.2",
|
"typescript": "catalog:",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"apps/crawler": {
|
"apps/crawler": {
|
||||||
"name": "@basango/crawler",
|
"name": "@basango/crawler",
|
||||||
"version": "0.1.0",
|
"version": "0.1.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@basango/logger": "workspace:*",
|
||||||
"bullmq": "^4.17.0",
|
"bullmq": "^4.17.0",
|
||||||
"date-fns": "^3.6.0",
|
"date-fns": "^3.6.0",
|
||||||
"ioredis": "^5.3.2",
|
"ioredis": "^5.3.2",
|
||||||
@@ -25,24 +26,28 @@
|
|||||||
"name": "@basango/db",
|
"name": "@basango/db",
|
||||||
"version": "1.0.0",
|
"version": "1.0.0",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@basango/logger": "workspace:*",
|
||||||
"@date-fns/utc": "^2.1.1",
|
"@date-fns/utc": "^2.1.1",
|
||||||
"drizzle-orm": "^0.44.7",
|
"drizzle-orm": "^0.44.7",
|
||||||
"pg": "^8.16.3",
|
"pg": "^8.16.3",
|
||||||
"snakecase-keys": "^9.0.2",
|
"snakecase-keys": "^9.0.2",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
|
"@types/bun": "^1.3.1",
|
||||||
|
"@types/pg": "^8.15.6",
|
||||||
"drizzle-kit": "^0.31.6",
|
"drizzle-kit": "^0.31.6",
|
||||||
|
"typescript": "catalog:",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"packages/logger": {
|
"packages/logger": {
|
||||||
"name": "@midday/logger",
|
"name": "@basango/logger",
|
||||||
"version": "0.0.0",
|
"version": "0.0.1",
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"pino": "^10.1.0",
|
"pino": "^10.1.0",
|
||||||
"pino-pretty": "^13.1.2",
|
"pino-pretty": "^13.1.2",
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"typescript": "^5.9.2",
|
"typescript": "catalog:",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
"packages/tsconfig": {
|
"packages/tsconfig": {
|
||||||
@@ -50,11 +55,17 @@
|
|||||||
"version": "0.0.0",
|
"version": "0.0.0",
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
|
"catalog": {
|
||||||
|
"@types/bun": "^1.3.1",
|
||||||
|
"typescript": "^5.9.3",
|
||||||
|
},
|
||||||
"packages": {
|
"packages": {
|
||||||
"@basango/crawler": ["@basango/crawler@workspace:apps/crawler"],
|
"@basango/crawler": ["@basango/crawler@workspace:apps/crawler"],
|
||||||
|
|
||||||
"@basango/db": ["@basango/db@workspace:packages/db"],
|
"@basango/db": ["@basango/db@workspace:packages/db"],
|
||||||
|
|
||||||
|
"@basango/logger": ["@basango/logger@workspace:packages/logger"],
|
||||||
|
|
||||||
"@basango/tsconfig": ["@basango/tsconfig@workspace:packages/tsconfig"],
|
"@basango/tsconfig": ["@basango/tsconfig@workspace:packages/tsconfig"],
|
||||||
|
|
||||||
"@biomejs/biome": ["@biomejs/biome@2.3.1", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.1", "@biomejs/cli-darwin-x64": "2.3.1", "@biomejs/cli-linux-arm64": "2.3.1", "@biomejs/cli-linux-arm64-musl": "2.3.1", "@biomejs/cli-linux-x64": "2.3.1", "@biomejs/cli-linux-x64-musl": "2.3.1", "@biomejs/cli-win32-arm64": "2.3.1", "@biomejs/cli-win32-x64": "2.3.1" }, "bin": { "biome": "bin/biome" } }, "sha512-A29evf1R72V5bo4o2EPxYMm5mtyGvzp2g+biZvRFx29nWebGyyeOSsDWGx3tuNNMFRepGwxmA9ZQ15mzfabK2w=="],
|
"@biomejs/biome": ["@biomejs/biome@2.3.1", "", { "optionalDependencies": { "@biomejs/cli-darwin-arm64": "2.3.1", "@biomejs/cli-darwin-x64": "2.3.1", "@biomejs/cli-linux-arm64": "2.3.1", "@biomejs/cli-linux-arm64-musl": "2.3.1", "@biomejs/cli-linux-x64": "2.3.1", "@biomejs/cli-linux-x64-musl": "2.3.1", "@biomejs/cli-win32-arm64": "2.3.1", "@biomejs/cli-win32-x64": "2.3.1" }, "bin": { "biome": "bin/biome" } }, "sha512-A29evf1R72V5bo4o2EPxYMm5mtyGvzp2g+biZvRFx29nWebGyyeOSsDWGx3tuNNMFRepGwxmA9ZQ15mzfabK2w=="],
|
||||||
@@ -145,8 +156,6 @@
|
|||||||
|
|
||||||
"@manypkg/tools": ["@manypkg/tools@2.1.0", "", { "dependencies": { "jju": "^1.4.0", "js-yaml": "^4.1.0", "tinyglobby": "^0.2.13" } }, "sha512-0FOIepYR4ugPYaHwK7hDeHDkfPOBVvayt9QpvRbi2LT/h2b0GaE/gM9Gag7fsnyYyNaTZ2IGyOuVg07IYepvYQ=="],
|
"@manypkg/tools": ["@manypkg/tools@2.1.0", "", { "dependencies": { "jju": "^1.4.0", "js-yaml": "^4.1.0", "tinyglobby": "^0.2.13" } }, "sha512-0FOIepYR4ugPYaHwK7hDeHDkfPOBVvayt9QpvRbi2LT/h2b0GaE/gM9Gag7fsnyYyNaTZ2IGyOuVg07IYepvYQ=="],
|
||||||
|
|
||||||
"@midday/logger": ["@midday/logger@workspace:packages/logger"],
|
|
||||||
|
|
||||||
"@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="],
|
"@msgpackr-extract/msgpackr-extract-darwin-arm64": ["@msgpackr-extract/msgpackr-extract-darwin-arm64@3.0.3", "", { "os": "darwin", "cpu": "arm64" }, "sha512-QZHtlVgbAdy2zAqNA9Gu1UpIuI8Xvsd1v8ic6B2pZmeFnFcMWiPLfWXh7TVw4eGEZ/C9TH281KwhVoeQUKbyjw=="],
|
||||||
|
|
||||||
"@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="],
|
"@msgpackr-extract/msgpackr-extract-darwin-x64": ["@msgpackr-extract/msgpackr-extract-darwin-x64@3.0.3", "", { "os": "darwin", "cpu": "x64" }, "sha512-mdzd3AVzYKuUmiWOQ8GNhl64/IoFGol569zNRdkLReh6LRLHOXxU4U8eq0JwaD8iFHdVGqSy4IjFL4reoWCDFw=="],
|
||||||
@@ -167,6 +176,14 @@
|
|||||||
|
|
||||||
"@pnpm/npm-conf": ["@pnpm/npm-conf@2.3.1", "", { "dependencies": { "@pnpm/config.env-replace": "^1.1.0", "@pnpm/network.ca-file": "^1.0.1", "config-chain": "^1.1.11" } }, "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw=="],
|
"@pnpm/npm-conf": ["@pnpm/npm-conf@2.3.1", "", { "dependencies": { "@pnpm/config.env-replace": "^1.1.0", "@pnpm/network.ca-file": "^1.0.1", "config-chain": "^1.1.11" } }, "sha512-c83qWb22rNRuB0UaVCI0uRPNRr8Z0FWnEIvT47jiHAmOIUHbBOg5XvV7pM5x+rKn9HRpjxquDbXYSXr3fAKFcw=="],
|
||||||
|
|
||||||
|
"@types/bun": ["@types/bun@1.3.1", "", { "dependencies": { "bun-types": "1.3.1" } }, "sha512-4jNMk2/K9YJtfqwoAa28c8wK+T7nvJFOjxI4h/7sORWcypRNxBpr+TPNaCfVWq70tLCJsqoFwcf0oI0JU/fvMQ=="],
|
||||||
|
|
||||||
|
"@types/node": ["@types/node@24.9.2", "", { "dependencies": { "undici-types": "~7.16.0" } }, "sha512-uWN8YqxXxqFMX2RqGOrumsKeti4LlmIMIyV0lgut4jx7KQBcBiW6vkDtIBvHnHIquwNfJhk8v2OtmO8zXWHfPA=="],
|
||||||
|
|
||||||
|
"@types/pg": ["@types/pg@8.15.6", "", { "dependencies": { "@types/node": "*", "pg-protocol": "*", "pg-types": "^2.2.0" } }, "sha512-NoaMtzhxOrubeL/7UZuNTrejB4MPAJ0RpxZqXQf2qXuVlTPuG6Y8p4u9dKRaue4yjmC7ZhzVO2/Yyyn25znrPQ=="],
|
||||||
|
|
||||||
|
"@types/react": ["@types/react@19.2.2", "", { "dependencies": { "csstype": "^3.0.2" } }, "sha512-6mDvHUFSjyT2B2yeNx2nUgMxh9LtOWvkhIU3uePn2I2oyNymUAX1NIsdgviM4CH+JSrp2D2hsMvJOkxY+0wNRA=="],
|
||||||
|
|
||||||
"argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
|
"argparse": ["argparse@2.0.1", "", {}, "sha512-8+9WqebbFzpX9OR+Wa6O29asIogeRMzcGtAINdpMHHyAg10f05aSFVBbcEqGf/PXw1EjAZ+q2/bEBg3DvurK3Q=="],
|
||||||
|
|
||||||
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
|
"atomic-sleep": ["atomic-sleep@1.0.0", "", {}, "sha512-kNOjDqAh7px0XWNI+4QbzoiR/nTkHAWNud2uvnJquD1/x5a7EQZMJT0AczqK0Qn67oY/TTQ1LbUKajZpp3I9tQ=="],
|
||||||
@@ -179,6 +196,8 @@
|
|||||||
|
|
||||||
"bullmq": ["bullmq@4.18.3", "", { "dependencies": { "cron-parser": "^4.6.0", "glob": "^8.0.3", "ioredis": "^5.3.2", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-H8t9vhfHEbJDaXp7aalSTe+Do+tR1nvr+lsT+jQxLhy+FFfFj/0p4aYJzADTNLdEqltuxneLVxCGVg92GkQx4w=="],
|
"bullmq": ["bullmq@4.18.3", "", { "dependencies": { "cron-parser": "^4.6.0", "glob": "^8.0.3", "ioredis": "^5.3.2", "lodash": "^4.17.21", "msgpackr": "^1.6.2", "node-abort-controller": "^3.1.1", "semver": "^7.5.4", "tslib": "^2.0.0", "uuid": "^9.0.0" } }, "sha512-H8t9vhfHEbJDaXp7aalSTe+Do+tR1nvr+lsT+jQxLhy+FFfFj/0p4aYJzADTNLdEqltuxneLVxCGVg92GkQx4w=="],
|
||||||
|
|
||||||
|
"bun-types": ["bun-types@1.3.1", "", { "dependencies": { "@types/node": "*" }, "peerDependencies": { "@types/react": "^19" } }, "sha512-NMrcy7smratanWJ2mMXdpatalovtxVggkj11bScuWuiOoXTiKIu2eVS1/7qbyI/4yHedtsn175n4Sm4JcdHLXw=="],
|
||||||
|
|
||||||
"change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="],
|
"change-case": ["change-case@5.4.4", "", {}, "sha512-HRQyTk2/YPEkt9TnUPbOpr64Uw3KOicFWPVBb+xiHvd6eBx/qPr9xqfBFDT8P2vWsvvz4jbEkfDe71W3VyNu2w=="],
|
||||||
|
|
||||||
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
|
"cluster-key-slot": ["cluster-key-slot@1.1.2", "", {}, "sha512-RMr0FhtfXemyinomL4hrWcYJxmX6deFdCxpJzhDttxgO1+bcCnkk+9drydLVDmAMG7NE6aN/fl4F7ucU/90gAA=="],
|
||||||
@@ -189,6 +208,8 @@
|
|||||||
|
|
||||||
"cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="],
|
"cron-parser": ["cron-parser@4.9.0", "", { "dependencies": { "luxon": "^3.2.1" } }, "sha512-p0SaNjrHOnQeR8/VnfGbmg9te2kfyYSQ7Sc/j/6DtPL3JQvKxmjO9TSjNFpujqV3vEYYBvNNvXSxzyksBWAx1Q=="],
|
||||||
|
|
||||||
|
"csstype": ["csstype@3.1.3", "", {}, "sha512-M1uQkMl8rQK/szD0LNhtqxIPLpimGm8sOBwU7lLnCpSbTyY3yeU1Vc7l4KT5zT4s/yOxHH5O7tIuuLOCnLADRw=="],
|
||||||
|
|
||||||
"date-fns": ["date-fns@3.6.0", "", {}, "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww=="],
|
"date-fns": ["date-fns@3.6.0", "", {}, "sha512-fRHTG8g/Gif+kSh50gaGEdToemgfj74aRX3swtiouboip5JDLAyDE9F11nHMIcvOaXeOC6D7SpNhi7uFyB7Uww=="],
|
||||||
|
|
||||||
"dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="],
|
"dateformat": ["dateformat@4.6.3", "", {}, "sha512-2P0p0pFGzHS5EMnhdxQi7aJN+iMheud0UhG4dlE1DLAlvL8JHjJJTX/CSm4JXwV0Ka5nGk3zC5mcb5bUQUxxMA=="],
|
||||||
@@ -387,7 +408,9 @@
|
|||||||
|
|
||||||
"type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="],
|
"type-fest": ["type-fest@4.41.0", "", {}, "sha512-TeTSQ6H5YHvpqVwBRcnLDCBnDOHWYu7IvGbHT6N8AOymcr9PJGjc1GTtiWZTYg0NCgYwvnYWEkVChQAr9bjfwA=="],
|
||||||
|
|
||||||
"typescript": ["typescript@5.9.2", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-CWBzXQrc/qOkhidw1OzBTQuYRbfyxDXJMVJ1XNwUHGROVmuaeiEm3OslpZ1RV96d7SKKjZKrSJu3+t/xlw3R9A=="],
|
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||||
|
|
||||||
|
"undici-types": ["undici-types@7.16.0", "", {}, "sha512-Zz+aZWSj8LE6zoxD+xrjh4VfkIG8Ya6LvYkZqtUQGJPZjYl53ypCaUwWqo7eI0x66KBGeRo+mlBEkMSeSZ38Nw=="],
|
||||||
|
|
||||||
"uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
|
"uuid": ["uuid@9.0.1", "", { "bin": { "uuid": "dist/bin/uuid" } }, "sha512-b+1eJOlsR9K8HJpow9Ok3fiWOWSIcIzXodvv0rQjVoOVNpWMpxf1wZNpt4y9h10odCNrqnYp1OBzRktckBe3sA=="],
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,105 @@
|
|||||||
|
Specifying packages in a monorepo
|
||||||
|
Declaring directories for packages
|
||||||
|
First, your package manager needs to describe the locations of your packages. We recommend starting with splitting your packages into apps/ for applications and services and packages/ for everything else, like libraries and tooling.
|
||||||
|
|
||||||
|
pnpm
|
||||||
|
yarn
|
||||||
|
npm
|
||||||
|
bun
|
||||||
|
./package.json
|
||||||
|
|
||||||
|
{
|
||||||
|
"workspaces": [
|
||||||
|
"apps/*",
|
||||||
|
"packages/*"
|
||||||
|
]
|
||||||
|
}
|
||||||
|
→
|
||||||
|
bun workspace documentation
|
||||||
|
Using this configuration, every directory with a package.json in the apps or packages directories will be considered a package.
|
||||||
|
|
||||||
|
Turborepo does not support nested packages like apps/** or packages/** due to ambiguous behavior among package managers in the JavaScript ecosystem. Using a structure that would put a package at apps/a and another at apps/a/b will result in an error.If you'd like to group packages by directory, you can do this using globs like packages/* and packages/group/* and not creating a packages/group/package.json file.
|
||||||
|
package.json in each package
|
||||||
|
In the directory of the package, there must be a package.json to make the package discoverable to your package manager and turbo. The requirements for the package.json of a package are below.
|
||||||
|
|
||||||
|
Root package.json
|
||||||
|
The root package.json is the base for your workspace. Below is a common example of what you would find in a root package.json:
|
||||||
|
|
||||||
|
pnpm
|
||||||
|
yarn
|
||||||
|
npm
|
||||||
|
bun
|
||||||
|
./package.json
|
||||||
|
|
||||||
|
{
|
||||||
|
"private": true,
|
||||||
|
"scripts": {
|
||||||
|
"build": "turbo run build",
|
||||||
|
"dev": "turbo run dev",
|
||||||
|
"lint": "turbo run lint"
|
||||||
|
},
|
||||||
|
"devDependencies": {
|
||||||
|
"turbo": "latest"
|
||||||
|
},
|
||||||
|
"packageManager": "bun@1.2.0",
|
||||||
|
"workspaces": ["apps/*", "packages/*"]
|
||||||
|
}
|
||||||
|
Root turbo.json
|
||||||
|
turbo.json is used to configure the behavior of turbo. To learn more about how to configure your tasks, visit the Configuring tasks page.
|
||||||
|
|
||||||
|
Package manager lockfile
|
||||||
|
A lockfile is key to reproducible behavior for both your package manager and turbo. Additionally, Turborepo uses the lockfile to understand the dependencies between your Internal Packages within your Workspace.
|
||||||
|
|
||||||
|
If you do not have a lockfile present when you run turbo, you may see unpredictable behavior.
|
||||||
|
Anatomy of a package
|
||||||
|
It's often best to start thinking about designing a package as its own unit within the Workspace. At a high-level, each package is almost like its own small "project", with its own package.json, tooling configuration, and source code. There are limits to this idea—but its a good mental model to start from.
|
||||||
|
|
||||||
|
Additionally, a package has specific entrypoints that other packages in your Workspace can use to access the package, specified by exports.
|
||||||
|
|
||||||
|
package.json for a package
|
||||||
|
name
|
||||||
|
The name field is used to identify the package. It should be unique within your workspace.
|
||||||
|
|
||||||
|
It's best practice to use a namespace prefix for your Internal Packages to avoid conflicts with other packages on the npm registry. For example, if your organization is named acme, you might name your packages @acme/package-name.We use @repo in our docs and examples because it is an unused, unclaimable namespace on the npm registry. You can choose to keep it or use your own prefix.
|
||||||
|
scripts
|
||||||
|
The scripts field is used to define scripts that can be run in the package's context. Turborepo will use the name of these scripts to identify what scripts to run (if any) in a package. We talk more about these scripts on the Running Tasks page.
|
||||||
|
|
||||||
|
exports
|
||||||
|
The exports field is used to specify the entrypoints for other packages that want to use the package. When you want to use code from one package in another package, you'll import from that entrypoint.
|
||||||
|
|
||||||
|
For example, if you had a @repo/math package, you might have the following exports field:
|
||||||
|
|
||||||
|
./packages/math/package.json
|
||||||
|
|
||||||
|
{
|
||||||
|
"exports": {
|
||||||
|
".": "./src/constants.ts",
|
||||||
|
"./add": "./src/add.ts",
|
||||||
|
"./subtract": "./src/subtract.ts"
|
||||||
|
}
|
||||||
|
}
|
||||||
|
Note that this example uses the Just-in-Time Package pattern for simplicity. It exports TypeScript directly, but you might choose to use the Compiled Package pattern instead.
|
||||||
|
|
||||||
|
The exports field in this example requires modern versions of Node.js and TypeScript.
|
||||||
|
This would allow you to import add and subtract functions from the @repo/math package like so:
|
||||||
|
|
||||||
|
./apps/my-app/src/index.ts
|
||||||
|
|
||||||
|
import { GRAVITATIONAL_CONSTANT, SPEED_OF_LIGHT } from '@repo/math';
|
||||||
|
import { add } from '@repo/math/add';
|
||||||
|
import { subtract } from '@repo/math/subtract';
|
||||||
|
Using exports this way provides three major benefits:
|
||||||
|
|
||||||
|
Avoiding barrel files: Barrel files are files that re-export other files in the same package, creating one entrypoint for the entire package. While they might appear convenient, they're difficult for compilers and bundlers to handle and can quickly lead to performance problems.
|
||||||
|
More powerful features: exports also has other powerful features compared to the main field like Conditional Exports. In general, we recommend using exports over main whenever possible as it is the more modern option.
|
||||||
|
IDE autocompletion: By specifying the entrypoints for your package using exports, you can ensure that your code editor can provide auto-completion for the package's exports.
|
||||||
|
imports (optional)
|
||||||
|
The imports field gives you a way to create subpaths to other modules within your package. You can think of these like "shortcuts" to write simpler import paths that are more resilient to refactors that move files. To learn how, visit the TypeScript page.
|
||||||
|
|
||||||
|
You may be more familiar with TypeScript's compilerOptions#paths option, which accomplishes a similar goal. As of TypeScript 5.4, TypeScript can infer subpaths from imports, making it a better option since you'll be working with Node.js conventions. For more information, visit our TypeScript guide.
|
||||||
|
Source code
|
||||||
|
Of course, you'll want some source code in your package. Packages commonly use an src directory to store their source code and compile to a dist directory (that should also be located within the package), although this is not a requirement.
|
||||||
|
|
||||||
|
Common pitfalls
|
||||||
|
If you're using TypeScript, you likely don't need a tsconfig.json in the root of your workspace. Packages should independently specify their own configurations, usually building off of a shared tsconfig.json from a separate package in the workspace. For more information, visit the TypeScript guide.
|
||||||
|
You want to avoid accessing files across package boundaries as much as possible. If you ever find yourself writing ../ to get from one package to another, you likely have an opportunity to re-think your approach by installing the package where it's needed and importing it into your code.
|
||||||
+31
-26
@@ -1,28 +1,33 @@
|
|||||||
{
|
{
|
||||||
"name": "basango",
|
"name": "basango",
|
||||||
"private": true,
|
"private": true,
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"build": "turbo run build",
|
"build": "turbo run build",
|
||||||
"clean": "git clean -xdf node_modules",
|
"clean": "git clean -xdf node_modules",
|
||||||
"clean:workspaces": "turbo run clean",
|
"clean:workspaces": "turbo run clean",
|
||||||
"dev": "turbo run dev --parallel",
|
"dev": "turbo run dev --parallel",
|
||||||
"test": "turbo run test --parallel",
|
"test": "turbo run test --parallel",
|
||||||
"lint": "turbo run lint && manypkg check",
|
"lint": "turbo run lint && manypkg check",
|
||||||
"format": "biome format --write .",
|
"format": "biome format --write .",
|
||||||
"typecheck": "turbo run typecheck"
|
"typecheck": "turbo run typecheck"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"@biomejs/biome": "^2.3.1",
|
"@biomejs/biome": "^2.3.1",
|
||||||
"@manypkg/cli": "^0.25.1",
|
"@manypkg/cli": "^0.25.1",
|
||||||
"turbo": "^2.5.8",
|
"turbo": "^2.5.8",
|
||||||
"typescript": "5.9.2"
|
"typescript": "catalog:"
|
||||||
},
|
},
|
||||||
"engines": {
|
"engines": {
|
||||||
"node": ">=22"
|
"node": ">=22"
|
||||||
},
|
},
|
||||||
"packageManager": "bun@1.2.8",
|
"packageManager": "bun@1.3.1",
|
||||||
"workspaces": [
|
"workspaces": [
|
||||||
"apps/*",
|
"apps/*",
|
||||||
"packages/*"
|
"packages/*"
|
||||||
]
|
],
|
||||||
|
"catalog": {
|
||||||
|
"typescript": "^5.9.3",
|
||||||
|
"@types/bun": "^1.3.1",
|
||||||
|
"zod": "^4.0.0"
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -5,6 +5,6 @@ export default {
|
|||||||
out: "./migrations",
|
out: "./migrations",
|
||||||
dialect: "postgresql",
|
dialect: "postgresql",
|
||||||
dbCredentials: {
|
dbCredentials: {
|
||||||
url: process.env.DATABASE_SESSION_POOLER!,
|
url: process.env.DATABASE_URL!,
|
||||||
},
|
},
|
||||||
} satisfies Config;
|
} satisfies Config;
|
||||||
@@ -1,17 +1,23 @@
|
|||||||
{
|
{
|
||||||
"name": "@basango/db",
|
"name": "@basango/db",
|
||||||
"version": "1.0.0",
|
"private": true,
|
||||||
"main": "index.ts",
|
"exports": {
|
||||||
"author": "",
|
"./client": "./src/client.ts",
|
||||||
"license": "ISC",
|
"./schema": "./src/schema.ts",
|
||||||
"description": "",
|
"./utils": "./src/utils/index.ts",
|
||||||
|
"./queries": "./src/queries/index.ts"
|
||||||
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
|
"@basango/logger": "workspace:*",
|
||||||
"@date-fns/utc": "^2.1.1",
|
"@date-fns/utc": "^2.1.1",
|
||||||
"drizzle-orm": "^0.44.7",
|
"drizzle-orm": "^0.44.7",
|
||||||
"pg": "^8.16.3",
|
"pg": "^8.16.3",
|
||||||
"snakecase-keys": "^9.0.2"
|
"snakecase-keys": "^9.0.2"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"drizzle-kit": "^0.31.6"
|
"@types/bun": "^1.3.1",
|
||||||
|
"@types/pg": "^8.15.6",
|
||||||
|
"drizzle-kit": "^0.31.6",
|
||||||
|
"typescript": "catalog:"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,6 +1,6 @@
|
|||||||
import { drizzle } from "drizzle-orm/node-postgres";
|
import { drizzle } from "drizzle-orm/node-postgres";
|
||||||
import { Pool } from "pg";
|
import { Pool } from "pg";
|
||||||
import * as schema from "@basango/db/schema";
|
import * as schema from "@db/schema";
|
||||||
|
|
||||||
const isDevelopment = process.env.NODE_ENV === "development";
|
const isDevelopment = process.env.NODE_ENV === "development";
|
||||||
|
|
||||||
|
|||||||
@@ -0,0 +1,2 @@
|
|||||||
|
export const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
||||||
|
export const PUBLICATION_GRAPH_DAYS = 180;
|
||||||
@@ -1,84 +0,0 @@
|
|||||||
import type { SQL } from "drizzle-orm";
|
|
||||||
import { and, desc, eq, sql } from "drizzle-orm";
|
|
||||||
|
|
||||||
import type { Database } from "@db/client";
|
|
||||||
import { articles, sources } from "@db/schema";
|
|
||||||
|
|
||||||
export interface ArticleExportRow {
|
|
||||||
articleId: string;
|
|
||||||
articleTitle: string;
|
|
||||||
articleLink: string;
|
|
||||||
articleCategories: string | null;
|
|
||||||
articleBody: string;
|
|
||||||
articleSource: string;
|
|
||||||
articleHash: string;
|
|
||||||
articlePublishedAt: string;
|
|
||||||
articleCrawledAt: string;
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface ArticleExportParams {
|
|
||||||
source?: string | null;
|
|
||||||
dateRange?: { start: number; end: number } | null;
|
|
||||||
batchSize?: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function* getArticlesForExport(
|
|
||||||
db: Database,
|
|
||||||
params: ArticleExportParams = {},
|
|
||||||
): AsyncGenerator<ArticleExportRow> {
|
|
||||||
const batchSize = params.batchSize && params.batchSize > 0
|
|
||||||
? params.batchSize
|
|
||||||
: 1000;
|
|
||||||
|
|
||||||
const filters: SQL[] = [];
|
|
||||||
|
|
||||||
if (params.source) {
|
|
||||||
filters.push(eq(sources.name, params.source));
|
|
||||||
}
|
|
||||||
|
|
||||||
if (params.dateRange) {
|
|
||||||
filters.push(
|
|
||||||
sql`${articles.publishedAt} BETWEEN to_timestamp(${params.dateRange.start}) AND to_timestamp(${params.dateRange.end})`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
let query = db
|
|
||||||
.select({
|
|
||||||
articleId: articles.id,
|
|
||||||
articleTitle: articles.title,
|
|
||||||
articleLink: articles.link,
|
|
||||||
articleCategories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
|
||||||
articleBody: articles.body,
|
|
||||||
articleSource: sources.name,
|
|
||||||
articleHash: articles.hash,
|
|
||||||
articlePublishedAt: articles.publishedAt,
|
|
||||||
articleCrawledAt: articles.crawledAt,
|
|
||||||
})
|
|
||||||
.from(articles)
|
|
||||||
.innerJoin(sources, eq(articles.sourceId, sources.id));
|
|
||||||
|
|
||||||
if (filters.length === 1) {
|
|
||||||
query = query.where(filters[0]);
|
|
||||||
} else if (filters.length > 1) {
|
|
||||||
query = query.where(and(...filters));
|
|
||||||
}
|
|
||||||
|
|
||||||
query = query.orderBy(desc(articles.publishedAt), desc(articles.id));
|
|
||||||
|
|
||||||
let offset = 0;
|
|
||||||
while (true) {
|
|
||||||
const rows = await query.limit(batchSize).offset(offset);
|
|
||||||
if (rows.length === 0) {
|
|
||||||
break;
|
|
||||||
}
|
|
||||||
|
|
||||||
for (const row of rows) {
|
|
||||||
yield {
|
|
||||||
...row,
|
|
||||||
articleCategories: row.articleCategories ?? null,
|
|
||||||
};
|
|
||||||
}
|
|
||||||
|
|
||||||
offset += batchSize;
|
|
||||||
}
|
|
||||||
}
|
|
||||||
@@ -1,87 +0,0 @@
|
|||||||
import { and, eq, sql } from "drizzle-orm";
|
|
||||||
|
|
||||||
import type { Database } from "@db/client";
|
|
||||||
import { articles, sources } from "@db/schema";
|
|
||||||
|
|
||||||
export interface SourceStatisticsRow {
|
|
||||||
sourceId: string;
|
|
||||||
sourceName: string;
|
|
||||||
sourceCrawledAt: string | null;
|
|
||||||
articlesCount: number;
|
|
||||||
articleMetadataAvailable: number;
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getSourceStatisticsList(
|
|
||||||
db: Database,
|
|
||||||
): Promise<SourceStatisticsRow[]> {
|
|
||||||
const rows = await db
|
|
||||||
.select({
|
|
||||||
sourceId: sources.id,
|
|
||||||
sourceName: sources.name,
|
|
||||||
sourceCrawledAt: sql<string | null>`max(${articles.crawledAt})`,
|
|
||||||
articlesCount: sql<number>`count(${articles.id})`,
|
|
||||||
articleMetadataAvailable: sql<number>`sum(CASE WHEN ${articles.metadata} IS NOT NULL THEN 1 ELSE 0 END)`,
|
|
||||||
})
|
|
||||||
.from(sources)
|
|
||||||
.leftJoin(articles, eq(articles.sourceId, sources.id))
|
|
||||||
.groupBy(sources.id, sources.name)
|
|
||||||
.orderBy(sources.name.asc());
|
|
||||||
|
|
||||||
return rows.map((row) => ({
|
|
||||||
sourceId: row.sourceId,
|
|
||||||
sourceName: row.sourceName,
|
|
||||||
sourceCrawledAt: row.sourceCrawledAt,
|
|
||||||
articlesCount: Number(row.articlesCount ?? 0),
|
|
||||||
articleMetadataAvailable: Number(row.articleMetadataAvailable ?? 0),
|
|
||||||
}));
|
|
||||||
}
|
|
||||||
|
|
||||||
export interface PublicationDateParams {
|
|
||||||
source: string;
|
|
||||||
category?: string | null;
|
|
||||||
}
|
|
||||||
|
|
||||||
async function selectPublicationBoundary(
|
|
||||||
db: Database,
|
|
||||||
fn: "min" | "max",
|
|
||||||
params: PublicationDateParams,
|
|
||||||
): Promise<string> {
|
|
||||||
const conditions = [eq(sources.name, params.source)];
|
|
||||||
|
|
||||||
if (params.category) {
|
|
||||||
conditions.push(
|
|
||||||
sql`${params.category} = ANY(${articles.categories})`,
|
|
||||||
);
|
|
||||||
}
|
|
||||||
|
|
||||||
const whereClause = conditions.length > 1
|
|
||||||
? and(...conditions)
|
|
||||||
: conditions[0];
|
|
||||||
|
|
||||||
const [result] = await db
|
|
||||||
.select({
|
|
||||||
boundary:
|
|
||||||
fn === "min"
|
|
||||||
? sql<string | null>`min(${articles.publishedAt})`
|
|
||||||
: sql<string | null>`max(${articles.publishedAt})`,
|
|
||||||
})
|
|
||||||
.from(articles)
|
|
||||||
.innerJoin(sources, eq(articles.sourceId, sources.id))
|
|
||||||
.where(whereClause);
|
|
||||||
|
|
||||||
return result?.boundary ?? new Date().toISOString();
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getEarliestPublicationDate(
|
|
||||||
db: Database,
|
|
||||||
params: PublicationDateParams,
|
|
||||||
): Promise<string> {
|
|
||||||
return selectPublicationBoundary(db, "min", params);
|
|
||||||
}
|
|
||||||
|
|
||||||
export async function getLatestPublicationDate(
|
|
||||||
db: Database,
|
|
||||||
params: PublicationDateParams,
|
|
||||||
): Promise<string> {
|
|
||||||
return selectPublicationBoundary(db, "max", params);
|
|
||||||
}
|
|
||||||
+104
-32
@@ -1,18 +1,9 @@
|
|||||||
import type { SQL, AnyColumn } from "drizzle-orm";
|
import type { AnyColumn, SQL } from "drizzle-orm";
|
||||||
import {
|
import { and, asc, desc, eq, gt, lt, or, sql } from "drizzle-orm";
|
||||||
and,
|
|
||||||
asc,
|
|
||||||
desc,
|
|
||||||
eq,
|
|
||||||
gt,
|
|
||||||
lt,
|
|
||||||
or,
|
|
||||||
sql,
|
|
||||||
} from "drizzle-orm";
|
|
||||||
|
|
||||||
import type { Database } from "@db/client";
|
import type { Database } from "@db/client";
|
||||||
import {
|
import {
|
||||||
appUsers,
|
users,
|
||||||
articles,
|
articles,
|
||||||
bookmarkArticles,
|
bookmarkArticles,
|
||||||
bookmarks,
|
bookmarks,
|
||||||
@@ -104,6 +95,86 @@ interface NormalizedArticleFilters {
|
|||||||
sortDirection: SortDirection;
|
sortDirection: SortDirection;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface ArticleExportRow {
|
||||||
|
articleId: string;
|
||||||
|
articleTitle: string;
|
||||||
|
articleLink: string;
|
||||||
|
articleCategories: string | null;
|
||||||
|
articleBody: string;
|
||||||
|
articleSource: string;
|
||||||
|
articleHash: string;
|
||||||
|
articlePublishedAt: string;
|
||||||
|
articleCrawledAt: string;
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface ArticleExportParams {
|
||||||
|
source?: string | null;
|
||||||
|
dateRange?: { start: number; end: number } | null;
|
||||||
|
batchSize?: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function* getArticlesForExport(
|
||||||
|
db: Database,
|
||||||
|
params: ArticleExportParams = {},
|
||||||
|
): AsyncGenerator<ArticleExportRow> {
|
||||||
|
const batchSize =
|
||||||
|
params.batchSize && params.batchSize > 0 ? params.batchSize : 1000;
|
||||||
|
|
||||||
|
const filters: SQL[] = [];
|
||||||
|
|
||||||
|
if (params.source) {
|
||||||
|
filters.push(eq(sources.name, params.source));
|
||||||
|
}
|
||||||
|
|
||||||
|
if (params.dateRange) {
|
||||||
|
filters.push(
|
||||||
|
sql`${articles.publishedAt} BETWEEN to_timestamp(${params.dateRange.start}) AND to_timestamp(${params.dateRange.end})`,
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
let query = db
|
||||||
|
.select({
|
||||||
|
articleId: articles.id,
|
||||||
|
articleTitle: articles.title,
|
||||||
|
articleLink: articles.link,
|
||||||
|
articleCategories: sql<
|
||||||
|
string | null
|
||||||
|
>`array_to_string(${articles.categories}, ',')`,
|
||||||
|
articleBody: articles.body,
|
||||||
|
articleSource: sources.name,
|
||||||
|
articleHash: articles.hash,
|
||||||
|
articlePublishedAt: articles.publishedAt,
|
||||||
|
articleCrawledAt: articles.crawledAt,
|
||||||
|
})
|
||||||
|
.from(articles)
|
||||||
|
.innerJoin(sources, eq(articles.sourceId, sources.id));
|
||||||
|
|
||||||
|
if (filters.length === 1) {
|
||||||
|
query = query.where(filters[0]);
|
||||||
|
} else if (filters.length > 1) {
|
||||||
|
query = query.where(and(...filters));
|
||||||
|
}
|
||||||
|
|
||||||
|
query = query.orderBy(desc(articles.publishedAt), desc(articles.id));
|
||||||
|
|
||||||
|
let offset = 0;
|
||||||
|
while (true) {
|
||||||
|
const rows = await query.limit(batchSize).offset(offset);
|
||||||
|
if (rows.length === 0) {
|
||||||
|
break;
|
||||||
|
}
|
||||||
|
|
||||||
|
for (const row of rows) {
|
||||||
|
yield {
|
||||||
|
...row,
|
||||||
|
articleCategories: row.articleCategories ?? null,
|
||||||
|
};
|
||||||
|
}
|
||||||
|
|
||||||
|
offset += batchSize;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
||||||
|
|
||||||
function normalizeArticleFilters(
|
function normalizeArticleFilters(
|
||||||
@@ -113,7 +184,8 @@ function normalizeArticleFilters(
|
|||||||
const trimmedCategory = filters?.category?.trim();
|
const trimmedCategory = filters?.category?.trim();
|
||||||
|
|
||||||
return {
|
return {
|
||||||
search: trimmedSearch && trimmedSearch.length > 0 ? trimmedSearch : undefined,
|
search:
|
||||||
|
trimmedSearch && trimmedSearch.length > 0 ? trimmedSearch : undefined,
|
||||||
category:
|
category:
|
||||||
trimmedCategory && trimmedCategory.length > 0
|
trimmedCategory && trimmedCategory.length > 0
|
||||||
? trimmedCategory
|
? trimmedCategory
|
||||||
@@ -123,9 +195,10 @@ function normalizeArticleFilters(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
|
|
||||||
function buildArticleFilterConditions(
|
function buildArticleFilterConditions(filters: NormalizedArticleFilters): {
|
||||||
filters: NormalizedArticleFilters,
|
conditions: SQL[];
|
||||||
): { conditions: SQL[]; searchQuery?: string } {
|
searchQuery?: string;
|
||||||
|
} {
|
||||||
const conditions: SQL[] = [];
|
const conditions: SQL[] = [];
|
||||||
let searchQuery: string | undefined;
|
let searchQuery: string | undefined;
|
||||||
|
|
||||||
@@ -181,7 +254,9 @@ async function fetchArticleOverview(
|
|||||||
article_id: articles.id,
|
article_id: articles.id,
|
||||||
article_title: articles.title,
|
article_title: articles.title,
|
||||||
article_link: articles.link,
|
article_link: articles.link,
|
||||||
article_categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
article_categories: sql<
|
||||||
|
string | null
|
||||||
|
>`array_to_string(${articles.categories}, ',')`,
|
||||||
article_excerpt: articles.excerpt,
|
article_excerpt: articles.excerpt,
|
||||||
article_published_at: articles.publishedAt,
|
article_published_at: articles.publishedAt,
|
||||||
article_image: articles.image,
|
article_image: articles.image,
|
||||||
@@ -242,9 +317,7 @@ async function fetchArticleOverview(
|
|||||||
orderings.push(desc(articles.publishedAt), desc(articles.id));
|
orderings.push(desc(articles.publishedAt), desc(articles.id));
|
||||||
}
|
}
|
||||||
|
|
||||||
const rows = await query
|
const rows = await query.orderBy(...orderings).limit(options.page.limit + 1);
|
||||||
.orderBy(...orderings)
|
|
||||||
.limit(options.page.limit + 1);
|
|
||||||
|
|
||||||
return buildPaginationResult(rows, options.page, {
|
return buildPaginationResult(rows, options.page, {
|
||||||
id: "article_id",
|
id: "article_id",
|
||||||
@@ -314,7 +387,9 @@ export async function getBookmarkedArticleList(
|
|||||||
article_id: articles.id,
|
article_id: articles.id,
|
||||||
article_title: articles.title,
|
article_title: articles.title,
|
||||||
article_link: articles.link,
|
article_link: articles.link,
|
||||||
article_categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
article_categories: sql<
|
||||||
|
string | null
|
||||||
|
>`array_to_string(${articles.categories}, ',')`,
|
||||||
article_excerpt: articles.excerpt,
|
article_excerpt: articles.excerpt,
|
||||||
article_published_at: articles.publishedAt,
|
article_published_at: articles.publishedAt,
|
||||||
article_image: articles.image,
|
article_image: articles.image,
|
||||||
@@ -377,9 +452,7 @@ export async function getBookmarkedArticleList(
|
|||||||
orderings.push(desc(articles.publishedAt), desc(articles.id));
|
orderings.push(desc(articles.publishedAt), desc(articles.id));
|
||||||
}
|
}
|
||||||
|
|
||||||
const rows = await query
|
const rows = await query.orderBy(...orderings).limit(page.limit + 1);
|
||||||
.orderBy(...orderings)
|
|
||||||
.limit(page.limit + 1);
|
|
||||||
|
|
||||||
return buildPaginationResult(rows, page, {
|
return buildPaginationResult(rows, page, {
|
||||||
id: "article_id",
|
id: "article_id",
|
||||||
@@ -398,7 +471,9 @@ export async function getArticleDetails(
|
|||||||
article_id: articles.id,
|
article_id: articles.id,
|
||||||
article_title: articles.title,
|
article_title: articles.title,
|
||||||
article_link: articles.link,
|
article_link: articles.link,
|
||||||
article_categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
article_categories: sql<
|
||||||
|
string | null
|
||||||
|
>`array_to_string(${articles.categories}, ',')`,
|
||||||
article_body: articles.body,
|
article_body: articles.body,
|
||||||
article_hash: articles.hash,
|
article_hash: articles.hash,
|
||||||
article_published_at: articles.publishedAt,
|
article_published_at: articles.publishedAt,
|
||||||
@@ -442,10 +517,7 @@ export async function getArticleCommentList(
|
|||||||
whereConditions.push(
|
whereConditions.push(
|
||||||
or(
|
or(
|
||||||
lt(comments.createdAt, cursor.date),
|
lt(comments.createdAt, cursor.date),
|
||||||
and(
|
and(eq(comments.createdAt, cursor.date), lt(comments.id, cursor.id)),
|
||||||
eq(comments.createdAt, cursor.date),
|
|
||||||
lt(comments.id, cursor.id),
|
|
||||||
),
|
|
||||||
),
|
),
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
@@ -456,11 +528,11 @@ export async function getArticleCommentList(
|
|||||||
comment_content: comments.content,
|
comment_content: comments.content,
|
||||||
comment_sentiment: comments.sentiment,
|
comment_sentiment: comments.sentiment,
|
||||||
comment_created_at: comments.createdAt,
|
comment_created_at: comments.createdAt,
|
||||||
user_id: appUsers.id,
|
user_id: users.id,
|
||||||
user_name: appUsers.name,
|
user_name: users.name,
|
||||||
})
|
})
|
||||||
.from(comments)
|
.from(comments)
|
||||||
.innerJoin(appUsers, eq(comments.userId, appUsers.id));
|
.innerJoin(users, eq(comments.userId, users.id));
|
||||||
|
|
||||||
if (whereConditions.length === 1) {
|
if (whereConditions.length === 1) {
|
||||||
query = query.where(whereConditions[0]);
|
query = query.where(whereConditions[0]);
|
||||||
@@ -0,0 +1,4 @@
|
|||||||
|
export * from "./articles";
|
||||||
|
export * from "./bookmarks";
|
||||||
|
export * from "./sources";
|
||||||
|
export * from "./users";
|
||||||
+125
-19
@@ -9,11 +9,8 @@ import {
|
|||||||
decodeCursor,
|
decodeCursor,
|
||||||
type PageRequest,
|
type PageRequest,
|
||||||
type PaginationMeta,
|
type PaginationMeta,
|
||||||
type PageState,
|
|
||||||
} from "@db/utils/pagination";
|
} from "@db/utils/pagination";
|
||||||
|
import { PUBLICATION_GRAPH_DAYS, SOURCE_IMAGE_BASE } from "@db/constant";
|
||||||
const SOURCE_IMAGE_BASE = "https://devscast.org/images/sources/";
|
|
||||||
const PUBLICATION_GRAPH_DAYS = 180;
|
|
||||||
|
|
||||||
export interface SourceOverviewRow {
|
export interface SourceOverviewRow {
|
||||||
source_id: string;
|
source_id: string;
|
||||||
@@ -62,12 +59,97 @@ export interface SourceDetailsResult {
|
|||||||
categoryShares: CategoryShare[];
|
categoryShares: CategoryShare[];
|
||||||
}
|
}
|
||||||
|
|
||||||
|
export interface SourceStatisticsRow {
|
||||||
|
sourceId: string;
|
||||||
|
sourceName: string;
|
||||||
|
sourceCrawledAt: string | null;
|
||||||
|
articlesCount: number;
|
||||||
|
articleMetadataAvailable: number;
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getSourceStatisticsList(
|
||||||
|
db: Database,
|
||||||
|
): Promise<SourceStatisticsRow[]> {
|
||||||
|
const rows = await db
|
||||||
|
.select({
|
||||||
|
sourceId: sources.id,
|
||||||
|
sourceName: sources.name,
|
||||||
|
sourceCrawledAt: sql<string | null>`max
|
||||||
|
(${articles.crawledAt})`,
|
||||||
|
articlesCount: sql<number>`count
|
||||||
|
(${articles.id})`,
|
||||||
|
articleMetadataAvailable: sql<number>`sum
|
||||||
|
(CASE WHEN ${articles.metadata} IS NOT NULL THEN 1 ELSE 0 END)`,
|
||||||
|
})
|
||||||
|
.from(sources)
|
||||||
|
.leftJoin(articles, eq(articles.sourceId, sources.id))
|
||||||
|
.groupBy(sources.id, sources.name)
|
||||||
|
.orderBy(sources.name.asc());
|
||||||
|
|
||||||
|
return rows.map((row) => ({
|
||||||
|
sourceId: row.sourceId,
|
||||||
|
sourceName: row.sourceName,
|
||||||
|
sourceCrawledAt: row.sourceCrawledAt,
|
||||||
|
articlesCount: Number(row.articlesCount ?? 0),
|
||||||
|
articleMetadataAvailable: Number(row.articleMetadataAvailable ?? 0),
|
||||||
|
}));
|
||||||
|
}
|
||||||
|
|
||||||
|
export interface PublicationDateParams {
|
||||||
|
source: string;
|
||||||
|
category?: string | null;
|
||||||
|
}
|
||||||
|
|
||||||
|
async function selectPublicationBoundary(
|
||||||
|
db: Database,
|
||||||
|
fn: "min" | "max",
|
||||||
|
params: PublicationDateParams,
|
||||||
|
): Promise<string> {
|
||||||
|
const conditions: SQL[] = [eq(sources.name, params.source)];
|
||||||
|
|
||||||
|
if (params.category) {
|
||||||
|
conditions.push(sql`${params.category} = ANY(${articles.categories})`);
|
||||||
|
}
|
||||||
|
|
||||||
|
const whereClause =
|
||||||
|
conditions.length > 1 ? and(...conditions) : conditions[0];
|
||||||
|
|
||||||
|
const [result] = await db
|
||||||
|
.select({
|
||||||
|
boundary:
|
||||||
|
fn === "min"
|
||||||
|
? sql<string | null>`min
|
||||||
|
(${articles.publishedAt})`
|
||||||
|
: sql<string | null>`max
|
||||||
|
(${articles.publishedAt})`,
|
||||||
|
})
|
||||||
|
.from(articles)
|
||||||
|
.innerJoin(sources, eq(articles.sourceId, sources.id))
|
||||||
|
.where(whereClause);
|
||||||
|
|
||||||
|
return result?.boundary ?? new Date().toISOString();
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getEarliestPublicationDate(
|
||||||
|
db: Database,
|
||||||
|
params: PublicationDateParams,
|
||||||
|
): Promise<string> {
|
||||||
|
return selectPublicationBoundary(db, "min", params);
|
||||||
|
}
|
||||||
|
|
||||||
|
export async function getLatestPublicationDate(
|
||||||
|
db: Database,
|
||||||
|
params: PublicationDateParams,
|
||||||
|
): Promise<string> {
|
||||||
|
return selectPublicationBoundary(db, "max", params);
|
||||||
|
}
|
||||||
|
|
||||||
function buildFollowExistsExpression(userId: string): SQL<boolean> {
|
function buildFollowExistsExpression(userId: string): SQL<boolean> {
|
||||||
return sql`EXISTS (
|
return sql`EXISTS
|
||||||
SELECT 1
|
(SELECT 1
|
||||||
FROM ${followedSources} f
|
FROM ${followedSources} f
|
||||||
WHERE f.source_id = ${sources.id} AND f.follower_id = ${userId}
|
WHERE f.source_id = ${sources.id}
|
||||||
)`;
|
AND f.follower_id = ${userId})`;
|
||||||
}
|
}
|
||||||
|
|
||||||
export async function getSourceOverviewList(
|
export async function getSourceOverviewList(
|
||||||
@@ -126,16 +208,27 @@ async function fetchPublicationGraph(
|
|||||||
|
|
||||||
const rows = await db
|
const rows = await db
|
||||||
.select({
|
.select({
|
||||||
day: sql<string>`date(${articles.publishedAt})`,
|
day: sql<string>`date
|
||||||
count: sql<number>`count(${articles.id})`,
|
(${articles.publishedAt})`,
|
||||||
|
count: sql<number>`count
|
||||||
|
(${articles.id})`,
|
||||||
})
|
})
|
||||||
.from(articles)
|
.from(articles)
|
||||||
.where(eq(articles.sourceId, sourceId))
|
.where(eq(articles.sourceId, sourceId))
|
||||||
.where(
|
.where(
|
||||||
sql`${articles.publishedAt} BETWEEN to_timestamp(${range.start}) AND to_timestamp(${range.end})`,
|
sql`${articles.publishedAt} BETWEEN to_timestamp(
|
||||||
|
${range.start}
|
||||||
|
)
|
||||||
|
AND
|
||||||
|
to_timestamp
|
||||||
|
(
|
||||||
|
${range.end}
|
||||||
|
)`,
|
||||||
)
|
)
|
||||||
.groupBy(sql`date(${articles.publishedAt})`)
|
.groupBy(sql`date
|
||||||
.orderBy(sql`date(${articles.publishedAt})`);
|
(${articles.publishedAt})`)
|
||||||
|
.orderBy(sql`date
|
||||||
|
(${articles.publishedAt})`);
|
||||||
|
|
||||||
const counts = new Map<string, number>();
|
const counts = new Map<string, number>();
|
||||||
for (const row of rows) {
|
for (const row of rows) {
|
||||||
@@ -164,7 +257,8 @@ async function fetchCategoryShares(
|
|||||||
): Promise<CategoryShare[]> {
|
): Promise<CategoryShare[]> {
|
||||||
const rows = await db
|
const rows = await db
|
||||||
.select({
|
.select({
|
||||||
categories: sql<string | null>`array_to_string(${articles.categories}, ',')`,
|
categories: sql<string | null>`array_to_string
|
||||||
|
(${articles.categories}, ',')`,
|
||||||
})
|
})
|
||||||
.from(articles)
|
.from(articles)
|
||||||
.where(eq(articles.sourceId, sourceId));
|
.where(eq(articles.sourceId, sourceId));
|
||||||
@@ -179,7 +273,10 @@ async function fetchCategoryShares(
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
const total = Array.from(counts.values()).reduce((acc, value) => acc + value, 0);
|
const total = Array.from(counts.values()).reduce(
|
||||||
|
(acc, value) => acc + value,
|
||||||
|
0,
|
||||||
|
);
|
||||||
|
|
||||||
const shares: CategoryShare[] = Array.from(counts.entries()).map(
|
const shares: CategoryShare[] = Array.from(counts.entries()).map(
|
||||||
([category, count]) => ({
|
([category, count]) => ({
|
||||||
@@ -211,9 +308,18 @@ export async function getSourceDetails(
|
|||||||
source_reliability: sources.reliability,
|
source_reliability: sources.reliability,
|
||||||
source_transparency: sources.transparency,
|
source_transparency: sources.transparency,
|
||||||
source_image: sql<string>`('${SOURCE_IMAGE_BASE}' || ${sources.name} || '.png')`,
|
source_image: sql<string>`('${SOURCE_IMAGE_BASE}' || ${sources.name} || '.png')`,
|
||||||
articles_count: sql<number>`count(${articles.id})`,
|
articles_count: sql<number>`count
|
||||||
source_crawled_at: sql<string | null>`max(${articles.crawledAt})`,
|
(${articles.id})`,
|
||||||
articles_metadata_available: sql<number>`count(*) FILTER (WHERE ${articles.metadata} IS NOT NULL)`,
|
source_crawled_at: sql<string | null>`max
|
||||||
|
(${articles.crawledAt})`,
|
||||||
|
articles_metadata_available: sql<number>`count
|
||||||
|
(*)
|
||||||
|
FILTER (WHERE
|
||||||
|
${articles.metadata}
|
||||||
|
IS
|
||||||
|
NOT
|
||||||
|
NULL
|
||||||
|
)`,
|
||||||
source_is_followed: followExpression,
|
source_is_followed: followExpression,
|
||||||
})
|
})
|
||||||
.from(sources)
|
.from(sources)
|
||||||
+8
-8
@@ -1,7 +1,7 @@
|
|||||||
import { eq } from "drizzle-orm";
|
import { eq } from "drizzle-orm";
|
||||||
|
|
||||||
import type { Database } from "@db/client";
|
import type { Database } from "@db/client";
|
||||||
import { appUsers } from "@db/schema";
|
import { users } from "@db/schema";
|
||||||
|
|
||||||
export interface UserProfileRow {
|
export interface UserProfileRow {
|
||||||
user_id: string;
|
user_id: string;
|
||||||
@@ -17,14 +17,14 @@ export async function getUserProfile(
|
|||||||
): Promise<UserProfileRow | null> {
|
): Promise<UserProfileRow | null> {
|
||||||
const [row] = await db
|
const [row] = await db
|
||||||
.select({
|
.select({
|
||||||
user_id: appUsers.id,
|
user_id: users.id,
|
||||||
user_name: appUsers.name,
|
user_name: users.name,
|
||||||
user_email: appUsers.email,
|
user_email: users.email,
|
||||||
user_created_at: appUsers.createdAt,
|
user_created_at: users.createdAt,
|
||||||
user_updated_at: appUsers.updatedAt,
|
user_updated_at: users.updatedAt,
|
||||||
})
|
})
|
||||||
.from(appUsers)
|
.from(users)
|
||||||
.where(eq(appUsers.id, params.userId))
|
.where(eq(users.id, params.userId))
|
||||||
.limit(1);
|
.limit(1);
|
||||||
|
|
||||||
return row ?? null;
|
return row ?? null;
|
||||||
+49
-3288
File diff suppressed because it is too large
Load Diff
@@ -7,14 +7,14 @@ import { randomBytes } from "node:crypto";
|
|||||||
export function generateApiKey(): string {
|
export function generateApiKey(): string {
|
||||||
// Generate 32 random bytes and convert to hex
|
// Generate 32 random bytes and convert to hex
|
||||||
const randomString = randomBytes(32).toString("hex");
|
const randomString = randomBytes(32).toString("hex");
|
||||||
return `mid_${randomString}`;
|
return `basango_${randomString}`;
|
||||||
}
|
}
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Validates if a string is a valid API key format
|
* Validates if a string is a valid API key format
|
||||||
* @param key The key to validate
|
* @param key The key to validate
|
||||||
* @returns True if the key starts with 'mid-' and has the correct length
|
* @returns True if the key starts with 'basango_' and has the correct length
|
||||||
*/
|
*/
|
||||||
export function isValidApiKeyFormat(key: string): boolean {
|
export function isValidApiKeyFormat(key: string): boolean {
|
||||||
return key.startsWith("mid_") && key.length === 68; // mid_ (4) + 64 hex chars
|
return key.startsWith("basango_") && key.length === 68; // basango_ (8) + 64 hex chars
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,6 @@
|
|||||||
|
import { sql } from "drizzle-orm";
|
||||||
|
import { db } from "@db/client";
|
||||||
|
|
||||||
|
export async function checkHealth() {
|
||||||
|
await db.execute(sql`SELECT 1`);
|
||||||
|
}
|
||||||
|
|||||||
@@ -0,0 +1,4 @@
|
|||||||
|
export * from "./api-keys";
|
||||||
|
export * from "./health";
|
||||||
|
export * from "./pagination";
|
||||||
|
export * from "./search-query";
|
||||||
@@ -32,13 +32,15 @@ const DEFAULT_LIMIT = 5;
|
|||||||
const MAX_LIMIT = 100;
|
const MAX_LIMIT = 100;
|
||||||
|
|
||||||
export function createPageState(request: PageRequest = {}): PageState {
|
export function createPageState(request: PageRequest = {}): PageState {
|
||||||
const page = Number.isFinite(request.page) && (request.page ?? 0) > 0
|
const page =
|
||||||
? Math.trunc(request.page!)
|
Number.isFinite(request.page) && (request.page ?? 0) > 0
|
||||||
: DEFAULT_PAGE;
|
? Math.trunc(request.page!)
|
||||||
|
: DEFAULT_PAGE;
|
||||||
|
|
||||||
let limit = Number.isFinite(request.limit) && (request.limit ?? 0) > 0
|
let limit =
|
||||||
? Math.trunc(request.limit!)
|
Number.isFinite(request.limit) && (request.limit ?? 0) > 0
|
||||||
: DEFAULT_LIMIT;
|
? Math.trunc(request.limit!)
|
||||||
|
: DEFAULT_LIMIT;
|
||||||
|
|
||||||
if (limit < DEFAULT_LIMIT) {
|
if (limit < DEFAULT_LIMIT) {
|
||||||
limit = DEFAULT_LIMIT;
|
limit = DEFAULT_LIMIT;
|
||||||
|
|||||||
@@ -1,11 +1,11 @@
|
|||||||
{
|
{
|
||||||
"extends": "@midday/tsconfig/base.json",
|
"extends": "@basango/tsconfig/base.json",
|
||||||
"include": ["src"],
|
"include": ["src"],
|
||||||
"exclude": ["node_modules"],
|
"exclude": ["node_modules"],
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"baseUrl": ".",
|
"baseUrl": ".",
|
||||||
"paths": {
|
"paths": {
|
||||||
"@/db": ["./src/*"]
|
"@db/*": ["./src/*"]
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -1,18 +1,18 @@
|
|||||||
{
|
{
|
||||||
"name": "@basango/logger",
|
"name": "@basango/logger",
|
||||||
"version": "0.0.1",
|
"version": "0.0.1",
|
||||||
"private": true,
|
"private": true,
|
||||||
"type": "module",
|
"type": "module",
|
||||||
"main": "src/index.ts",
|
"main": "src/index.ts",
|
||||||
"types": "src/index.ts",
|
"types": "src/index.ts",
|
||||||
"scripts": {
|
"scripts": {
|
||||||
"typecheck": "tsc --noEmit"
|
"typecheck": "tsc --noEmit"
|
||||||
},
|
},
|
||||||
"devDependencies": {
|
"devDependencies": {
|
||||||
"typescript": "^5.9.2"
|
"typescript": "catalog:"
|
||||||
},
|
},
|
||||||
"dependencies": {
|
"dependencies": {
|
||||||
"pino": "^10.1.0",
|
"pino": "^10.1.0",
|
||||||
"pino-pretty": "^13.1.2"
|
"pino-pretty": "^13.1.2"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,20 +1,20 @@
|
|||||||
import pino from "pino";
|
import pino from "pino";
|
||||||
|
|
||||||
export const logger = pino({
|
export const logger = pino({
|
||||||
level: process.env.LOG_LEVEL || "info",
|
level: process.env.LOG_LEVEL || "info",
|
||||||
// Use pretty printing in development, structured JSON in production
|
// Use pretty printing in development, structured JSON in production
|
||||||
...(process.env.NODE_ENV === "development" && {
|
...(process.env.NODE_ENV === "development" && {
|
||||||
transport: {
|
transport: {
|
||||||
target: "pino-pretty",
|
target: "pino-pretty",
|
||||||
options: {
|
options: {
|
||||||
colorize: true,
|
colorize: true,
|
||||||
translateTime: "HH:MM:ss",
|
translateTime: "HH:MM:ss",
|
||||||
ignore: "pid,hostname",
|
ignore: "pid,hostname",
|
||||||
messageFormat: true,
|
messageFormat: true,
|
||||||
hideObject: false,
|
hideObject: false,
|
||||||
},
|
},
|
||||||
},
|
},
|
||||||
}),
|
}),
|
||||||
});
|
});
|
||||||
|
|
||||||
export default logger;
|
export default logger;
|
||||||
|
|||||||
@@ -1,5 +1,5 @@
|
|||||||
{
|
{
|
||||||
"extends": "@basango/tsconfig/base.json",
|
"extends": "@basango/tsconfig/base.json",
|
||||||
"include": ["src/**/*"],
|
"include": ["src/**/*"],
|
||||||
"exclude": ["node_modules"],
|
"exclude": ["node_modules"]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,19 +1,19 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json.schemastore.org/tsconfig",
|
"$schema": "https://json.schemastore.org/tsconfig",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"declaration": true,
|
"declaration": true,
|
||||||
"declarationMap": true,
|
"declarationMap": true,
|
||||||
"esModuleInterop": true,
|
"esModuleInterop": true,
|
||||||
"incremental": false,
|
"incremental": false,
|
||||||
"isolatedModules": true,
|
"isolatedModules": true,
|
||||||
"lib": ["es2022", "DOM", "DOM.Iterable"],
|
"lib": ["es2022", "DOM", "DOM.Iterable"],
|
||||||
"module": "NodeNext",
|
"module": "NodeNext",
|
||||||
"moduleDetection": "force",
|
"moduleDetection": "force",
|
||||||
"moduleResolution": "NodeNext",
|
"moduleResolution": "NodeNext",
|
||||||
"noUncheckedIndexedAccess": true,
|
"noUncheckedIndexedAccess": true,
|
||||||
"resolveJsonModule": true,
|
"resolveJsonModule": true,
|
||||||
"skipLibCheck": true,
|
"skipLibCheck": true,
|
||||||
"strict": true,
|
"strict": true,
|
||||||
"target": "ES2022"
|
"target": "ES2022"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json.schemastore.org/tsconfig",
|
"$schema": "https://json.schemastore.org/tsconfig",
|
||||||
"extends": "./base.json",
|
"extends": "./base.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"plugins": [{ "name": "next" }],
|
"plugins": [{ "name": "next" }],
|
||||||
"module": "ESNext",
|
"module": "ESNext",
|
||||||
"moduleResolution": "Bundler",
|
"moduleResolution": "Bundler",
|
||||||
"allowJs": true,
|
"allowJs": true,
|
||||||
"jsx": "preserve",
|
"jsx": "preserve",
|
||||||
"noEmit": true
|
"noEmit": true
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,12 +1,12 @@
|
|||||||
{
|
{
|
||||||
"name": "@basango/tsconfig",
|
"name": "@basango/tsconfig",
|
||||||
"version": "0.0.0",
|
"version": "0.0.0",
|
||||||
"private": true,
|
"private": true,
|
||||||
"license": "MIT",
|
"license": "MIT",
|
||||||
"publishConfig": {
|
"publishConfig": {
|
||||||
"access": "public"
|
"access": "public"
|
||||||
},
|
},
|
||||||
"files": [
|
"files": [
|
||||||
"base.json"
|
"base.json"
|
||||||
]
|
]
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,7 +1,7 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://json.schemastore.org/tsconfig",
|
"$schema": "https://json.schemastore.org/tsconfig",
|
||||||
"extends": "./base.json",
|
"extends": "./base.json",
|
||||||
"compilerOptions": {
|
"compilerOptions": {
|
||||||
"jsx": "react-jsx"
|
"jsx": "react-jsx"
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -1,3 +1,3 @@
|
|||||||
{
|
{
|
||||||
"extends": "@basango/tsconfig/base.json"
|
"extends": "@basango/tsconfig/base.json"
|
||||||
}
|
}
|
||||||
|
|||||||
+41
-41
@@ -1,43 +1,43 @@
|
|||||||
{
|
{
|
||||||
"$schema": "https://turborepo.com/schema.json",
|
"$schema": "https://turborepo.com/schema.json",
|
||||||
"globalDependencies": ["**/.env"],
|
"globalDependencies": ["**/.env"],
|
||||||
"ui": "tui",
|
"ui": "tui",
|
||||||
"tasks": {
|
"tasks": {
|
||||||
"topo": {
|
"topo": {
|
||||||
"dependsOn": ["^topo"]
|
"dependsOn": ["^topo"]
|
||||||
},
|
},
|
||||||
"build": {
|
"build": {
|
||||||
"dependsOn": ["^build"],
|
"dependsOn": ["^build"],
|
||||||
"inputs": ["$TURBO_DEFAULT$", ".env*"],
|
"inputs": ["$TURBO_DEFAULT$", ".env*"],
|
||||||
"outputs": [
|
"outputs": [
|
||||||
".next/**",
|
".next/**",
|
||||||
"!.next/cache/**",
|
"!.next/cache/**",
|
||||||
"next-env.d.ts",
|
"next-env.d.ts",
|
||||||
".expo/**",
|
".expo/**",
|
||||||
"dist/**",
|
"dist/**",
|
||||||
"build/**",
|
"build/**",
|
||||||
"lib/**"
|
"lib/**"
|
||||||
],
|
],
|
||||||
"passThroughEnv": []
|
"passThroughEnv": []
|
||||||
},
|
},
|
||||||
"start": {
|
"start": {
|
||||||
"cache": false
|
"cache": false
|
||||||
},
|
},
|
||||||
"test": {
|
"test": {
|
||||||
"cache": false
|
"cache": false
|
||||||
},
|
},
|
||||||
"dev": {
|
"dev": {
|
||||||
"inputs": ["$TURBO_DEFAULT$", ".env"],
|
"inputs": ["$TURBO_DEFAULT$", ".env"],
|
||||||
"cache": false,
|
"cache": false,
|
||||||
"persistent": true
|
"persistent": true
|
||||||
},
|
},
|
||||||
"format": {},
|
"format": {},
|
||||||
"lint": {
|
"lint": {
|
||||||
"dependsOn": ["^topo"]
|
"dependsOn": ["^topo"]
|
||||||
},
|
},
|
||||||
"typecheck": {
|
"typecheck": {
|
||||||
"dependsOn": ["^topo"],
|
"dependsOn": ["^topo"],
|
||||||
"outputs": []
|
"outputs": []
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|||||||
@@ -0,0 +1,258 @@
|
|||||||
|
{
|
||||||
|
"lockfileVersion": 1,
|
||||||
|
"workspaces": {
|
||||||
|
"": {
|
||||||
|
"name": "basango-monorepo",
|
||||||
|
"devDependencies": {
|
||||||
|
"@types/node": "^20.11.30",
|
||||||
|
"typescript": "^5.4.0",
|
||||||
|
"vitest": "^1.6.0",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
"packages": {
|
||||||
|
"@esbuild/aix-ppc64": ["@esbuild/aix-ppc64@0.21.5", "", { "os": "aix", "cpu": "ppc64" }, "sha512-1SDgH6ZSPTlggy1yI6+Dbkiz8xzpHJEVAlF/AM1tHPLsf5STom9rwtjE4hKAF20FfXXNTFqEYXyJNWh1GiZedQ=="],
|
||||||
|
|
||||||
|
"@esbuild/android-arm": ["@esbuild/android-arm@0.21.5", "", { "os": "android", "cpu": "arm" }, "sha512-vCPvzSjpPHEi1siZdlvAlsPxXl7WbOVUBBAowWug4rJHb68Ox8KualB+1ocNvT5fjv6wpkX6o/iEpbDrf68zcg=="],
|
||||||
|
|
||||||
|
"@esbuild/android-arm64": ["@esbuild/android-arm64@0.21.5", "", { "os": "android", "cpu": "arm64" }, "sha512-c0uX9VAUBQ7dTDCjq+wdyGLowMdtR/GoC2U5IYk/7D1H1JYC0qseD7+11iMP2mRLN9RcCMRcjC4YMclCzGwS/A=="],
|
||||||
|
|
||||||
|
"@esbuild/android-x64": ["@esbuild/android-x64@0.21.5", "", { "os": "android", "cpu": "x64" }, "sha512-D7aPRUUNHRBwHxzxRvp856rjUHRFW1SdQATKXH2hqA0kAZb1hKmi02OpYRacl0TxIGz/ZmXWlbZgjwWYaCakTA=="],
|
||||||
|
|
||||||
|
"@esbuild/darwin-arm64": ["@esbuild/darwin-arm64@0.21.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-DwqXqZyuk5AiWWf3UfLiRDJ5EDd49zg6O9wclZ7kUMv2WRFr4HKjXp/5t8JZ11QbQfUS6/cRCKGwYhtNAY88kQ=="],
|
||||||
|
|
||||||
|
"@esbuild/darwin-x64": ["@esbuild/darwin-x64@0.21.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-se/JjF8NlmKVG4kNIuyWMV/22ZaerB+qaSi5MdrXtd6R08kvs2qCN4C09miupktDitvh8jRFflwGFBQcxZRjbw=="],
|
||||||
|
|
||||||
|
"@esbuild/freebsd-arm64": ["@esbuild/freebsd-arm64@0.21.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-5JcRxxRDUJLX8JXp/wcBCy3pENnCgBR9bN6JsY4OmhfUtIHe3ZW0mawA7+RDAcMLrMIZaf03NlQiX9DGyB8h4g=="],
|
||||||
|
|
||||||
|
"@esbuild/freebsd-x64": ["@esbuild/freebsd-x64@0.21.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-J95kNBj1zkbMXtHVH29bBriQygMXqoVQOQYA+ISs0/2l3T9/kj42ow2mpqerRBxDJnmkUDCaQT/dfNXWX/ZZCQ=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-arm": ["@esbuild/linux-arm@0.21.5", "", { "os": "linux", "cpu": "arm" }, "sha512-bPb5AHZtbeNGjCKVZ9UGqGwo8EUu4cLq68E95A53KlxAPRmUyYv2D6F0uUI65XisGOL1hBP5mTronbgo+0bFcA=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-arm64": ["@esbuild/linux-arm64@0.21.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-ibKvmyYzKsBeX8d8I7MH/TMfWDXBF3db4qM6sy+7re0YXya+K1cem3on9XgdT2EQGMu4hQyZhan7TeQ8XkGp4Q=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-ia32": ["@esbuild/linux-ia32@0.21.5", "", { "os": "linux", "cpu": "ia32" }, "sha512-YvjXDqLRqPDl2dvRODYmmhz4rPeVKYvppfGYKSNGdyZkA01046pLWyRKKI3ax8fbJoK5QbxblURkwK/MWY18Tg=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-loong64": ["@esbuild/linux-loong64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-uHf1BmMG8qEvzdrzAqg2SIG/02+4/DHB6a9Kbya0XDvwDEKCoC8ZRWI5JJvNdUjtciBGFQ5PuBlpEOXQj+JQSg=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-mips64el": ["@esbuild/linux-mips64el@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-IajOmO+KJK23bj52dFSNCMsz1QP1DqM6cwLUv3W1QwyxkyIWecfafnI555fvSGqEKwjMXVLokcV5ygHW5b3Jbg=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-ppc64": ["@esbuild/linux-ppc64@0.21.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-1hHV/Z4OEfMwpLO8rp7CvlhBDnjsC3CttJXIhBi+5Aj5r+MBvy4egg7wCbe//hSsT+RvDAG7s81tAvpL2XAE4w=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-riscv64": ["@esbuild/linux-riscv64@0.21.5", "", { "os": "linux", "cpu": "none" }, "sha512-2HdXDMd9GMgTGrPWnJzP2ALSokE/0O5HhTUvWIbD3YdjME8JwvSCnNGBnTThKGEB91OZhzrJ4qIIxk/SBmyDDA=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-s390x": ["@esbuild/linux-s390x@0.21.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-zus5sxzqBJD3eXxwvjN1yQkRepANgxE9lgOW2qLnmr8ikMTphkjgXu1HR01K4FJg8h1kEEDAqDcZQtbrRnB41A=="],
|
||||||
|
|
||||||
|
"@esbuild/linux-x64": ["@esbuild/linux-x64@0.21.5", "", { "os": "linux", "cpu": "x64" }, "sha512-1rYdTpyv03iycF1+BhzrzQJCdOuAOtaqHTWJZCWvijKD2N5Xu0TtVC8/+1faWqcP9iBCWOmjmhoH94dH82BxPQ=="],
|
||||||
|
|
||||||
|
"@esbuild/netbsd-x64": ["@esbuild/netbsd-x64@0.21.5", "", { "os": "none", "cpu": "x64" }, "sha512-Woi2MXzXjMULccIwMnLciyZH4nCIMpWQAs049KEeMvOcNADVxo0UBIQPfSmxB3CWKedngg7sWZdLvLczpe0tLg=="],
|
||||||
|
|
||||||
|
"@esbuild/openbsd-x64": ["@esbuild/openbsd-x64@0.21.5", "", { "os": "openbsd", "cpu": "x64" }, "sha512-HLNNw99xsvx12lFBUwoT8EVCsSvRNDVxNpjZ7bPn947b8gJPzeHWyNVhFsaerc0n3TsbOINvRP2byTZ5LKezow=="],
|
||||||
|
|
||||||
|
"@esbuild/sunos-x64": ["@esbuild/sunos-x64@0.21.5", "", { "os": "sunos", "cpu": "x64" }, "sha512-6+gjmFpfy0BHU5Tpptkuh8+uw3mnrvgs+dSPQXQOv3ekbordwnzTVEb4qnIvQcYXq6gzkyTnoZ9dZG+D4garKg=="],
|
||||||
|
|
||||||
|
"@esbuild/win32-arm64": ["@esbuild/win32-arm64@0.21.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-Z0gOTd75VvXqyq7nsl93zwahcTROgqvuAcYDUr+vOv8uHhNSKROyU961kgtCD1e95IqPKSQKH7tBTslnS3tA8A=="],
|
||||||
|
|
||||||
|
"@esbuild/win32-ia32": ["@esbuild/win32-ia32@0.21.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-SWXFF1CL2RVNMaVs+BBClwtfZSvDgtL//G/smwAc5oVK/UPu2Gu9tIaRgFmYFFKrmg3SyAjSrElf0TiJ1v8fYA=="],
|
||||||
|
|
||||||
|
"@esbuild/win32-x64": ["@esbuild/win32-x64@0.21.5", "", { "os": "win32", "cpu": "x64" }, "sha512-tQd/1efJuzPC6rCFwEvLtci/xNFcTZknmXs98FYDfGE4wP9ClFV98nyKrzJKVPMhdDnjzLhdUyMX4PsQAPjwIw=="],
|
||||||
|
|
||||||
|
"@jest/schemas": ["@jest/schemas@29.6.3", "", { "dependencies": { "@sinclair/typebox": "^0.27.8" } }, "sha512-mo5j5X+jIZmJQveBKeS/clAueipV7KgiX1vMgCxam1RNYiqE1w62n0/tJJnHtjW8ZHcQco5gY85jA3mi0L+nSA=="],
|
||||||
|
|
||||||
|
"@jridgewell/sourcemap-codec": ["@jridgewell/sourcemap-codec@1.5.5", "", {}, "sha512-cYQ9310grqxueWbl+WuIUIaiUaDcj7WOq5fVhEljNVgRfOUhY9fy2zTvfoqWsnebh8Sl70VScFbICvJnLKB0Og=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-android-arm-eabi": ["@rollup/rollup-android-arm-eabi@4.52.5", "", { "os": "android", "cpu": "arm" }, "sha512-8c1vW4ocv3UOMp9K+gToY5zL2XiiVw3k7f1ksf4yO1FlDFQ1C2u72iACFnSOceJFsWskc2WZNqeRhFRPzv+wtQ=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-android-arm64": ["@rollup/rollup-android-arm64@4.52.5", "", { "os": "android", "cpu": "arm64" }, "sha512-mQGfsIEFcu21mvqkEKKu2dYmtuSZOBMmAl5CFlPGLY94Vlcm+zWApK7F/eocsNzp8tKmbeBP8yXyAbx0XHsFNA=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-darwin-arm64": ["@rollup/rollup-darwin-arm64@4.52.5", "", { "os": "darwin", "cpu": "arm64" }, "sha512-takF3CR71mCAGA+v794QUZ0b6ZSrgJkArC+gUiG6LB6TQty9T0Mqh3m2ImRBOxS2IeYBo4lKWIieSvnEk2OQWA=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-darwin-x64": ["@rollup/rollup-darwin-x64@4.52.5", "", { "os": "darwin", "cpu": "x64" }, "sha512-W901Pla8Ya95WpxDn//VF9K9u2JbocwV/v75TE0YIHNTbhqUTv9w4VuQ9MaWlNOkkEfFwkdNhXgcLqPSmHy0fA=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-freebsd-arm64": ["@rollup/rollup-freebsd-arm64@4.52.5", "", { "os": "freebsd", "cpu": "arm64" }, "sha512-QofO7i7JycsYOWxe0GFqhLmF6l1TqBswJMvICnRUjqCx8b47MTo46W8AoeQwiokAx3zVryVnxtBMcGcnX12LvA=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-freebsd-x64": ["@rollup/rollup-freebsd-x64@4.52.5", "", { "os": "freebsd", "cpu": "x64" }, "sha512-jr21b/99ew8ujZubPo9skbrItHEIE50WdV86cdSoRkKtmWa+DDr6fu2c/xyRT0F/WazZpam6kk7IHBerSL7LDQ=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-arm-gnueabihf": ["@rollup/rollup-linux-arm-gnueabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-PsNAbcyv9CcecAUagQefwX8fQn9LQ4nZkpDboBOttmyffnInRy8R8dSg6hxxl2Re5QhHBf6FYIDhIj5v982ATQ=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-arm-musleabihf": ["@rollup/rollup-linux-arm-musleabihf@4.52.5", "", { "os": "linux", "cpu": "arm" }, "sha512-Fw4tysRutyQc/wwkmcyoqFtJhh0u31K+Q6jYjeicsGJJ7bbEq8LwPWV/w0cnzOqR2m694/Af6hpFayLJZkG2VQ=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-arm64-gnu": ["@rollup/rollup-linux-arm64-gnu@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-a+3wVnAYdQClOTlyapKmyI6BLPAFYs0JM8HRpgYZQO02rMR09ZcV9LbQB+NL6sljzG38869YqThrRnfPMCDtZg=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-arm64-musl": ["@rollup/rollup-linux-arm64-musl@4.52.5", "", { "os": "linux", "cpu": "arm64" }, "sha512-AvttBOMwO9Pcuuf7m9PkC1PUIKsfaAJ4AYhy944qeTJgQOqJYJ9oVl2nYgY7Rk0mkbsuOpCAYSs6wLYB2Xiw0Q=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-loong64-gnu": ["@rollup/rollup-linux-loong64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-DkDk8pmXQV2wVrF6oq5tONK6UHLz/XcEVow4JTTerdeV1uqPeHxwcg7aFsfnSm9L+OO8WJsWotKM2JJPMWrQtA=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-ppc64-gnu": ["@rollup/rollup-linux-ppc64-gnu@4.52.5", "", { "os": "linux", "cpu": "ppc64" }, "sha512-W/b9ZN/U9+hPQVvlGwjzi+Wy4xdoH2I8EjaCkMvzpI7wJUs8sWJ03Rq96jRnHkSrcHTpQe8h5Tg3ZzUPGauvAw=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-riscv64-gnu": ["@rollup/rollup-linux-riscv64-gnu@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-sjQLr9BW7R/ZiXnQiWPkErNfLMkkWIoCz7YMn27HldKsADEKa5WYdobaa1hmN6slu9oWQbB6/jFpJ+P2IkVrmw=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-riscv64-musl": ["@rollup/rollup-linux-riscv64-musl@4.52.5", "", { "os": "linux", "cpu": "none" }, "sha512-hq3jU/kGyjXWTvAh2awn8oHroCbrPm8JqM7RUpKjalIRWWXE01CQOf/tUNWNHjmbMHg/hmNCwc/Pz3k1T/j/Lg=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-s390x-gnu": ["@rollup/rollup-linux-s390x-gnu@4.52.5", "", { "os": "linux", "cpu": "s390x" }, "sha512-gn8kHOrku8D4NGHMK1Y7NA7INQTRdVOntt1OCYypZPRt6skGbddska44K8iocdpxHTMMNui5oH4elPH4QOLrFQ=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-x64-gnu": ["@rollup/rollup-linux-x64-gnu@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-hXGLYpdhiNElzN770+H2nlx+jRog8TyynpTVzdlc6bndktjKWyZyiCsuDAlpd+j+W+WNqfcyAWz9HxxIGfZm1Q=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-linux-x64-musl": ["@rollup/rollup-linux-x64-musl@4.52.5", "", { "os": "linux", "cpu": "x64" }, "sha512-arCGIcuNKjBoKAXD+y7XomR9gY6Mw7HnFBv5Rw7wQRvwYLR7gBAgV7Mb2QTyjXfTveBNFAtPt46/36vV9STLNg=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-openharmony-arm64": ["@rollup/rollup-openharmony-arm64@4.52.5", "", { "os": "none", "cpu": "arm64" }, "sha512-QoFqB6+/9Rly/RiPjaomPLmR/13cgkIGfA40LHly9zcH1S0bN2HVFYk3a1eAyHQyjs3ZJYlXvIGtcCs5tko9Cw=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-win32-arm64-msvc": ["@rollup/rollup-win32-arm64-msvc@4.52.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-w0cDWVR6MlTstla1cIfOGyl8+qb93FlAVutcor14Gf5Md5ap5ySfQ7R9S/NjNaMLSFdUnKGEasmVnu3lCMqB7w=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-win32-ia32-msvc": ["@rollup/rollup-win32-ia32-msvc@4.52.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-Aufdpzp7DpOTULJCuvzqcItSGDH73pF3ko/f+ckJhxQyHtp67rHw3HMNxoIdDMUITJESNE6a8uh4Lo4SLouOUg=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-win32-x64-gnu": ["@rollup/rollup-win32-x64-gnu@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-UGBUGPFp1vkj6p8wCRraqNhqwX/4kNQPS57BCFc8wYh0g94iVIW33wJtQAx3G7vrjjNtRaxiMUylM0ktp/TRSQ=="],
|
||||||
|
|
||||||
|
"@rollup/rollup-win32-x64-msvc": ["@rollup/rollup-win32-x64-msvc@4.52.5", "", { "os": "win32", "cpu": "x64" }, "sha512-TAcgQh2sSkykPRWLrdyy2AiceMckNf5loITqXxFI5VuQjS5tSuw3WlwdN8qv8vzjLAUTvYaH/mVjSFpbkFbpTg=="],
|
||||||
|
|
||||||
|
"@sinclair/typebox": ["@sinclair/typebox@0.27.8", "", {}, "sha512-+Fj43pSMwJs4KRrH/938Uf+uAELIgVBmQzg/q1YG10djyfA3TnrU8N8XzqCh/okZdszqBQTZf96idMfE5lnwTA=="],
|
||||||
|
|
||||||
|
"@types/estree": ["@types/estree@1.0.8", "", {}, "sha512-dWHzHa2WqEXI/O1E9OjrocMTKJl2mSrEolh1Iomrv6U+JuNwaHXsXx9bLu5gG7BUWFIN0skIQJQ/L1rIex4X6w=="],
|
||||||
|
|
||||||
|
"@types/node": ["@types/node@20.19.24", "", { "dependencies": { "undici-types": "~6.21.0" } }, "sha512-FE5u0ezmi6y9OZEzlJfg37mqqf6ZDSF2V/NLjUyGrR9uTZ7Sb9F7bLNZ03S4XVUNRWGA7Ck4c1kK+YnuWjl+DA=="],
|
||||||
|
|
||||||
|
"@vitest/expect": ["@vitest/expect@1.6.1", "", { "dependencies": { "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "chai": "^4.3.10" } }, "sha512-jXL+9+ZNIJKruofqXuuTClf44eSpcHlgj3CiuNihUF3Ioujtmc0zIa3UJOW5RjDK1YLBJZnWBlPuqhYycLioog=="],
|
||||||
|
|
||||||
|
"@vitest/runner": ["@vitest/runner@1.6.1", "", { "dependencies": { "@vitest/utils": "1.6.1", "p-limit": "^5.0.0", "pathe": "^1.1.1" } }, "sha512-3nSnYXkVkf3mXFfE7vVyPmi3Sazhb/2cfZGGs0JRzFsPFvAMBEcrweV1V1GsrstdXeKCTXlJbvnQwGWgEIHmOA=="],
|
||||||
|
|
||||||
|
"@vitest/snapshot": ["@vitest/snapshot@1.6.1", "", { "dependencies": { "magic-string": "^0.30.5", "pathe": "^1.1.1", "pretty-format": "^29.7.0" } }, "sha512-WvidQuWAzU2p95u8GAKlRMqMyN1yOJkGHnx3M1PL9Raf7AQ1kwLKg04ADlCa3+OXUZE7BceOhVZiuWAbzCKcUQ=="],
|
||||||
|
|
||||||
|
"@vitest/spy": ["@vitest/spy@1.6.1", "", { "dependencies": { "tinyspy": "^2.2.0" } }, "sha512-MGcMmpGkZebsMZhbQKkAf9CX5zGvjkBTqf8Zx3ApYWXr3wG+QvEu2eXWfnIIWYSJExIp4V9FCKDEeygzkYrXMw=="],
|
||||||
|
|
||||||
|
"@vitest/utils": ["@vitest/utils@1.6.1", "", { "dependencies": { "diff-sequences": "^29.6.3", "estree-walker": "^3.0.3", "loupe": "^2.3.7", "pretty-format": "^29.7.0" } }, "sha512-jOrrUvXM4Av9ZWiG1EajNto0u96kWAhJ1LmPmJhXXQx/32MecEKd10pOLYgS2BQx1TgkGhloPU1ArDW2vvaY6g=="],
|
||||||
|
|
||||||
|
"acorn": ["acorn@8.15.0", "", { "bin": { "acorn": "bin/acorn" } }, "sha512-NZyJarBfL7nWwIq+FDL6Zp/yHEhePMNnnJ0y3qfieCrmNvYct8uvtiV41UvlSe6apAfk0fY1FbWx+NwfmpvtTg=="],
|
||||||
|
|
||||||
|
"acorn-walk": ["acorn-walk@8.3.4", "", { "dependencies": { "acorn": "^8.11.0" } }, "sha512-ueEepnujpqee2o5aIYnvHU6C0A42MNdsIDeqy5BydrkuC5R1ZuUFnm27EeFJGoEHJQgn3uleRvmTXaJgfXbt4g=="],
|
||||||
|
|
||||||
|
"ansi-styles": ["ansi-styles@5.2.0", "", {}, "sha512-Cxwpt2SfTzTtXcfOlzGEee8O+c+MmUgGrNiBcXnuWxuFJHe6a5Hz7qwhwe5OgaSYI0IJvkLqWX1ASG+cJOkEiA=="],
|
||||||
|
|
||||||
|
"assertion-error": ["assertion-error@1.1.0", "", {}, "sha512-jgsaNduz+ndvGyFt3uSuWqvy4lCnIJiovtouQN5JZHOKCS2QuhEdbcQHFhVksz2N2U9hXJo8odG7ETyWlEeuDw=="],
|
||||||
|
|
||||||
|
"cac": ["cac@6.7.14", "", {}, "sha512-b6Ilus+c3RrdDk+JhLKUAQfzzgLEPy6wcXqS7f/xe1EETvsDP6GORG7SFuOs6cID5YkqchW/LXZbX5bc8j7ZcQ=="],
|
||||||
|
|
||||||
|
"chai": ["chai@4.5.0", "", { "dependencies": { "assertion-error": "^1.1.0", "check-error": "^1.0.3", "deep-eql": "^4.1.3", "get-func-name": "^2.0.2", "loupe": "^2.3.6", "pathval": "^1.1.1", "type-detect": "^4.1.0" } }, "sha512-RITGBfijLkBddZvnn8jdqoTypxvqbOLYQkGGxXzeFjVHvudaPw0HNFD9x928/eUwYWd2dPCugVqspGALTZZQKw=="],
|
||||||
|
|
||||||
|
"check-error": ["check-error@1.0.3", "", { "dependencies": { "get-func-name": "^2.0.2" } }, "sha512-iKEoDYaRmd1mxM90a2OEfWhjsjPpYPuQ+lMYsoxB126+t8fw7ySEO48nmDg5COTjxDI65/Y2OWpeEHk3ZOe8zg=="],
|
||||||
|
|
||||||
|
"confbox": ["confbox@0.1.8", "", {}, "sha512-RMtmw0iFkeR4YV+fUOSucriAQNb9g8zFR52MWCtl+cCZOFRNL6zeB395vPzFhEjjn4fMxXudmELnl/KF/WrK6w=="],
|
||||||
|
|
||||||
|
"cross-spawn": ["cross-spawn@7.0.6", "", { "dependencies": { "path-key": "^3.1.0", "shebang-command": "^2.0.0", "which": "^2.0.1" } }, "sha512-uV2QOWP2nWzsy2aMp8aRibhi9dlzF5Hgh5SHaB9OiTGEyDTiJJyx0uy51QXdyWbtAHNua4XJzUKca3OzKUd3vA=="],
|
||||||
|
|
||||||
|
"debug": ["debug@4.4.3", "", { "dependencies": { "ms": "^2.1.3" } }, "sha512-RGwwWnwQvkVfavKVt22FGLw+xYSdzARwm0ru6DhTVA3umU5hZc28V3kO4stgYryrTlLpuvgI9GiijltAjNbcqA=="],
|
||||||
|
|
||||||
|
"deep-eql": ["deep-eql@4.1.4", "", { "dependencies": { "type-detect": "^4.0.0" } }, "sha512-SUwdGfqdKOwxCPeVYjwSyRpJ7Z+fhpwIAtmCUdZIWZ/YP5R9WAsyuSgpLVDi9bjWoN2LXHNss/dk3urXtdQxGg=="],
|
||||||
|
|
||||||
|
"diff-sequences": ["diff-sequences@29.6.3", "", {}, "sha512-EjePK1srD3P08o2j4f0ExnylqRs5B9tJjcp9t1krH2qRi8CCdsYfwe9JgSLurFBWwq4uOlipzfk5fHNvwFKr8Q=="],
|
||||||
|
|
||||||
|
"esbuild": ["esbuild@0.21.5", "", { "optionalDependencies": { "@esbuild/aix-ppc64": "0.21.5", "@esbuild/android-arm": "0.21.5", "@esbuild/android-arm64": "0.21.5", "@esbuild/android-x64": "0.21.5", "@esbuild/darwin-arm64": "0.21.5", "@esbuild/darwin-x64": "0.21.5", "@esbuild/freebsd-arm64": "0.21.5", "@esbuild/freebsd-x64": "0.21.5", "@esbuild/linux-arm": "0.21.5", "@esbuild/linux-arm64": "0.21.5", "@esbuild/linux-ia32": "0.21.5", "@esbuild/linux-loong64": "0.21.5", "@esbuild/linux-mips64el": "0.21.5", "@esbuild/linux-ppc64": "0.21.5", "@esbuild/linux-riscv64": "0.21.5", "@esbuild/linux-s390x": "0.21.5", "@esbuild/linux-x64": "0.21.5", "@esbuild/netbsd-x64": "0.21.5", "@esbuild/openbsd-x64": "0.21.5", "@esbuild/sunos-x64": "0.21.5", "@esbuild/win32-arm64": "0.21.5", "@esbuild/win32-ia32": "0.21.5", "@esbuild/win32-x64": "0.21.5" }, "bin": { "esbuild": "bin/esbuild" } }, "sha512-mg3OPMV4hXywwpoDxu3Qda5xCKQi+vCTZq8S9J/EpkhB2HzKXq4SNFZE3+NK93JYxc8VMSep+lOUSC/RVKaBqw=="],
|
||||||
|
|
||||||
|
"estree-walker": ["estree-walker@3.0.3", "", { "dependencies": { "@types/estree": "^1.0.0" } }, "sha512-7RUKfXgSMMkzt6ZuXmqapOurLGPPfgj6l9uRZ7lRGolvk0y2yocc35LdcxKC5PQZdn2DMqioAQ2NoWcrTKmm6g=="],
|
||||||
|
|
||||||
|
"execa": ["execa@8.0.1", "", { "dependencies": { "cross-spawn": "^7.0.3", "get-stream": "^8.0.1", "human-signals": "^5.0.0", "is-stream": "^3.0.0", "merge-stream": "^2.0.0", "npm-run-path": "^5.1.0", "onetime": "^6.0.0", "signal-exit": "^4.1.0", "strip-final-newline": "^3.0.0" } }, "sha512-VyhnebXciFV2DESc+p6B+y0LjSm0krU4OgJN44qFAhBY0TJ+1V61tYD2+wHusZ6F9n5K+vl8k0sTy7PEfV4qpg=="],
|
||||||
|
|
||||||
|
"fsevents": ["fsevents@2.3.3", "", { "os": "darwin" }, "sha512-5xoDfX+fL7faATnagmWPpbFtwh/R77WmMMqqHGS65C3vvB0YHrgF+B1YmZ3441tMj5n63k0212XNoJwzlhffQw=="],
|
||||||
|
|
||||||
|
"get-func-name": ["get-func-name@2.0.2", "", {}, "sha512-8vXOvuE167CtIc3OyItco7N/dpRtBbYOsPsXCz7X/PMnlGjYjSGuZJgM1Y7mmew7BKf9BqvLX2tnOVy1BBUsxQ=="],
|
||||||
|
|
||||||
|
"get-stream": ["get-stream@8.0.1", "", {}, "sha512-VaUJspBffn/LMCJVoMvSAdmscJyS1auj5Zulnn5UoYcY531UWmdwhRWkcGKnGU93m5HSXP9LP2usOryrBtQowA=="],
|
||||||
|
|
||||||
|
"human-signals": ["human-signals@5.0.0", "", {}, "sha512-AXcZb6vzzrFAUE61HnN4mpLqd/cSIwNQjtNWR0euPm6y0iqx3G4gOXaIDdtdDwZmhwe82LA6+zinmW4UBWVePQ=="],
|
||||||
|
|
||||||
|
"is-stream": ["is-stream@3.0.0", "", {}, "sha512-LnQR4bZ9IADDRSkvpqMGvt/tEJWclzklNgSw48V5EAaAeDd6qGvN8ei6k5p0tvxSR171VmGyHuTiAOfxAbr8kA=="],
|
||||||
|
|
||||||
|
"isexe": ["isexe@2.0.0", "", {}, "sha512-RHxMLp9lnKHGHRng9QFhRCMbYAcVpn69smSGcq3f36xjgVVWThj4qqLbTLlq7Ssj8B+fIQ1EuCEGI2lKsyQeIw=="],
|
||||||
|
|
||||||
|
"js-tokens": ["js-tokens@9.0.1", "", {}, "sha512-mxa9E9ITFOt0ban3j6L5MpjwegGz6lBQmM1IJkWeBZGcMxto50+eWdjC/52xDbS2vy0k7vIMK0Fe2wfL9OQSpQ=="],
|
||||||
|
|
||||||
|
"local-pkg": ["local-pkg@0.5.1", "", { "dependencies": { "mlly": "^1.7.3", "pkg-types": "^1.2.1" } }, "sha512-9rrA30MRRP3gBD3HTGnC6cDFpaE1kVDWxWgqWJUN0RvDNAo+Nz/9GxB+nHOH0ifbVFy0hSA1V6vFDvnx54lTEQ=="],
|
||||||
|
|
||||||
|
"loupe": ["loupe@2.3.7", "", { "dependencies": { "get-func-name": "^2.0.1" } }, "sha512-zSMINGVYkdpYSOBmLi0D1Uo7JU9nVdQKrHxC8eYlV+9YKK9WePqAlL7lSlorG/U2Fw1w0hTBmaa/jrQ3UbPHtA=="],
|
||||||
|
|
||||||
|
"magic-string": ["magic-string@0.30.21", "", { "dependencies": { "@jridgewell/sourcemap-codec": "^1.5.5" } }, "sha512-vd2F4YUyEXKGcLHoq+TEyCjxueSeHnFxyyjNp80yg0XV4vUhnDer/lvvlqM/arB5bXQN5K2/3oinyCRyx8T2CQ=="],
|
||||||
|
|
||||||
|
"merge-stream": ["merge-stream@2.0.0", "", {}, "sha512-abv/qOcuPfk3URPfDzmZU1LKmuw8kT+0nIHvKrKgFrwifol/doWcdA4ZqsWQ8ENrFKkd67Mfpo/LovbIUsbt3w=="],
|
||||||
|
|
||||||
|
"mimic-fn": ["mimic-fn@4.0.0", "", {}, "sha512-vqiC06CuhBTUdZH+RYl8sFrL096vA45Ok5ISO6sE/Mr1jRbGH4Csnhi8f3wKVl7x8mO4Au7Ir9D3Oyv1VYMFJw=="],
|
||||||
|
|
||||||
|
"mlly": ["mlly@1.8.0", "", { "dependencies": { "acorn": "^8.15.0", "pathe": "^2.0.3", "pkg-types": "^1.3.1", "ufo": "^1.6.1" } }, "sha512-l8D9ODSRWLe2KHJSifWGwBqpTZXIXTeo8mlKjY+E2HAakaTeNpqAyBZ8GSqLzHgw4XmHmC8whvpjJNMbFZN7/g=="],
|
||||||
|
|
||||||
|
"ms": ["ms@2.1.3", "", {}, "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA=="],
|
||||||
|
|
||||||
|
"nanoid": ["nanoid@3.3.11", "", { "bin": { "nanoid": "bin/nanoid.cjs" } }, "sha512-N8SpfPUnUp1bK+PMYW8qSWdl9U+wwNWI4QKxOYDy9JAro3WMX7p2OeVRF9v+347pnakNevPmiHhNmZ2HbFA76w=="],
|
||||||
|
|
||||||
|
"npm-run-path": ["npm-run-path@5.3.0", "", { "dependencies": { "path-key": "^4.0.0" } }, "sha512-ppwTtiJZq0O/ai0z7yfudtBpWIoxM8yE6nHi1X47eFR2EWORqfbu6CnPlNsjeN683eT0qG6H/Pyf9fCcvjnnnQ=="],
|
||||||
|
|
||||||
|
"onetime": ["onetime@6.0.0", "", { "dependencies": { "mimic-fn": "^4.0.0" } }, "sha512-1FlR+gjXK7X+AsAHso35MnyN5KqGwJRi/31ft6x0M194ht7S+rWAvd7PHss9xSKMzE0asv1pyIHaJYq+BbacAQ=="],
|
||||||
|
|
||||||
|
"p-limit": ["p-limit@5.0.0", "", { "dependencies": { "yocto-queue": "^1.0.0" } }, "sha512-/Eaoq+QyLSiXQ4lyYV23f14mZRQcXnxfHrN0vCai+ak9G0pp9iEQukIIZq5NccEvwRB8PUnZT0KsOoDCINS1qQ=="],
|
||||||
|
|
||||||
|
"path-key": ["path-key@3.1.1", "", {}, "sha512-ojmeN0qd+y0jszEtoY48r0Peq5dwMEkIlCOu6Q5f41lfkswXuKtYrhgoTpLnyIcHm24Uhqx+5Tqm2InSwLhE6Q=="],
|
||||||
|
|
||||||
|
"pathe": ["pathe@1.1.2", "", {}, "sha512-whLdWMYL2TwI08hn8/ZqAbrVemu0LNaNNJZX73O6qaIdCTfXutsLhMkjdENX0qhsQ9uIimo4/aQOmXkoon2nDQ=="],
|
||||||
|
|
||||||
|
"pathval": ["pathval@1.1.1", "", {}, "sha512-Dp6zGqpTdETdR63lehJYPeIOqpiNBNtc7BpWSLrOje7UaIsE5aY92r/AunQA7rsXvet3lrJ3JnZX29UPTKXyKQ=="],
|
||||||
|
|
||||||
|
"picocolors": ["picocolors@1.1.1", "", {}, "sha512-xceH2snhtb5M9liqDsmEw56le376mTZkEX/jEb/RxNFyegNul7eNslCXP9FDj/Lcu0X8KEyMceP2ntpaHrDEVA=="],
|
||||||
|
|
||||||
|
"pkg-types": ["pkg-types@1.3.1", "", { "dependencies": { "confbox": "^0.1.8", "mlly": "^1.7.4", "pathe": "^2.0.1" } }, "sha512-/Jm5M4RvtBFVkKWRu2BLUTNP8/M2a+UwuAX+ae4770q1qVGtfjG+WTCupoZixokjmHiry8uI+dlY8KXYV5HVVQ=="],
|
||||||
|
|
||||||
|
"postcss": ["postcss@8.5.6", "", { "dependencies": { "nanoid": "^3.3.11", "picocolors": "^1.1.1", "source-map-js": "^1.2.1" } }, "sha512-3Ybi1tAuwAP9s0r1UQ2J4n5Y0G05bJkpUIO0/bI9MhwmD70S5aTWbXGBwxHrelT+XM1k6dM0pk+SwNkpTRN7Pg=="],
|
||||||
|
|
||||||
|
"pretty-format": ["pretty-format@29.7.0", "", { "dependencies": { "@jest/schemas": "^29.6.3", "ansi-styles": "^5.0.0", "react-is": "^18.0.0" } }, "sha512-Pdlw/oPxN+aXdmM9R00JVC9WVFoCLTKJvDVLgmJ+qAffBMxsV85l/Lu7sNx4zSzPyoL2euImuEwHhOXdEgNFZQ=="],
|
||||||
|
|
||||||
|
"react-is": ["react-is@18.3.1", "", {}, "sha512-/LLMVyas0ljjAtoYiPqYiL8VWXzUUdThrmU5+n20DZv+a+ClRoevUzw5JxU+Ieh5/c87ytoTBV9G1FiKfNJdmg=="],
|
||||||
|
|
||||||
|
"rollup": ["rollup@4.52.5", "", { "dependencies": { "@types/estree": "1.0.8" }, "optionalDependencies": { "@rollup/rollup-android-arm-eabi": "4.52.5", "@rollup/rollup-android-arm64": "4.52.5", "@rollup/rollup-darwin-arm64": "4.52.5", "@rollup/rollup-darwin-x64": "4.52.5", "@rollup/rollup-freebsd-arm64": "4.52.5", "@rollup/rollup-freebsd-x64": "4.52.5", "@rollup/rollup-linux-arm-gnueabihf": "4.52.5", "@rollup/rollup-linux-arm-musleabihf": "4.52.5", "@rollup/rollup-linux-arm64-gnu": "4.52.5", "@rollup/rollup-linux-arm64-musl": "4.52.5", "@rollup/rollup-linux-loong64-gnu": "4.52.5", "@rollup/rollup-linux-ppc64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-gnu": "4.52.5", "@rollup/rollup-linux-riscv64-musl": "4.52.5", "@rollup/rollup-linux-s390x-gnu": "4.52.5", "@rollup/rollup-linux-x64-gnu": "4.52.5", "@rollup/rollup-linux-x64-musl": "4.52.5", "@rollup/rollup-openharmony-arm64": "4.52.5", "@rollup/rollup-win32-arm64-msvc": "4.52.5", "@rollup/rollup-win32-ia32-msvc": "4.52.5", "@rollup/rollup-win32-x64-gnu": "4.52.5", "@rollup/rollup-win32-x64-msvc": "4.52.5", "fsevents": "~2.3.2" }, "bin": { "rollup": "dist/bin/rollup" } }, "sha512-3GuObel8h7Kqdjt0gxkEzaifHTqLVW56Y/bjN7PSQtkKr0w3V/QYSdt6QWYtd7A1xUtYQigtdUfgj1RvWVtorw=="],
|
||||||
|
|
||||||
|
"shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="],
|
||||||
|
|
||||||
|
"shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="],
|
||||||
|
|
||||||
|
"siginfo": ["siginfo@2.0.0", "", {}, "sha512-ybx0WO1/8bSBLEWXZvEd7gMW3Sn3JFlW3TvX1nREbDLRNQNaeNN8WK0meBwPdAaOI7TtRRRJn/Es1zhrrCHu7g=="],
|
||||||
|
|
||||||
|
"signal-exit": ["signal-exit@4.1.0", "", {}, "sha512-bzyZ1e88w9O1iNJbKnOlvYTrWPDl46O1bG0D3XInv+9tkPrxrN8jUUTiFlDkkmKWgn1M6CfIA13SuGqOa9Korw=="],
|
||||||
|
|
||||||
|
"source-map-js": ["source-map-js@1.2.1", "", {}, "sha512-UXWMKhLOwVKb728IUtQPXxfYU+usdybtUrK/8uGE8CQMvrhOpwvzDBwj0QhSL7MQc7vIsISBG8VQ8+IDQxpfQA=="],
|
||||||
|
|
||||||
|
"stackback": ["stackback@0.0.2", "", {}, "sha512-1XMJE5fQo1jGH6Y/7ebnwPOBEkIEnT4QF32d5R1+VXdXveM0IBMJt8zfaxX1P3QhVwrYe+576+jkANtSS2mBbw=="],
|
||||||
|
|
||||||
|
"std-env": ["std-env@3.10.0", "", {}, "sha512-5GS12FdOZNliM5mAOxFRg7Ir0pWz8MdpYm6AY6VPkGpbA7ZzmbzNcBJQ0GPvvyWgcY7QAhCgf9Uy89I03faLkg=="],
|
||||||
|
|
||||||
|
"strip-final-newline": ["strip-final-newline@3.0.0", "", {}, "sha512-dOESqjYr96iWYylGObzd39EuNTa5VJxyvVAEm5Jnh7KGo75V43Hk1odPQkNDyXNmUR6k+gEiDVXnjB8HJ3crXw=="],
|
||||||
|
|
||||||
|
"strip-literal": ["strip-literal@2.1.1", "", { "dependencies": { "js-tokens": "^9.0.1" } }, "sha512-631UJ6O00eNGfMiWG78ck80dfBab8X6IVFB51jZK5Icd7XAs60Z5y7QdSd/wGIklnWvRbUNloVzhOKKmutxQ6Q=="],
|
||||||
|
|
||||||
|
"tinybench": ["tinybench@2.9.0", "", {}, "sha512-0+DUvqWMValLmha6lr4kD8iAMK1HzV0/aKnCtWb9v9641TnP/MFb7Pc2bxoxQjTXAErryXVgUOfv2YqNllqGeg=="],
|
||||||
|
|
||||||
|
"tinypool": ["tinypool@0.8.4", "", {}, "sha512-i11VH5gS6IFeLY3gMBQ00/MmLncVP7JLXOw1vlgkytLmJK7QnEr7NXf0LBdxfmNPAeyetukOk0bOYrJrFGjYJQ=="],
|
||||||
|
|
||||||
|
"tinyspy": ["tinyspy@2.2.1", "", {}, "sha512-KYad6Vy5VDWV4GH3fjpseMQ/XU2BhIYP7Vzd0LG44qRWm/Yt2WCOTicFdvmgo6gWaqooMQCawTtILVQJupKu7A=="],
|
||||||
|
|
||||||
|
"type-detect": ["type-detect@4.1.0", "", {}, "sha512-Acylog8/luQ8L7il+geoSxhEkazvkslg7PSNKOX59mbB9cOveP5aq9h74Y7YU8yDpJwetzQQrfIwtf4Wp4LKcw=="],
|
||||||
|
|
||||||
|
"typescript": ["typescript@5.9.3", "", { "bin": { "tsc": "bin/tsc", "tsserver": "bin/tsserver" } }, "sha512-jl1vZzPDinLr9eUt3J/t7V6FgNEw9QjvBPdysz9KfQDD41fQrC2Y4vKQdiaUpFT4bXlb1RHhLpp8wtm6M5TgSw=="],
|
||||||
|
|
||||||
|
"ufo": ["ufo@1.6.1", "", {}, "sha512-9a4/uxlTWJ4+a5i0ooc1rU7C7YOw3wT+UGqdeNNHWnOF9qcMBgLRS+4IYUqbczewFx4mLEig6gawh7X6mFlEkA=="],
|
||||||
|
|
||||||
|
"undici-types": ["undici-types@6.21.0", "", {}, "sha512-iwDZqg0QAGrg9Rav5H4n0M64c3mkR59cJ6wQp+7C4nI0gsmExaedaYLNO44eT4AtBBwjbTiGPMlt2Md0T9H9JQ=="],
|
||||||
|
|
||||||
|
"vite": ["vite@5.4.21", "", { "dependencies": { "esbuild": "^0.21.3", "postcss": "^8.4.43", "rollup": "^4.20.0" }, "optionalDependencies": { "fsevents": "~2.3.3" }, "peerDependencies": { "@types/node": "^18.0.0 || >=20.0.0", "less": "*", "lightningcss": "^1.21.0", "sass": "*", "sass-embedded": "*", "stylus": "*", "sugarss": "*", "terser": "^5.4.0" }, "optionalPeers": ["@types/node", "less", "lightningcss", "sass", "sass-embedded", "stylus", "sugarss", "terser"], "bin": { "vite": "bin/vite.js" } }, "sha512-o5a9xKjbtuhY6Bi5S3+HvbRERmouabWbyUcpXXUA1u+GNUKoROi9byOJ8M0nHbHYHkYICiMlqxkg1KkYmm25Sw=="],
|
||||||
|
|
||||||
|
"vite-node": ["vite-node@1.6.1", "", { "dependencies": { "cac": "^6.7.14", "debug": "^4.3.4", "pathe": "^1.1.1", "picocolors": "^1.0.0", "vite": "^5.0.0" }, "bin": { "vite-node": "vite-node.mjs" } }, "sha512-YAXkfvGtuTzwWbDSACdJSg4A4DZiAqckWe90Zapc/sEX3XvHcw1NdurM/6od8J207tSDqNbSsgdCacBgvJKFuA=="],
|
||||||
|
|
||||||
|
"vitest": ["vitest@1.6.1", "", { "dependencies": { "@vitest/expect": "1.6.1", "@vitest/runner": "1.6.1", "@vitest/snapshot": "1.6.1", "@vitest/spy": "1.6.1", "@vitest/utils": "1.6.1", "acorn-walk": "^8.3.2", "chai": "^4.3.10", "debug": "^4.3.4", "execa": "^8.0.1", "local-pkg": "^0.5.0", "magic-string": "^0.30.5", "pathe": "^1.1.1", "picocolors": "^1.0.0", "std-env": "^3.5.0", "strip-literal": "^2.0.0", "tinybench": "^2.5.1", "tinypool": "^0.8.3", "vite": "^5.0.0", "vite-node": "1.6.1", "why-is-node-running": "^2.2.2" }, "peerDependencies": { "@edge-runtime/vm": "*", "@types/node": "^18.0.0 || >=20.0.0", "@vitest/browser": "1.6.1", "@vitest/ui": "1.6.1", "happy-dom": "*", "jsdom": "*" }, "optionalPeers": ["@edge-runtime/vm", "@types/node", "@vitest/browser", "@vitest/ui", "happy-dom", "jsdom"], "bin": { "vitest": "vitest.mjs" } }, "sha512-Ljb1cnSJSivGN0LqXd/zmDbWEM0RNNg2t1QW/XUhYl/qPqyu7CsqeWtqQXHVaJsecLPuDoak2oJcZN2QoRIOag=="],
|
||||||
|
|
||||||
|
"which": ["which@2.0.2", "", { "dependencies": { "isexe": "^2.0.0" }, "bin": { "node-which": "./bin/node-which" } }, "sha512-BLI3Tl1TW3Pvl70l3yq3Y64i+awpwXqsGBYWkkqMtnbXgrMD+yj7rhW0kuEDxzJaYXGjEW5ogapKNMEKNMjibA=="],
|
||||||
|
|
||||||
|
"why-is-node-running": ["why-is-node-running@2.3.0", "", { "dependencies": { "siginfo": "^2.0.0", "stackback": "0.0.2" }, "bin": { "why-is-node-running": "cli.js" } }, "sha512-hUrmaWBdVDcxvYqnyh09zunKzROWjbZTiNy8dBEjkS7ehEDQibXJ7XvlmtbwuTclUiIyN+CyXQD4Vmko8fNm8w=="],
|
||||||
|
|
||||||
|
"yocto-queue": ["yocto-queue@1.2.1", "", {}, "sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg=="],
|
||||||
|
|
||||||
|
"mlly/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
|
||||||
|
|
||||||
|
"npm-run-path/path-key": ["path-key@4.0.0", "", {}, "sha512-haREypq7xkM7ErfgIyA0z+Bj4AGKlMSdlQE2jvJo6huWD1EdkKYV+G/T4nq0YEF2vgTT8kqMFKo1uHn950r4SQ=="],
|
||||||
|
|
||||||
|
"pkg-types/pathe": ["pathe@2.0.3", "", {}, "sha512-WUjGcAqP1gQacoQe+OBJsFA7Ld4DyXuUIjZ5cc75cLHvJ7dtNsTugphxIADwspS+AraAUePCKrSVtPLFj/F88w=="],
|
||||||
|
}
|
||||||
|
}
|
||||||
Reference in New Issue
Block a user