fix(app): remove legacy and scoped namespace alias
This commit is contained in:
@@ -1,19 +1,24 @@
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { config, env } from "@/config";
|
||||
import { UnsupportedSourceKindError } from "@/errors";
|
||||
import { SyncHttpClient } from "@/http/http-client";
|
||||
import { QueueManager, createQueueManager } from "@/process/async/queue";
|
||||
import { config, env } from "#crawler/config";
|
||||
import { UnsupportedSourceKindError } from "#crawler/errors";
|
||||
import { SyncHttpClient } from "#crawler/http/http-client";
|
||||
import { QueueManager, createQueueManager } from "#crawler/process/async/queue";
|
||||
import {
|
||||
DetailsTaskPayload,
|
||||
ListingTaskPayload,
|
||||
ProcessingTaskPayload,
|
||||
} from "@/process/async/schemas";
|
||||
import { createPersistors, resolveCrawlerConfig } from "@/process/crawler";
|
||||
import { HtmlCrawler } from "@/process/parsers/html";
|
||||
import { WordPressCrawler } from "@/process/parsers/wordpress";
|
||||
import { Article, HtmlSourceConfig, SourceKindSchema, WordPressSourceConfig } from "@/schema";
|
||||
import { createDateRange, formatDateRange, formatPageRange, resolveSourceConfig } from "@/utils";
|
||||
} from "#crawler/process/async/schemas";
|
||||
import { createPersistors, resolveCrawlerConfig } from "#crawler/process/crawler";
|
||||
import { HtmlCrawler } from "#crawler/process/parsers/html";
|
||||
import { WordPressCrawler } from "#crawler/process/parsers/wordpress";
|
||||
import { Article, HtmlSourceConfig, WordPressSourceConfig } from "#crawler/schema";
|
||||
import {
|
||||
createDateRange,
|
||||
formatDateRange,
|
||||
formatPageRange,
|
||||
resolveSourceConfig,
|
||||
} from "#crawler/utils";
|
||||
|
||||
export const collectHtmlListing = async (
|
||||
payload: ListingTaskPayload,
|
||||
@@ -107,7 +112,7 @@ export const collectArticle = async (
|
||||
});
|
||||
const persistors = createPersistors(source);
|
||||
|
||||
if (source.sourceKind === SourceKindSchema.enum.html) {
|
||||
if (source.sourceKind === "html") {
|
||||
const crawler = new HtmlCrawler(settings, { persistors });
|
||||
const html = await crawler.crawl(payload.url);
|
||||
|
||||
@@ -118,7 +123,7 @@ export const collectArticle = async (
|
||||
} as ProcessingTaskPayload);
|
||||
}
|
||||
|
||||
if (source.sourceKind === SourceKindSchema.enum.wordpress) {
|
||||
if (source.sourceKind === "wordpress") {
|
||||
const crawler = new WordPressCrawler(settings, { persistors });
|
||||
|
||||
const article = await crawler.fetchOne(payload.data ?? {}, settings.dateRange);
|
||||
@@ -134,11 +139,24 @@ export const collectArticle = async (
|
||||
export const forwardForProcessing = async (payload: ProcessingTaskPayload): Promise<Article> => {
|
||||
logger.info({ article: payload.article.title }, "Ready for downstream processing");
|
||||
|
||||
const client = new SyncHttpClient(config.fetch.client);
|
||||
const endpoint = env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT");
|
||||
try {
|
||||
logger.info({ article: payload.article.title }, "Forwarding article to API");
|
||||
|
||||
await client.post(endpoint, { json: payload.article });
|
||||
logger.info({ article: payload.article.title }, "Forwarded article to API");
|
||||
const client = new SyncHttpClient(config.fetch.client);
|
||||
const response = await client.post(env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT"), {
|
||||
headers: {
|
||||
Authorization: `${env("BASANGO_CRAWLER_TOKEN")}`,
|
||||
},
|
||||
json: payload.article,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
logger.info({ ...data }, "Article successfully forwarded to API");
|
||||
}
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Failed to forward article to API");
|
||||
}
|
||||
|
||||
return payload.article;
|
||||
};
|
||||
|
||||
@@ -3,7 +3,7 @@ import { randomUUID } from "node:crypto";
|
||||
import { JobsOptions, Queue, QueueOptions } from "bullmq";
|
||||
import IORedis from "ioredis";
|
||||
|
||||
import { FetchAsyncConfig, config } from "@/config";
|
||||
import { FetchAsyncConfig, config } from "#crawler/config";
|
||||
import {
|
||||
DetailsTaskPayload,
|
||||
DetailsTaskPayloadSchema,
|
||||
@@ -11,8 +11,8 @@ import {
|
||||
ListingTaskPayloadSchema,
|
||||
ProcessingTaskPayload,
|
||||
ProcessingTaskPayloadSchema,
|
||||
} from "@/process/async/schemas";
|
||||
import { parseRedisUrl } from "@/utils";
|
||||
} from "#crawler/process/async/schemas";
|
||||
import { parseRedisUrl } from "#crawler/utils";
|
||||
|
||||
export interface QueueBackend<T = unknown> {
|
||||
add: (name: string, data: T, opts?: JobsOptions) => Promise<{ id: string }>;
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
import { z } from "zod";
|
||||
|
||||
import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "@/schema";
|
||||
import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "#crawler/schema";
|
||||
|
||||
export const ListingTaskPayloadSchema = z.object({
|
||||
category: z.string().optional(),
|
||||
|
||||
@@ -1,13 +1,13 @@
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import * as handlers from "@/process/async/handlers";
|
||||
import { createQueueManager } from "@/process/async/queue";
|
||||
import * as handlers from "#crawler/process/async/handlers";
|
||||
import { createQueueManager } from "#crawler/process/async/queue";
|
||||
import {
|
||||
DetailsTaskPayloadSchema,
|
||||
ListingTaskPayloadSchema,
|
||||
ProcessingTaskPayloadSchema,
|
||||
} from "@/process/async/schemas";
|
||||
import { CrawlingOptions } from "@/process/crawler";
|
||||
} from "#crawler/process/async/schemas";
|
||||
import { CrawlingOptions } from "#crawler/process/crawler";
|
||||
|
||||
export const collectListing = async (payload: unknown): Promise<number> => {
|
||||
const data = ListingTaskPayloadSchema.parse(payload);
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import { QueueEvents, Worker } from "bullmq";
|
||||
import IORedis from "ioredis";
|
||||
|
||||
import { QueueFactory, QueueManager } from "@/process/async/queue";
|
||||
import { collectArticle, collectListing, forwardForProcessing } from "@/process/async/tasks";
|
||||
import { QueueFactory, QueueManager } from "#crawler/process/async/queue";
|
||||
import { collectArticle, collectListing, forwardForProcessing } from "#crawler/process/async/tasks";
|
||||
|
||||
export interface WorkerOptions {
|
||||
queueNames?: string[];
|
||||
|
||||
Reference in New Issue
Block a user