feat(domain): centralize data definition

apps/crawler/src/process/async/handlers.ts

@@ -1,8 +1,8 @@
+import type { HtmlSourceConfig, WordPressSourceConfig } from "@basango/domain/crawler";
+import { Article } from "@basango/domain/models";
 import { logger } from "@basango/logger";
 
-import { config, env } from "#crawler/config";
 import { UnsupportedSourceKindError } from "#crawler/errors";
-import { SyncHttpClient } from "#crawler/http/http-client";
 import { QueueManager, createQueueManager } from "#crawler/process/async/queue";
 import {
   DetailsTaskPayload,
@@ -12,11 +12,11 @@ import {
 import { createPersistors, resolveCrawlerConfig } from "#crawler/process/crawler";
 import { HtmlCrawler } from "#crawler/process/parsers/html";
 import { WordPressCrawler } from "#crawler/process/parsers/wordpress";
-import { Article, HtmlSourceConfig, WordPressSourceConfig } from "#crawler/schema";
+import { forward } from "#crawler/process/persistence";
 import {
-  createDateRange,
-  formatDateRange,
+  createTimestampRange,
   formatPageRange,
+  formatTimestampRange,
   resolveSourceConfig,
 } from "#crawler/utils";
 
@@ -45,7 +45,7 @@ export const collectHtmlListing = async (
 
         await manager.enqueueArticle({
           category: payload.category,
-          dateRange: createDateRange(payload.dateRange),
+          dateRange: createTimestampRange(payload.dateRange),
           sourceId: payload.sourceId,
           url,
         } as DetailsTaskPayload);
@@ -85,7 +85,7 @@ export const collectWordPressListing = async (
         await manager.enqueueArticle({
           category: payload.category,
           data,
-          dateRange: createDateRange(payload.dateRange),
+          dateRange: createTimestampRange(payload.dateRange),
           sourceId: payload.sourceId,
           url,
         } as DetailsTaskPayload);
@@ -106,7 +106,7 @@ export const collectArticle = async (
   const source = resolveSourceConfig(payload.sourceId);
   const settings = resolveCrawlerConfig(source, {
     category: payload.category,
-    dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined,
+    dateRange: payload.dateRange ? formatTimestampRange(payload.dateRange) : undefined,
     pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined,
     sourceId: payload.sourceId,
   });
@@ -141,19 +141,7 @@ export const forwardForProcessing = async (payload: ProcessingTaskPayload): Prom
 
   try {
     logger.info({ article: payload.article.title }, "Forwarding article to API");
-
-    const client = new SyncHttpClient(config.fetch.client);
-    const response = await client.post(env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT"), {
-      headers: {
-        Authorization: `${env("BASANGO_CRAWLER_TOKEN")}`,
-      },
-      json: payload.article,
-    });
-
-    if (response.ok) {
-      const data = await response.json();
-      logger.info({ ...data }, "Article successfully forwarded to API");
-    }
+    await forward(payload.article);
   } catch (error) {
     logger.error({ error }, "Failed to forward article to API");
   }

apps/crawler/src/process/async/schemas.ts

@@ -1,7 +1,7 @@
+import { PageRangeSchema, TimestampRangeSchema } from "@basango/domain/crawler";
+import { articleSchema } from "@basango/domain/models";
 import { z } from "zod";
 
-import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "#crawler/schema";
-
 export const ListingTaskPayloadSchema = z.object({
   category: z.string().optional(),
   dateRange: z.string().optional(),
@@ -12,7 +12,7 @@ export const ListingTaskPayloadSchema = z.object({
 export const DetailsTaskPayloadSchema = z.object({
   category: z.string().optional(),
   data: z.any().optional(),
-  dateRange: DateRangeSchema.optional(),
+  dateRange: TimestampRangeSchema.optional(),
   page: z.number().int().nonnegative().optional(),
   pageRange: PageRangeSchema.optional(),
   sourceId: z.string(),
@@ -20,7 +20,7 @@ export const DetailsTaskPayloadSchema = z.object({
 });
 
 export const ProcessingTaskPayloadSchema = z.object({
-  article: ArticleSchema,
+  article: articleSchema,
   sourceId: z.string(),
 });