feat(api): setting up

apps/crawler/package.json

@@ -1,14 +1,14 @@
 {
   "dependencies": {
+    "@basango/encryption": "workspace:*",
     "@basango/logger": "workspace:*",
-    "@devscast/config": "^1.0.3",
     "bullmq": "^4.18.3",
-    "date-fns": "^3.6.0",
+    "date-fns": "catalog:",
     "ioredis": "^5.8.2",
     "node-html-parser": "^7.0.1",
     "tiktoken": "^1.0.22",
     "turndown": "^7.2.2",
-    "zod": "^4.1.12"
+    "zod": "catalog:"
   },
   "devDependencies": {
     "@types/turndown": "^5.0.6",

apps/crawler/src/process/persistence.ts

@@ -1,6 +1,7 @@
 import fs from "node:fs";
 import path from "node:path";
 
+import { md5 } from "@basango/encryption";
 import logger from "@basango/logger";
 
 import { Article } from "@/schema";
@@ -45,11 +46,12 @@ export const persist = async (payload: Article, persistors: Persistor[]): Promis
 
   const article = {
     ...data,
+    hash: md5(data.link),
     tokenStatistics: {
-      body: countTokens(payload.body),
-      categories: countTokens(payload.categories.join(",")),
-      excerpt: countTokens(payload.body.substring(0, 200)),
-      title: countTokens(payload.title),
+      body: countTokens(data.body),
+      categories: countTokens(data.categories.join(",")),
+      excerpt: countTokens(data.body.substring(0, 200)),
+      title: countTokens(data.title),
     },
   } as Article;
 

apps/crawler/src/schema.ts

@@ -109,6 +109,7 @@ export const ArticleTokenStatisticsSchema = z.object({
 export const ArticleSchema = z.object({
   body: z.string(),
   categories: z.array(z.string()).default([]),
+  hash: z.string().optional(),
   link: z.url(),
   metadata: ArticleMetadataSchema.optional(),
   source: z.string(),