[backend, crawler] feat: support token statistics

2025-10-25 03:23:15 +02:00
parent 8e456cff75
commit 799cda6e06
32 changed files with 414 additions and 60 deletions
@@ -17,6 +17,7 @@ dependencies = [
    "markdownify>=0.13.1",
    "readability-lxml>=0.8.1",
    "beautifulsoup4>=4.13.5",
+    "tiktoken>=0.12.0",
 ]

 [dependency-groups]