bernard-ng/basango
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat(dashboard): add reports

Browse Source
This commit is contained in:
bernard-ng
2025-11-18 13:48:34 +02:00
parent dbcd1d7485
commit 126505fc88
32 changed files with 553 additions and 170 deletions
Download Patch File Download Diff File Expand all files Collapse all files
apps/crawler/src/http/user-agent.ts
+1 -1
View File
@@ -8,7 +8,7 @@ import { DEFAULT_OPEN_GRAPH_USER_AGENT, DEFAULT_USER_AGENT } from "@basango/doma
* @author Bernard Ngandu <bernard@devscast.tech>
*/
export class UserAgents {
private static readonly USER_AGENTS: string[] = [
public static readonly USER_AGENTS: string[] = [
"Mozilla/5.0 (iPhone; CPU iPhone OS 10_4_8; like Mac OS X) AppleWebKit/603.39 (KHTML, like Gecko) Chrome/52.0.3638.271 Mobile Safari/537.5",
"Mozilla/50.0 (Linux; U; Linux x86_64; en-US) Gecko/20130401 Firefox/52.7",
"Mozilla/5.0 (Linux; U; Android 5.0; SM-P815 Build/LRX22G) AppleWebKit/600.4 (KHTML, like Gecko) Chrome/48.0.1562.260 Mobile Safari/600.0",
apps/crawler/src/process/parsers/wordpress.ts
+7 -5
View File
@@ -36,12 +36,12 @@ export class WordPressCrawler extends BaseCrawler {
readonly source: WordPressSourceConfig;
private categoryMap: Map<number, string> = new Map();
private static readonly POST_QUERY =
public static readonly POST_QUERY =
"_fields=date,slug,link,title.rendered,content.rendered,categories&orderby=date&order=desc";
private static readonly CATEGORY_QUERY =
public static readonly CATEGORY_QUERY =
"_fields=id,slug,count&orderby=count&order=desc&per_page=100";
private static readonly TOTAL_PAGES_HEADER = "x-wp-totalpages";
private static readonly TOTAL_POSTS_HEADER = "x-wp-total";
public static readonly TOTAL_PAGES_HEADER = "x-wp-totalpages";
public static readonly TOTAL_POSTS_HEADER = "x-wp-total";
constructor(settings: FetchCrawlerConfig, options: { persistors?: Persistor[] } = {}) {
super(settings, options);
@@ -196,7 +196,9 @@ export class WordPressCrawler extends BaseCrawler {
* @param page - Page number
*/
buildEndpointUrl(page: number): string {
return `${this.baseUrl()}wp-json/wp/v2/posts?${WordPressCrawler.POST_QUERY}&page=${page}&per_page=100`;
return `${this.baseUrl()}wp-json/wp/v2/posts?${
WordPressCrawler.POST_QUERY
}&page=${page}&per_page=100`;
}
/**