feat(domain): centralize data definition

This commit is contained in:
2025-11-17 00:04:27 +02:00
parent e7585aa76c
commit f39635e04f
96 changed files with 3474 additions and 1167 deletions
+4
View File
@@ -1,6 +1,7 @@
{
"dependencies": {
"@basango/db": "workspace:*",
"@basango/domain": "workspace:*",
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"@hono/node-server": "^1.19.6",
@@ -19,6 +20,9 @@
"exports": {
"./trpc/routers/_app": "./src/trpc/routers/_app.ts"
},
"imports": {
"#api/*": "./src/*"
},
"name": "@basango/api",
"private": true,
"scripts": {
+3 -6
View File
@@ -1,10 +1,10 @@
import { createArticle } from "@basango/db/queries";
import { createArticleResponseSchema, createArticleSchema } from "@basango/domain/models";
import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
import type { Context } from "#api/rest/init";
import { withCrawlerAuth } from "#api/rest/middlewares/crawler";
import { withDatabase } from "#api/rest/middlewares/db";
import type { Context } from "#api/rest/types";
import { createArticleResponseSchema, createArticleSchema } from "#api/schemas/articles";
import { validateResponse } from "#api/utils/response";
const app = new OpenAPIHono<Context>();
@@ -44,10 +44,7 @@ app.openapi(
const input = c.req.valid("json");
const result = await createArticle(db, input);
return c.json(
validateResponse(result, createArticleResponseSchema) as { id: string; sourceId: string },
201,
);
return c.json(validateResponse(result, createArticleResponseSchema), 201);
},
);
-73
View File
@@ -1,73 +0,0 @@
import { z } from "@hono/zod-openapi";
const metadataSchema = z.object({
description: z.string().optional().openapi({
description: "A brief description or summary of the article.",
example: "This article discusses the latest advancements in AI technology.",
}),
image: z.url().optional().openapi({
description: "The URL of the main image associated with the article.",
example: "https://example.com/image.jpg",
}),
title: z.string().optional().openapi({
description: "The title of the article for metadata purposes.",
example: "The Rise of AI",
}),
});
export const createArticleSchema = z
.object({
body: z.string().min(1).openapi({
description: "The main content of the article.",
example: "This is the body of the article...",
}),
categories: z
.array(z.string())
.openapi({
description: "The categories or tags associated with the article.",
example: ["Technology", "AI"],
})
.optional()
.default([]),
hash: z.string().min(1).openapi({
description: "The unique hash of the article link.",
example: "d41d8cd98f00b204e9800998ecf8427e",
}),
link: z.url().openapi({
description: "The URL of the article.",
example: "https://example.com/article",
}),
metadata: metadataSchema.optional(),
publishedAt: z
.string()
.refine((value) => !Number.isNaN(Date.parse(value)), {
message: "Invalid date format",
})
.transform((value) => new Date(value))
.openapi({
description: "The publication date of the article in ISO 8601 format.",
example: "2023-01-01T00:00:00Z",
}),
sourceId: z.string().openapi({
description: "The unique identifier of the source from which the article was crawled.",
example: "radiookapi.net",
}),
title: z.string().min(1).openapi({
description: "The title of the article.",
example: "The Rise of AI",
}),
})
.openapi("CreateArticle");
export const createArticleResponseSchema = z
.object({
id: z.uuid().openapi({
description: "The unique identifier of the article.",
example: "b3b7c8e2-1f2a-4c3d-9e4f-5a6b7c8d9e0f",
}),
sourceId: z.uuid().openapi({
description: "The unique identifier of the source associated with the article.",
example: "a1a2b3c4-d5e6-7f8g-9h0i-j1k2l3m4n5o6",
}),
})
.openapi("CreateArticleResponse");
-106
View File
@@ -1,106 +0,0 @@
import { z } from "zod";
const idSchema = z.uuid().openapi({
description: "The unique identifier of the source.",
example: "b3e1c8f4-5d6a-4c9e-8f1e-2d3c4b5a6f7g",
});
const biasSchema = z.enum(["neutral", "slightly", "partisan", "extreme"]).openapi({
description: "The bias level of the source.",
example: "neutral",
});
const reliabilitySchema = z
.enum(["trusted", "reliable", "average", "low_trust", "unreliable"])
.openapi({
description: "The reliability level of the source.",
example: "trusted",
});
const transparencySchema = z.enum(["high", "medium", "low"]).openapi({
description: "The transparency level of the source.",
example: "high",
});
const credibilitySchema = z
.object({
bias: biasSchema.default("neutral"),
reliability: reliabilitySchema.default("average"),
transparency: transparencySchema.default("medium"),
})
.openapi({
description: "Credibility information about the source.",
});
export const createSourceSchema = z.object({
credibility: credibilitySchema.optional(),
description: z.string().max(1024).optional().openapi({
description: "A brief description of the source.",
example: "Radio Okapi is a Congolese radio station that provides news and information.",
}),
displayName: z.string().min(1).max(255).optional().openapi({
description: "The display name of the source.",
example: "Radio Okapi",
}),
name: z.string().min(1).max(255).openapi({
description: "The name of the source.",
example: "radiookapi.com",
}),
url: z.url().openapi({
description: "The URL of the source.",
example: "https://techcrunch.com",
}),
});
export const getSourceSchema = z.object({
id: idSchema,
});
export const getSourcePublicationGraphSchema = z.object({
days: z
.number()
.optional()
.openapi({
default: 60,
description: "",
example: 60,
})
.openapi({
description: "The number of days to include in the publication graph.",
}),
id: idSchema,
range: z
.object({
from: z.date().openapi({
description: "The start date of the range.",
}),
to: z.date().openapi({
description: "The end date of the range.",
}),
})
.optional()
.openapi({
description: "The date range for the publication graph.",
}),
});
export const getSourceCategorySharesSchema = z.object({
id: idSchema,
limit: z.number().int().min(1).max(100).optional().openapi({
default: 10,
description: "The maximum number of categories to return.",
example: 10,
}),
});
export const updateSourceSchema = z.object({
credibility: credibilitySchema.optional(),
description: createSourceSchema.shape.description,
displayName: createSourceSchema.shape.displayName,
id: idSchema,
name: createSourceSchema.shape.name.optional(),
});
export const createSourceResponseSchema = z.object({
id: idSchema,
...createSourceSchema.shape,
});
+26 -2
View File
@@ -1,10 +1,34 @@
import { createArticle } from "@basango/db/queries";
import {
createArticle,
getArticles,
getArticlesPublicationGraph,
getArticlesSourceDistribution,
} from "@basango/db/queries";
import {
createArticleSchema,
getArticlesSchema,
getDistributionsSchema,
getPublicationsSchema,
} from "@basango/domain/models";
import { createArticleSchema } from "#api/schemas/articles";
import { createTRPCRouter, protectedProcedure } from "#api/trpc/init";
export const articlesRouter = createTRPCRouter({
create: protectedProcedure.input(createArticleSchema).mutation(async ({ ctx, input }) => {
return createArticle(ctx.db, input);
}),
getPublications: protectedProcedure.input(getPublicationsSchema).query(async ({ ctx, input }) => {
return getArticlesPublicationGraph(ctx.db, input);
}),
getSourceDistribution: protectedProcedure
.input(getDistributionsSchema)
.query(async ({ ctx, input }) => {
return getArticlesSourceDistribution(ctx.db, input);
}),
list: protectedProcedure.input(getArticlesSchema).query(async ({ ctx, input }) => {
return getArticles(ctx.db, input);
}),
});
+8 -8
View File
@@ -6,14 +6,14 @@ import {
getSources,
updateSource,
} from "@basango/db/queries";
import {
createSourceSchema,
getSourceCategorySharesSchema,
getSourcePublicationGraphSchema,
getCategorySharesSchema,
getPublicationsSchema,
getSourceSchema,
updateSourceSchema,
} from "#api/schemas/sources";
} from "@basango/domain/models";
import { createTRPCRouter, protectedProcedure } from "#api/trpc/init";
export const sourcesRouter = createTRPCRouter({
@@ -21,24 +21,24 @@ export const sourcesRouter = createTRPCRouter({
return createSource(ctx.db, input);
}),
get: protectedProcedure.query(async ({ ctx }) => getSources(ctx.db)),
getById: protectedProcedure.input(getSourceSchema).query(async ({ ctx, input }) => {
return getSourceById(ctx.db, input.id);
}),
getCategoryShares: protectedProcedure
.input(getSourceCategorySharesSchema)
.input(getCategorySharesSchema)
.query(async ({ ctx, input }) => {
return getSourceCategoryShares(ctx.db, input);
}),
getPublicationGraph: protectedProcedure
.input(getSourcePublicationGraphSchema)
.input(getPublicationsSchema)
.query(async ({ ctx, input }) => {
return getSourcePublicationGraph(ctx.db, input);
}),
list: protectedProcedure.query(async ({ ctx }) => getSources(ctx.db)),
update: protectedProcedure.input(updateSourceSchema).mutation(async ({ ctx, input }) => {
return updateSource(ctx.db, input);
}),
-4
View File
@@ -1,4 +0,0 @@
export function parseInputValue(value?: string | null) {
if (value === null) return null;
return value ? JSON.parse(value) : undefined;
}
+8 -9
View File
@@ -1,20 +1,19 @@
import { logger } from "@basango/logger";
import { z } from "zod";
export function validateResponse(data: unknown, schema: z.ZodSchema) {
type ValidationSuccess<T> = z.infer<T>;
export function validateResponse<T extends z.ZodTypeAny>(
data: unknown,
schema: T,
): ValidationSuccess<T> {
const result = schema.safeParse(data);
if (!result.success) {
const cause = z.treeifyError(result.error);
logger.error({ cause }, "Response validation failed");
logger.error(cause);
return {
data: null,
details: cause,
error: "Response validation failed",
success: false,
};
throw new Error("Response validation failed");
}
return result.data;
-10
View File
@@ -1,14 +1,4 @@
{
"compilerOptions": {
"baseUrl": ".",
"composite": true,
"incremental": true,
"paths": {
"@basango/db": ["../../packages/db/src/*"],
"#api/*": ["./src/*"],
"#db/*": ["../../packages/db/src/*"]
}
},
"extends": "@basango/tsconfig/base.json",
"include": ["src"]
}
+2 -2
View File
@@ -166,8 +166,8 @@ basango/apps/crawler/
│ │ ├── crawler.ts # Main crawler interface
│ │ └── persistence.ts # Data persistence layer
│ ├── scripts/ # CLI entry points
│ │ ├── crawl.ts # Sync crawling script
│ │ ├── queue.ts # Async job scheduling
│ │ ├── sync.ts # Sync crawling script
│ │ ├── async.ts # Async job scheduling
│ │ ├── worker.ts # Worker process
│ │ └── utils.ts # CLI utilities
│ └── __tests__/ # Test files
+7 -3
View File
@@ -1,5 +1,6 @@
{
"dependencies": {
"@basango/domain": "workspace:*",
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"bullmq": "^4.18.3",
@@ -13,13 +14,16 @@
"@types/turndown": "^5.0.6",
"vitest": "^4.0.7"
},
"imports": {
"#crawler/*": "./src/*"
},
"name": "@basango/crawler",
"private": true,
"scripts": {
"clean": "rm -rf .turbo node_modules",
"crawler:async": "bun run src/scripts/queue.ts",
"crawler:push": "bun run src/scripts/sync.ts",
"crawler:sync": "bun run src/scripts/crawl.ts",
"crawler:async": "bun run src/scripts/async.ts",
"crawler:push": "bun run src/scripts/push.ts",
"crawler:sync": "bun run src/scripts/sync.ts",
"crawler:worker": "bun run src/scripts/worker.ts",
"dev": "bun run src/scripts/worker.ts",
"test": "vitest --run",
+5 -6
View File
@@ -1,15 +1,14 @@
import path from "node:path";
import { loadConfig as defineConfig } from "@devscast/config";
import { z } from "zod";
import {
DateRangeSchema,
HtmlSourceConfigSchema,
PageRangeSchema,
TimestampRangeSchema,
UpdateDirectionSchema,
WordPressSourceConfigSchema,
} from "#crawler/schema";
} from "@basango/domain/crawler";
import { loadConfig as defineConfig } from "@devscast/config";
import { z } from "zod";
export const PROJECT_DIR = path.resolve(__dirname, "../");
@@ -43,7 +42,7 @@ export const PipelineConfigSchema = z.object({
}),
crawler: z.object({
category: z.string().optional(),
dateRange: DateRangeSchema.optional(),
dateRange: TimestampRangeSchema.optional(),
direction: UpdateDirectionSchema.default("forward"),
isUpdate: z.boolean().default(false),
maxWorkers: z.number().int().positive().default(5),
-29
View File
@@ -1,29 +0,0 @@
/**
* Default date format used for parsing and formatting dates.
* Follows the "yyyy-LL-dd" pattern (e.g., "2024-06-15").
*/
export const DEFAULT_DATE_FORMAT = "yyyy-LL-dd";
/**
* Default User-Agent string for HTTP requests made by the crawler.
* Some websites may block requests with missing or generic User-Agent headers.
*/
export const DEFAULT_USER_AGENT = "Basango/0.1 (+https://github.com/bernard-ng/basango)";
/**
* User-Agent string used for Open Graph requests.
* Some services require a specific User-Agent to return Open Graph data.
*/
export const OPEN_GRAPH_USER_AGENT = "facebookexternalhit/1.1";
/**
* HTTP status codes considered transient errors.
* Used for retry logic in HTTP clients.
*/
export const TRANSIENT_HTTP_STATUSES = [429, 500, 502, 503, 504];
/**
* Default header name for Retry-After responses.
* Used when handling rate limiting.
*/
export const DEFAULT_RETRY_AFTER_HEADER = "retry-after";
+5 -4
View File
@@ -1,11 +1,12 @@
import { setTimeout as delay } from "node:timers/promises";
import { FetchClientConfig } from "#crawler/config";
import {
DEFAULT_RETRY_AFTER_HEADER,
DEFAULT_TRANSIENT_HTTP_STATUSES,
DEFAULT_USER_AGENT,
TRANSIENT_HTTP_STATUSES,
} from "#crawler/constants";
} from "@basango/domain/constants";
import { FetchClientConfig } from "#crawler/config";
import { UserAgents } from "#crawler/http/user-agent";
export type HttpHeaders = Record<string, string>;
@@ -187,7 +188,7 @@ export class SyncHttpClient extends BaseHttpClient {
const response = await this.fetchImpl(target, init);
if (
TRANSIENT_HTTP_STATUSES.includes(response.status as number) &&
DEFAULT_TRANSIENT_HTTP_STATUSES.includes(response.status as number) &&
attempt < this.config.maxRetries
) {
await this.maybeDelay(attempt, response, retryAfterHeader);
+3 -3
View File
@@ -1,10 +1,10 @@
import { DEFAULT_OPEN_GRAPH_USER_AGENT } from "@basango/domain/constants";
import { ArticleMetadata } from "@basango/domain/models";
import { parse } from "node-html-parser";
import { config } from "#crawler/config";
import { OPEN_GRAPH_USER_AGENT } from "#crawler/constants";
import { SyncHttpClient } from "#crawler/http/http-client";
import { UserAgents } from "#crawler/http/user-agent";
import { ArticleMetadata } from "#crawler/schema";
import { createAbsoluteUrl } from "#crawler/utils";
/**
@@ -45,7 +45,7 @@ export class OpenGraph {
constructor() {
const settings = config.fetch.client;
const provider = new UserAgents(true, OPEN_GRAPH_USER_AGENT);
const provider = new UserAgents(true, DEFAULT_OPEN_GRAPH_USER_AGENT);
this.client = new SyncHttpClient(settings, {
defaultHeaders: { "User-Agent": provider.og() },
+2 -2
View File
@@ -1,4 +1,4 @@
import { DEFAULT_USER_AGENT, OPEN_GRAPH_USER_AGENT } from "#crawler/constants";
import { DEFAULT_OPEN_GRAPH_USER_AGENT, DEFAULT_USER_AGENT } from "@basango/domain/constants";
/**
* User agent provider with optional rotation.
@@ -30,7 +30,7 @@ export class UserAgents {
}
og(): string {
return OPEN_GRAPH_USER_AGENT;
return DEFAULT_OPEN_GRAPH_USER_AGENT;
}
get(): string {
+9 -21
View File
@@ -1,8 +1,8 @@
import type { HtmlSourceConfig, WordPressSourceConfig } from "@basango/domain/crawler";
import { Article } from "@basango/domain/models";
import { logger } from "@basango/logger";
import { config, env } from "#crawler/config";
import { UnsupportedSourceKindError } from "#crawler/errors";
import { SyncHttpClient } from "#crawler/http/http-client";
import { QueueManager, createQueueManager } from "#crawler/process/async/queue";
import {
DetailsTaskPayload,
@@ -12,11 +12,11 @@ import {
import { createPersistors, resolveCrawlerConfig } from "#crawler/process/crawler";
import { HtmlCrawler } from "#crawler/process/parsers/html";
import { WordPressCrawler } from "#crawler/process/parsers/wordpress";
import { Article, HtmlSourceConfig, WordPressSourceConfig } from "#crawler/schema";
import { forward } from "#crawler/process/persistence";
import {
createDateRange,
formatDateRange,
createTimestampRange,
formatPageRange,
formatTimestampRange,
resolveSourceConfig,
} from "#crawler/utils";
@@ -45,7 +45,7 @@ export const collectHtmlListing = async (
await manager.enqueueArticle({
category: payload.category,
dateRange: createDateRange(payload.dateRange),
dateRange: createTimestampRange(payload.dateRange),
sourceId: payload.sourceId,
url,
} as DetailsTaskPayload);
@@ -85,7 +85,7 @@ export const collectWordPressListing = async (
await manager.enqueueArticle({
category: payload.category,
data,
dateRange: createDateRange(payload.dateRange),
dateRange: createTimestampRange(payload.dateRange),
sourceId: payload.sourceId,
url,
} as DetailsTaskPayload);
@@ -106,7 +106,7 @@ export const collectArticle = async (
const source = resolveSourceConfig(payload.sourceId);
const settings = resolveCrawlerConfig(source, {
category: payload.category,
dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined,
dateRange: payload.dateRange ? formatTimestampRange(payload.dateRange) : undefined,
pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined,
sourceId: payload.sourceId,
});
@@ -141,19 +141,7 @@ export const forwardForProcessing = async (payload: ProcessingTaskPayload): Prom
try {
logger.info({ article: payload.article.title }, "Forwarding article to API");
const client = new SyncHttpClient(config.fetch.client);
const response = await client.post(env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT"), {
headers: {
Authorization: `${env("BASANGO_CRAWLER_TOKEN")}`,
},
json: payload.article,
});
if (response.ok) {
const data = await response.json();
logger.info({ ...data }, "Article successfully forwarded to API");
}
await forward(payload.article);
} catch (error) {
logger.error({ error }, "Failed to forward article to API");
}
+4 -4
View File
@@ -1,7 +1,7 @@
import { PageRangeSchema, TimestampRangeSchema } from "@basango/domain/crawler";
import { articleSchema } from "@basango/domain/models";
import { z } from "zod";
import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "#crawler/schema";
export const ListingTaskPayloadSchema = z.object({
category: z.string().optional(),
dateRange: z.string().optional(),
@@ -12,7 +12,7 @@ export const ListingTaskPayloadSchema = z.object({
export const DetailsTaskPayloadSchema = z.object({
category: z.string().optional(),
data: z.any().optional(),
dateRange: DateRangeSchema.optional(),
dateRange: TimestampRangeSchema.optional(),
page: z.number().int().nonnegative().optional(),
pageRange: PageRangeSchema.optional(),
sourceId: z.string(),
@@ -20,7 +20,7 @@ export const DetailsTaskPayloadSchema = z.object({
});
export const ProcessingTaskPayloadSchema = z.object({
article: ArticleSchema,
article: articleSchema,
sourceId: z.string(),
});
+3 -3
View File
@@ -1,9 +1,9 @@
import type { AnySourceConfig } from "@basango/domain/crawler";
import logger from "@basango/logger";
import { FetchCrawlerConfig, config } from "#crawler/config";
import { JsonlPersistor, Persistor } from "#crawler/process/persistence";
import { AnySourceConfig } from "#crawler/schema";
import { createDateRange, createPageRange } from "#crawler/utils";
import { createPageRange, createTimestampRange } from "#crawler/utils";
export interface CrawlingOptions {
sourceId: string;
@@ -19,7 +19,7 @@ export const resolveCrawlerConfig = (
return {
...config.fetch.crawler,
category: options.category,
dateRange: createDateRange(options.dateRange),
dateRange: createTimestampRange(options.dateRange),
pageRange: createPageRange(options.pageRange),
source,
};
+6 -2
View File
@@ -1,10 +1,11 @@
import type { AnySourceConfig } from "@basango/domain/crawler";
import { Article } from "@basango/domain/models";
import { HTMLElement, parse as parseHtml } from "node-html-parser";
import { FetchCrawlerConfig, config } from "#crawler/config";
import { SyncHttpClient } from "#crawler/http/http-client";
import { OpenGraph } from "#crawler/http/open-graph";
import type { Persistor } from "#crawler/process/persistence";
import { AnySourceConfig, Article } from "#crawler/schema";
export interface CrawlerOptions {
persistors?: Persistor[];
@@ -97,7 +98,10 @@ export abstract class BaseCrawler {
* @param record - The article record
* @param url - The URL to fetch Open Graph data from
*/
protected async enrichWithOpenGraph(record: Article, url?: string): Promise<Article> {
protected async enrichWithOpenGraph(
record: Partial<Article>,
url?: string,
): Promise<Partial<Article>> {
try {
const metadata = url ? await this.openGraph.consumeUrl(url) : undefined;
return { ...record, metadata };
+3 -2
View File
@@ -1,3 +1,5 @@
import type { HtmlSourceConfig, TimestampRange } from "@basango/domain/crawler";
import { Article } from "@basango/domain/models";
import { logger } from "@basango/logger";
import { fromUnixTime, getUnixTime, isMatch as isDateMatch, parse } from "date-fns";
import { HTMLElement } from "node-html-parser";
@@ -12,7 +14,6 @@ import {
} from "#crawler/errors";
import { BaseCrawler } from "#crawler/process/parsers/base";
import { Persistor, persist } from "#crawler/process/persistence";
import { Article, DateRange, HtmlSourceConfig } from "#crawler/schema";
import { createAbsoluteUrl, isTimestampInRange } from "#crawler/utils";
const md = new TurndownService({
@@ -106,7 +107,7 @@ export class HtmlCrawler extends BaseCrawler {
* @param html - The HTML content of the article
* @param dateRange - Optional date range for filtering
*/
async fetchOne(html: string, dateRange?: DateRange | null): Promise<Article> {
async fetchOne(html: string, dateRange?: TimestampRange | null): Promise<Partial<Article>> {
const root = this.parseHtml(html);
const selectors = this.source.sourceSelectors;
@@ -1,3 +1,5 @@
import type { PageRange, TimestampRange, WordPressSourceConfig } from "@basango/domain/crawler";
import { Article } from "@basango/domain/models";
import { logger } from "@basango/logger";
import { fromUnixTime } from "date-fns";
import TurndownService from "turndown";
@@ -10,7 +12,6 @@ import {
} from "#crawler/errors";
import { BaseCrawler } from "#crawler/process/parsers/base";
import { Persistor, persist } from "#crawler/process/persistence";
import { Article, DateRange, PageRange, WordPressSourceConfig } from "#crawler/schema";
import { isTimestampInRange } from "#crawler/utils";
const md = new TurndownService({
@@ -107,7 +108,7 @@ export class WordPressCrawler extends BaseCrawler {
* @param input - Decoded JSON object or raw JSON string
* @param dateRange - Optional date range for filtering
*/
async fetchOne(input: unknown, dateRange?: DateRange | null): Promise<Article> {
async fetchOne(input: unknown, dateRange?: TimestampRange | null): Promise<Article> {
// input can be the decoded JSON object or a raw JSON string
let data: WordPressPost | null = null;
try {
+46 -10
View File
@@ -1,13 +1,15 @@
import fs from "node:fs";
import path from "node:path";
import type { Article } from "@basango/domain/models";
import { md5 } from "@basango/encryption";
import logger from "@basango/logger";
import { Article } from "#crawler/schema";
import { config, env } from "#crawler/config";
import { HttpError, SyncHttpClient } from "#crawler/http/http-client";
export interface Persistor {
persist(record: Article): Promise<void> | void;
persist(record: Partial<Article>): Promise<void> | void;
close: () => Promise<void> | void;
}
@@ -35,17 +37,20 @@ const sanitize = (text: string): string => {
return s.trim();
};
export const persist = async (payload: Article, persistors: Persistor[]): Promise<Article> => {
export const persist = async (
payload: Partial<Article>,
persistors: Persistor[],
): Promise<Article> => {
const data = {
...payload,
body: sanitize(payload.body),
categories: payload.categories.map(sanitize),
title: sanitize(payload.title),
body: sanitize(payload.body!),
categories: payload.categories!.map(sanitize),
title: sanitize(payload.title!),
};
const article = {
...data,
hash: md5(data.link),
hash: md5(data.link!),
} as Article;
for (const persistor of persistors) {
@@ -60,6 +65,37 @@ export const persist = async (payload: Article, persistors: Persistor[]): Promis
return article;
};
export const forward = async (payload: Partial<Article>): Promise<void> => {
const client = new SyncHttpClient(config.fetch.client);
const endpoint = env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT");
const token = env("BASANGO_CRAWLER_TOKEN");
try {
const response = await client.post(endpoint, {
headers: {
Authorization: `${token}`,
},
json: payload,
});
if (response.ok) {
const data = await response.json();
logger.info({ ...data }, "Article forwarded");
return;
}
logger.error({ status: response.status, url: payload.link }, "Forwarding failed");
} catch (error) {
if (error instanceof HttpError) {
const data = await error.response.json();
logger.error({ ...data, url: payload.link }, "Error forwarding article");
return;
}
logger.error({ error, url: payload.link }, "Error forwarding article");
}
};
export class JsonlPersistor implements Persistor {
private readonly filePath: string;
private readonly encoding: BufferEncoding;
@@ -78,15 +114,15 @@ export class JsonlPersistor implements Persistor {
}
}
persist(record: Article): Promise<void> {
persist(payload: Partial<Article>): Promise<void> {
if (this.closed) {
return Promise.reject(new Error("Persistor has been closed"));
}
const payload = `${JSON.stringify(record)}\n`;
const record = `${JSON.stringify(payload)}\n`;
this.pending = this.pending.then(async () => {
fs.appendFileSync(this.filePath, payload, { encoding: this.encoding });
fs.appendFileSync(this.filePath, record, { encoding: this.encoding });
});
return this.pending;
-130
View File
@@ -1,130 +0,0 @@
import { z } from "zod";
export const UpdateDirectionSchema = z.enum(["forward", "backward"]);
export const SourceKindSchema = z.enum(["wordpress", "html"]);
export const DateRangeSchema = z
.object({
end: z.number().int(),
start: z.number().int(),
})
.superRefine((value, ctx) => {
if (value.start === 0 || value.end === 0) {
ctx.addIssue({
code: "custom",
message: "Timestamp cannot be zero",
});
}
if (value.end < value.start) {
ctx.addIssue({
code: "custom",
message: "End timestamp must be greater than or equal to start",
});
}
});
export const PageRangeSchema = z
.object({
end: z.number().int().min(0),
start: z.number().int().min(0),
})
.superRefine((value, ctx) => {
if (value.end < value.start) {
ctx.addIssue({
code: "custom",
message: "End page must be greater than or equal to start page",
});
}
});
export const PageRangeSpecSchema = z
.string()
.regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end")
.transform((spec) => {
const [startText, endText] = spec.split(":");
return {
end: Number.parseInt(String(endText), 10),
start: Number.parseInt(String(startText), 10),
};
});
export const DateRangeSpecSchema = z
.string()
.regex(/.+:.+/, "Expected start:end format")
.transform((spec) => {
const [startRaw, endRaw] = spec.split(":");
return { endRaw: String(endRaw), startRaw: String(startRaw) };
});
export const SourceDateSchema = z.object({
format: z.string().default("yyyy-LL-dd HH:mm"),
});
const BaseSourceSchema = z.object({
categories: z.array(z.string()).default([]),
requiresDetails: z.boolean().default(false),
requiresRateLimit: z.boolean().default(false),
sourceDate: SourceDateSchema,
sourceId: z.string(),
sourceKind: SourceKindSchema,
sourceUrl: z.url(),
supportsCategories: z.boolean().default(false),
});
export const HtmlSourceConfigSchema = BaseSourceSchema.extend({
paginationTemplate: z.string(),
sourceKind: z.literal("html"),
sourceSelectors: z.object({
articleBody: z.string(),
articleCategories: z.string().optional(),
articleDate: z.string(),
articleLink: z.string(),
articles: z.string(),
articleTitle: z.string(),
pagination: z.string().default("ul.pagination > li a"),
}),
});
export const WordPressSourceConfigSchema = BaseSourceSchema.extend({
sourceDate: SourceDateSchema.default(SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" })),
sourceKind: z.literal("wordpress"),
});
export const ArticleMetadataSchema = z.object({
description: z.string().optional(),
image: z.string().optional(),
title: z.string().optional(),
url: z.url().optional(),
});
export const ArticleTokenStatisticsSchema = z.object({
body: z.number().int().nonnegative().default(0),
categories: z.number().int().nonnegative().default(0),
excerpt: z.number().int().nonnegative().default(0),
title: z.number().int().nonnegative().default(0),
});
export const ArticleSchema = z.object({
body: z.string(),
categories: z.array(z.string()).default([]),
hash: z.string().optional(),
link: z.url(),
metadata: ArticleMetadataSchema.optional(),
publishedAt: z.date(),
sourceId: z.string(),
title: z.string(),
tokenStatistics: ArticleTokenStatisticsSchema.optional(),
});
export type ArticleMetadata = z.infer<typeof ArticleMetadataSchema>;
export type Article = z.infer<typeof ArticleSchema>;
export type DateRange = z.infer<typeof DateRangeSchema>;
export type PageRange = z.infer<typeof PageRangeSchema>;
export type HtmlSourceConfig = z.infer<typeof HtmlSourceConfigSchema>;
export type WordPressSourceConfig = z.infer<typeof WordPressSourceConfigSchema>;
export type AnySourceConfig = HtmlSourceConfig | WordPressSourceConfig;
export interface CreateDateRangeOptions {
format?: string;
separator?: string;
}
@@ -1,3 +1,5 @@
#! /usr/bin/env bun
import { logger } from "@basango/logger";
import { scheduleAsyncCrawl } from "#crawler/process/async/tasks";
-23
View File
@@ -1,23 +0,0 @@
import { logger } from "@basango/logger";
import { runSyncCrawl } from "#crawler/process/sync/tasks";
import { CRAWLING_USAGE, parseCrawlingCliArgs } from "#crawler/scripts/utils";
const main = async (): Promise<void> => {
const options = parseCrawlingCliArgs();
if (options.sourceId === undefined) {
console.log(CRAWLING_USAGE);
process.exitCode = 1;
return;
}
try {
await runSyncCrawl({ ...options });
} catch (error) {
logger.error({ error }, "Synchronous crawl failed");
process.exitCode = 1;
}
};
void main();
+79
View File
@@ -0,0 +1,79 @@
#! /usr/bin/env bun
import fs from "node:fs";
import path from "node:path";
import { createInterface } from "node:readline";
import { parseArgs } from "node:util";
import type { Article } from "@basango/domain/models";
import { logger } from "@basango/logger";
import { config } from "#crawler/config";
import { forward } from "#crawler/process/persistence";
const USAGE = `
Usage: bun run crawler:sync -- --sourceId <id>
`;
const parseCliArgs = (): { sourceId?: string } => {
const { values } = parseArgs({
options: {
sourceId: { type: "string" },
},
});
return values as { sourceId?: string };
};
const main = async (): Promise<void> => {
const { sourceId } = parseCliArgs();
if (!sourceId) {
console.log(USAGE);
process.exitCode = 1;
return;
}
const filePath = path.join(config.paths.data, `${sourceId}.jsonl`);
if (!fs.existsSync(filePath)) {
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL not found");
process.exitCode = 1;
return;
}
const stat = fs.statSync(filePath);
if (stat.size === 0) {
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL is empty");
process.exitCode = 1;
return;
}
logger.info({ filePath, sourceId }, "Syncing articles from JSONL to backend");
const stream = fs.createReadStream(filePath, { encoding: "utf-8" });
const rl = createInterface({ crlfDelay: Infinity, input: stream });
let count = 0;
try {
for await (const raw of rl) {
const line = raw.trim();
if (!line) continue;
try {
const article = JSON.parse(line) as Article & { publishedAt: string };
await forward({
...article,
publishedAt: new Date(article.publishedAt),
});
count += 1;
} catch (error) {
logger.error({ error, linePreview: line.slice(0, 100) }, "Invalid JSONL line");
}
}
} finally {
rl.close();
}
logger.info({ forwarded: count, sourceId }, "Sync completed");
};
void main();
+11 -95
View File
@@ -1,109 +1,25 @@
import fs from "node:fs";
import path from "node:path";
import { createInterface } from "node:readline";
import { parseArgs } from "node:util";
#! /usr/bin/env bun
import { logger } from "@basango/logger";
import { config, env } from "#crawler/config";
import { HttpError, SyncHttpClient } from "#crawler/http/http-client";
import type { Article } from "#crawler/schema";
const USAGE = `
Usage: bun run crawler:sync -- --sourceId <id>
`;
const parseCliArgs = (): { sourceId?: string } => {
const { values } = parseArgs({
options: {
sourceId: { type: "string" },
},
});
return values as { sourceId?: string };
};
const forwardArticle = async (article: Article): Promise<void> => {
const client = new SyncHttpClient(config.fetch.client);
const endpoint = env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT");
const token = env("BASANGO_CRAWLER_TOKEN");
try {
const response = await client.post(endpoint, {
headers: {
Authorization: `${token}`,
},
json: article,
});
if (response.ok) {
const data = await response.json();
logger.info({ ...data }, "Article forwarded");
return;
}
logger.error({ link: article.link, status: response.status }, "Forwarding failed");
} catch (error) {
if (error instanceof HttpError) {
const data = await error.response.json();
logger.error({ ...data, link: article.link }, "Error forwarding article");
return;
}
logger.error({ error, link: article.link }, "Error forwarding article");
}
};
import { runSyncCrawl } from "#crawler/process/sync/tasks";
import { CRAWLING_USAGE, parseCrawlingCliArgs } from "#crawler/scripts/utils";
const main = async (): Promise<void> => {
const { sourceId } = parseCliArgs();
if (!sourceId) {
console.log(USAGE);
const options = parseCrawlingCliArgs();
if (options.sourceId === undefined) {
console.log(CRAWLING_USAGE);
process.exitCode = 1;
return;
}
const filePath = path.join(config.paths.data, `${sourceId}.jsonl`);
if (!fs.existsSync(filePath)) {
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL not found");
process.exitCode = 1;
return;
}
const stat = fs.statSync(filePath);
if (stat.size === 0) {
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL is empty");
process.exitCode = 1;
return;
}
logger.info({ filePath, sourceId }, "Syncing articles from JSONL to backend");
const stream = fs.createReadStream(filePath, { encoding: "utf-8" });
const rl = createInterface({ crlfDelay: Infinity, input: stream });
let count = 0;
try {
for await (const raw of rl) {
const line = raw.trim();
if (!line) continue;
try {
const article = JSON.parse(line) as Article & { publishedAt: string };
await forwardArticle({
...article,
publishedAt: new Date(article.publishedAt),
});
count += 1;
} catch (error) {
logger.error({ error, linePreview: line.slice(0, 100) }, "Invalid JSONL line");
}
}
} finally {
rl.close();
await runSyncCrawl({ ...options });
} catch (error) {
logger.error({ error }, "Synchronous crawl failed");
process.exitCode = 1;
}
logger.info({ forwarded: count, sourceId }, "Sync completed");
};
void main();
+2
View File
@@ -1,3 +1,5 @@
#! /usr/bin/env bun
import { logger } from "@basango/logger";
import { createQueueManager } from "#crawler/process/async/queue";
+23 -35
View File
@@ -1,20 +1,19 @@
import { format, getUnixTime, isMatch, parse } from "date-fns";
import type { RedisOptions } from "ioredis";
import { config } from "#crawler/config";
import { DEFAULT_DATE_FORMAT } from "#crawler/constants";
import { DEFAULT_DATE_FORMAT } from "@basango/domain/constants";
import {
AnySourceConfig,
CreateDateRangeOptions,
DateRange,
DateRangeSchema,
DateRangeSpecSchema,
DateSpecSchema,
HtmlSourceConfig,
PageRange,
PageRangeSchema,
PageRangeSpecSchema,
PageSpecSchema,
TimestampRange,
TimestampRangeSchema,
WordPressSourceConfig,
} from "#crawler/schema";
} from "@basango/domain/crawler";
import { format, fromUnixTime, getUnixTime, isMatch, parse } from "date-fns";
import type { RedisOptions } from "ioredis";
import { config } from "#crawler/config";
/**
* Resolve a source configuration by its ID.
@@ -71,7 +70,7 @@ const parseDate = (value: string, format: string): Date => {
*/
export const createPageRange = (spec: string | undefined): PageRange | undefined => {
if (!spec) return undefined;
const parsed = PageRangeSpecSchema.parse(spec);
const parsed = PageSpecSchema.parse(spec);
return PageRangeSchema.parse(parsed);
};
@@ -80,10 +79,13 @@ export const createPageRange = (spec: string | undefined): PageRange | undefined
* @param spec - The date range specification (e.g., "2023-01-01:2023-12-31")
* @param options - Options for date range creation
*/
export const createDateRange = (
export const createTimestampRange = (
spec: string | undefined,
options: CreateDateRangeOptions = {},
): DateRange | undefined => {
options: {
format?: string;
separator?: string;
} = {},
): TimestampRange | undefined => {
if (!spec) return undefined;
const { format = DEFAULT_DATE_FORMAT, separator = ":" } = options;
if (!separator) {
@@ -91,7 +93,7 @@ export const createDateRange = (
}
const normalized = spec.replace(separator, ":");
const parsedSpec = DateRangeSpecSchema.parse(normalized);
const parsedSpec = DateSpecSchema.parse(normalized);
const startDate = parseDate(parsedSpec.startRaw, format);
const endDate = parseDate(parsedSpec.endRaw, format);
@@ -101,7 +103,7 @@ export const createDateRange = (
start: getUnixTime(startDate),
};
return DateRangeSchema.parse(range);
return TimestampRangeSchema.parse(range);
};
/**
@@ -109,9 +111,9 @@ export const createDateRange = (
* @param range - The date range
* @param fmt - The date format (default: DEFAULT_DATE_FORMAT)
*/
export const formatDateRange = (range: DateRange, fmt = DEFAULT_DATE_FORMAT): string => {
const start = format(new Date(range.start * 1000), fmt);
const end = format(new Date(range.end * 1000), fmt);
export const formatTimestampRange = (range: TimestampRange, fmt = DEFAULT_DATE_FORMAT): string => {
const start = format(fromUnixTime(range.start), fmt);
const end = format(fromUnixTime(range.end), fmt);
return `${start}:${end}`;
};
@@ -128,7 +130,7 @@ export const formatPageRange = (range: PageRange): string => {
* @param range - The date range
* @param timestamp - The timestamp to check
*/
export const isTimestampInRange = (range: DateRange, timestamp: number): boolean => {
export const isTimestampInRange = (range: TimestampRange, timestamp: number): boolean => {
return range.start <= timestamp && timestamp <= range.end;
};
@@ -145,17 +147,3 @@ export const createAbsoluteUrl = (base: string, href: string): string => {
return href;
}
};
/**
* extract the domain name from a URL.
* @param url - The URL string
* @returns The domain name or null if invalid URL
*/
export const extractDomainName = (url: string): string | null => {
try {
const parsed = new URL(url);
return parsed.hostname;
} catch {
return null;
}
};
+1 -8
View File
@@ -1,11 +1,4 @@
{
"compilerOptions": {
"baseUrl": ".",
"paths": {
"#crawler/*": ["./src/*"]
}
},
"extends": "@basango/tsconfig/base.json",
"include": ["src"],
"references": []
"include": ["src"]
}
+1 -1
View File
@@ -16,7 +16,7 @@ const nextConfig = {
},
poweredByHeader: false,
reactStrictMode: true,
transpilePackages: ["@basango/ui", "@basango/api"],
transpilePackages: ["@basango/ui", "@basango/api", "@basango/domain"],
};
export default nextConfig;
+7 -1
View File
@@ -1,6 +1,7 @@
{
"dependencies": {
"@basango/api": "workspace:*",
"@basango/domain": "workspace:*",
"@basango/ui": "workspace:*",
"@date-fns/tz": "^1.4.1",
"@hookform/resolvers": "^5.2.2",
@@ -18,10 +19,12 @@
"next-themes": "^0.4.6",
"nuqs": "^2.7.3",
"react": "catalog:",
"react-day-picker": "^9.11.1",
"react-dom": "catalog:",
"react-hook-form": "^7.66.0",
"recharts": "^3.4.1",
"server-only": "^0.0.1",
"sonner": "^2.0.7",
"superjson": "^2.2.5",
"zod": "^4.1.12",
"zustand": "^5.0.8"
@@ -34,12 +37,15 @@
"@types/react-dom": "catalog:",
"typescript": "catalog:"
},
"imports": {
"#dashboard/*": "./src/*"
},
"name": "@basango/dashboard",
"private": true,
"scripts": {
"build": "next build",
"clean": "rm -rf .next node_modules",
"dev": "next dev",
"lint": "eslint",
"start": "next start"
}
}
@@ -1,22 +1,21 @@
import { Metadata } from "next";
import { ArticlesFeed } from "#dashboard/components/articles-feed";
import { PageLayout } from "#dashboard/components/shell/page-layout";
import { HydrateClient, batchPrefetch, trpc } from "#dashboard/trpc/server";
export const metadata: Metadata = {
title: "Articles | Basango Dashboard",
};
export default function Page() {
batchPrefetch([trpc.articles.list.infiniteQueryOptions({ limit: 12 })]);
return (
<PageLayout leading="Manage your articles" title="Articles">
<div className="flex flex-1 flex-col gap-4">
<div className="grid auto-rows-min gap-4 md:grid-cols-3">
<div className="bg-muted/50 aspect-video rounded-xl" />
<div className="bg-muted/50 aspect-video rounded-xl" />
<div className="bg-muted/50 aspect-video rounded-xl" />
</div>
<div className="bg-muted/50 min-h-screen flex-1 rounded-xl md:min-h-min" />
</div>
</PageLayout>
<HydrateClient>
<PageLayout leading="Track crawled content and trends" title="Articles">
<ArticlesFeed />
</PageLayout>
</HydrateClient>
);
}
@@ -0,0 +1,30 @@
import { Metadata } from "next";
import { PublicationGraphChart } from "#dashboard/components/charts/articles/publication-graph-chart";
import { SourceDistributionChart } from "#dashboard/components/charts/articles/source-distribution-chart";
import { PageLayout } from "#dashboard/components/shell/page-layout";
import { HydrateClient, batchPrefetch, trpc } from "#dashboard/trpc/server";
export const metadata: Metadata = {
title: "Dashboard | Basango",
};
export default async function Page() {
batchPrefetch([
trpc.articles.getPublicationGraph.queryOptions({}),
trpc.articles.getSourceDistribution.queryOptions({ limit: 8 }),
]);
return (
<HydrateClient>
<PageLayout leading="Keep track of article volume and source coverage" title="Dashboard">
<div className="grid grid-cols-1 gap-4 lg:grid-cols-4">
<div className="lg:col-span-3">
<PublicationGraphChart />
</div>
<SourceDistributionChart />
</div>
</PageLayout>
</HydrateClient>
);
}
@@ -1,9 +1,12 @@
import { Source } from "@basango/domain/models/sources";
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@basango/ui/components/tabs";
import { Metadata } from "next";
import { SourceCategorySharesChart } from "#dashboard/components/charts/source-category-shares-chart";
import { SourcePublicationgGraphChart } from "#dashboard/components/charts/source-publication-graph-chart";
import { ArticlesFeed } from "#dashboard/components/articles-feed";
import { CategorySharesChart } from "#dashboard/components/charts/sources/category-shares-chart";
import { PublicationGraphChart } from "#dashboard/components/charts/sources/publication-graph-chart";
import { PageLayout } from "#dashboard/components/shell/page-layout";
import { SourceDetailsTab } from "#dashboard/components/source-details-tab";
import { HydrateClient, batchPrefetch, getQueryClient, trpc } from "#dashboard/trpc/server";
export const metadata: Metadata = {
@@ -16,11 +19,12 @@ export default async function Page({ params }: { params: Promise<{ id: string }>
batchPrefetch([
trpc.sources.getById.queryOptions({ id }),
trpc.sources.getCategoryShares.queryOptions({ id }),
trpc.sources.getCategoryShares.queryOptions({ id, limit: 10 }),
trpc.sources.getPublicationGraph.queryOptions({ id }),
trpc.articles.list.infiniteQueryOptions({ limit: 12, sourceId: id }),
]);
const source = await queryClient.fetchQuery(trpc.sources.getById.queryOptions({ id }));
const source: Source = await queryClient.fetchQuery(trpc.sources.getById.queryOptions({ id }));
return (
<HydrateClient>
@@ -29,20 +33,17 @@ export default async function Page({ params }: { params: Promise<{ id: string }>
<TabsList>
<TabsTrigger value="overview">Overview</TabsTrigger>
<TabsTrigger value="articles">Articles</TabsTrigger>
<TabsTrigger value="details">Details</TabsTrigger>
</TabsList>
<TabsContent className="space-y-4" value="overview">
<SourceCategorySharesChart sourceId={source.id} />
<SourcePublicationgGraphChart sourceId={source.id} />
<CategorySharesChart sourceId={source.id} />
<PublicationGraphChart sourceId={source.id} />
</TabsContent>
<TabsContent value="articles">
<div className="flex flex-1 flex-col gap-4">
<div className="grid auto-rows-min gap-4 md:grid-cols-3">
<div className="bg-muted/50 aspect-video rounded-xl" />
<div className="bg-muted/50 aspect-video rounded-xl" />
<div className="bg-muted/50 aspect-video rounded-xl" />
</div>
<div className="bg-muted/50 min-h-screen flex-1 rounded-xl md:min-h-min" />
</div>
<ArticlesFeed sourceId={source.id} />
</TabsContent>
<TabsContent value="details">
<SourceDetailsTab source={source} />
</TabsContent>
</Tabs>
</PageLayout>
@@ -1,33 +1,44 @@
import { RouterOutputs } from "@basango/api/trpc/routers/_app";
import { Source } from "@basango/domain/models/sources";
import { Button } from "@basango/ui/components/button";
import { PlusIcon } from "lucide-react";
import { Metadata } from "next";
import Link from "next/link";
import { SourceCreateModal } from "#dashboard/components/modals/source-create-modal";
import { PageLayout } from "#dashboard/components/shell/page-layout";
import { SourceCard } from "#dashboard/components/widgets/source-card";
import { SourceCard } from "#dashboard/components/source-card";
import { HydrateClient, getQueryClient, prefetch, trpc } from "#dashboard/trpc/server";
export const metadata: Metadata = {
title: "Sources | Basango Dashboard",
};
type SourceDetails = RouterOutputs["sources"]["get"][number];
export default async function Page() {
const queryClient = getQueryClient();
prefetch(trpc.sources.get.queryOptions());
const sources = await queryClient.fetchQuery(trpc.sources.get.queryOptions());
const sources: Source[] = await queryClient.fetchQuery(trpc.sources.get.queryOptions());
return (
<HydrateClient>
<PageLayout leading="Manage your news sources" title="Sources">
<div className="mb-6 flex justify-end">
<Link href="?createSource=true">
<Button type="button">
<PlusIcon className="mr-2 size-4" />
Add source
</Button>
</Link>
</div>
<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
{sources.map((source: SourceDetails) => (
{sources.map((source: Source) => (
<Link href={`/sources/${source.id}`} key={source.id}>
<SourceCard source={source} />
</Link>
))}
</div>
<SourceCreateModal />
</PageLayout>
</HydrateClient>
);
@@ -0,0 +1,138 @@
"use client";
import type { RouterOutputs } from "@basango/api/trpc/routers/_app";
import { Badge } from "@basango/ui/components/badge";
import { Button } from "@basango/ui/components/button";
import { Card, CardContent, CardFooter, CardHeader, CardTitle } from "@basango/ui/components/card";
import {
DropdownMenu,
DropdownMenuContent,
DropdownMenuItem,
DropdownMenuTrigger,
} from "@basango/ui/components/dropdown-menu";
import { Skeleton } from "@basango/ui/components/skeleton";
import { ExternalLink, Link2, MoreHorizontal } from "lucide-react";
import Link from "next/link";
import * as React from "react";
import { formatDate, formatRelativeTime } from "#dashboard/utils/utils";
type Article = RouterOutputs["articles"]["list"]["items"][number];
type ArticleCardProps = {
article: Article;
};
function getDescription(article: Article) {
return (
article.metadata?.description ??
article.excerpt ??
"No description was provided for this article."
);
}
export function ArticleCard({ article }: ArticleCardProps) {
const [copied, setCopied] = React.useState(false);
const description = getDescription(article);
const imageUrl = article.image ?? undefined;
const copyLink = React.useCallback(async () => {
try {
await navigator.clipboard.writeText(article.link);
setCopied(true);
setTimeout(() => setCopied(false), 1500);
} catch {
setCopied(false);
}
}, [article.link]);
return (
<Card className="flex h-full flex-col overflow-hidden border border-border/80 p-0">
<CardHeader className="relative h-40 overflow-hidden p-0">
<div className="relative h-full w-full bg-muted">
{imageUrl ? (
<img
alt={article.title}
className="h-full w-full object-cover transition duration-200 hover:scale-105"
loading="lazy"
src={imageUrl}
/>
) : (
<div className="flex h-full w-full items-center justify-center text-xs text-muted-foreground">
No image available
</div>
)}
<div className="absolute left-3 top-3">
<Badge variant="secondary">{article.sourceName}</Badge>
</div>
<div className="absolute right-3 top-3">
<DropdownMenu>
<DropdownMenuTrigger asChild>
<Button
className="size-8 rounded-full bg-background/80 backdrop-blur"
size="icon"
variant="ghost"
>
<MoreHorizontal className="h-4 w-4" />
</Button>
</DropdownMenuTrigger>
<DropdownMenuContent align="end">
<DropdownMenuItem asChild>
<Link href={article.link} rel="noreferrer" target="_blank">
<ExternalLink className="mr-2 h-4 w-4" />
Open original
</Link>
</DropdownMenuItem>
<DropdownMenuItem onClick={copyLink}>
<Link2 className="mr-2 h-4 w-4" />
{copied ? "Copied!" : "Copy link"}
</DropdownMenuItem>
</DropdownMenuContent>
</DropdownMenu>
</div>
</div>
</CardHeader>
<CardContent className="flex flex-1 flex-col gap-3 p-4">
<CardTitle className="text-base leading-tight">
<Link
className="transition hover:text-primary hover:underline"
href={article.link}
rel="noreferrer"
target="_blank"
>
{article.title}
</Link>
</CardTitle>
<p className="text-sm text-muted-foreground line-clamp-3">{description}</p>
</CardContent>
<CardFooter className="flex items-center justify-between gap-2 px-4 py-3 text-xs text-muted-foreground">
<div className="flex flex-col">
<span className="font-medium text-foreground">
{formatDate(article.publishedAt.toISOString(), "PP", false)}
</span>
<span>{formatRelativeTime(new Date(article.publishedAt))}</span>
</div>
<span>{article.readingTime} min</span>
</CardFooter>
</Card>
);
}
export function ArticleCardSkeleton() {
return (
<Card className="flex h-full flex-col overflow-hidden p-0">
<div className="h-60 w-full bg-muted">
<Skeleton className="h-full w-full" />
</div>
<CardContent className="flex flex-1 flex-col gap-3 p-4">
<Skeleton className="h-4 w-3/4" />
<Skeleton className="h-4 w-full" />
<Skeleton className="h-4 w-5/6" />
</CardContent>
<CardFooter className="flex items-center justify-between px-4 py-3">
<Skeleton className="h-4 w-1/3" />
<Skeleton className="h-8 w-16" />
</CardFooter>
</Card>
);
}
@@ -0,0 +1,94 @@
"use client";
import { Alert, AlertDescription, AlertTitle } from "@basango/ui/components/alert";
import { Button } from "@basango/ui/components/button";
import { useInfiniteQuery } from "@tanstack/react-query";
import { Loader2 } from "lucide-react";
import * as React from "react";
import { useTRPC } from "#dashboard/trpc/client";
import { ArticleCard, ArticleCardSkeleton } from "./article-card";
type ArticlesTableProps = {
sourceId?: string;
};
const PLACEHOLDER_COUNT = 8;
export function ArticlesFeed({ sourceId }: ArticlesTableProps) {
const trpc = useTRPC();
const query = useInfiniteQuery(
trpc.articles.list.infiniteQueryOptions(
{
limit: 12,
sourceId,
},
{
getNextPageParam: (lastPage) => (lastPage.meta.hasNext ? lastPage.meta.nextCursor : null),
initialCursor: null,
},
),
);
const articles = React.useMemo(
() => query.data?.pages.flatMap((page) => page.items) ?? [],
[query.data],
);
const isInitialLoading = query.isLoading && !query.data;
return (
<div className="space-y-4">
{query.isError && (
<Alert variant="destructive">
<AlertTitle>Unable to load articles</AlertTitle>
<AlertDescription>
{query.error.message ?? "An unexpected error occurred while fetching articles."}
</AlertDescription>
</Alert>
)}
{isInitialLoading ? (
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-4">
{Array.from({ length: PLACEHOLDER_COUNT }).map((_, index) => (
<ArticleCardSkeleton key={index} />
))}
</div>
) : articles.length > 0 ? (
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-4">
{articles.map((article) => (
<ArticleCard article={article} key={article.id} />
))}
</div>
) : (
<div className="rounded-lg border border-dashed px-6 py-12 text-center text-sm text-muted-foreground">
No articles match your filters yet.
</div>
)}
<div className="flex items-center justify-center">
{query.hasNextPage ? (
<Button
disabled={query.isFetchingNextPage}
onClick={() => query.fetchNextPage()}
type="button"
variant="outline"
>
{query.isFetchingNextPage ? (
<>
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
Loading
</>
) : (
"Load more"
)}
</Button>
) : articles.length > 0 ? (
<p className="text-xs text-muted-foreground">You&apos;re all caught up.</p>
) : null}
</div>
</div>
);
}
@@ -0,0 +1,50 @@
"use client";
import { ChartTooltip, ChartTooltipContent } from "@basango/ui/components/chart";
import { Area, AreaChart as BaseAreachart, CartesianGrid, XAxis, YAxis } from "recharts";
import { formatDate, formatNumber } from "#dashboard/utils/utils";
type AreaChartProps = {
data: unknown;
};
export function AreaChart({ data }: AreaChartProps) {
return (
<BaseAreachart accessibilityLayer data={data}>
<CartesianGrid strokeDasharray="3 3" vertical={false} />
<XAxis
axisLine={false}
dataKey="date"
minTickGap={32}
tickFormatter={(value) => formatDate(String(value))}
tickLine={false}
tickMargin={8}
/>
<YAxis
allowDecimals={false}
axisLine={false}
tickFormatter={(value) => formatNumber(Number(value))}
tickLine={false}
width={48}
/>
<ChartTooltip
content={
<ChartTooltipContent
labelFormatter={(value) => formatDate(String(value), "PP")}
nameKey="count"
/>
}
cursor={{ stroke: "var(--border)", strokeDasharray: "4 4" }}
/>
<Area
dataKey="count"
fill="var(--color-count)"
fillOpacity={0.15}
stroke="var(--color-count)"
strokeWidth={2}
type="monotone"
/>
</BaseAreachart>
);
}
@@ -0,0 +1,81 @@
// @ts-nocheck
"use client";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import { ChartConfig, ChartContainer } from "@basango/ui/components/chart";
import { useQuery } from "@tanstack/react-query";
import { AreaChart } from "#dashboard/components/charts/area-chart";
import {
ChartPeriodPicker,
useChartPeriodFilter,
} from "#dashboard/components/charts/chart-filters";
import { Status } from "#dashboard/components/charts/status";
import { useTRPC } from "#dashboard/trpc/client";
import { formatNumber } from "#dashboard/utils/utils";
const chartConfig = {
count: {
color: "var(--chart-1)",
label: "Articles",
},
} satisfies ChartConfig;
export function PublicationGraphChart() {
const trpc = useTRPC();
const period = useChartPeriodFilter();
const { data } = useQuery(
trpc.articles.getPublicationGraph.queryOptions({
range: period.range,
}),
);
return (
<Card className="pt-0">
<CardHeader className="flex items-start gap-2 space-y-0 border-b py-5 sm:flex-row sm:items-center">
<div className="grid flex-1 gap-1">
<CardTitle>{formatNumber(data?.meta?.current)} articles</CardTitle>
<CardDescription>
<div className="flex items-center justify-start gap-1 text-xs">
<Status value={data?.delta} />
<span className="text-muted-foreground">vs previous</span>
</div>
</CardDescription>
</div>
<div className="flex items-center gap-3">
<ChartPeriodPicker defaultDays={period.defaultDays} paramKey="articlesPeriod" />
</div>
</CardHeader>
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
<ChartContainer className="aspect-auto h-[250px] w-full" config={chartConfig}>
<AreaChart data={data?.items} />
</ChartContainer>
</CardContent>
<CardFooter>
<CardDescription>
Showing total crawled articles for the selected period,
<div className="flex flex-wrap items-center gap-2 text-xs sm:text-sm">
<span className="font-semibold text-foreground">
{formatNumber(data?.meta?.current)} vs {formatNumber(data?.meta?.previous)} articles
</span>
<Status icons={false} percentage={true} value={data?.deltaPercentage} />
<span className="text-muted-foreground">period</span>
{data?.meta?.previous === 0 && data?.meta?.current === 0 && (
<span className="text-muted-foreground">(no articles yet)</span>
)}
</div>
</CardDescription>
</CardFooter>
</Card>
);
}
@@ -0,0 +1,82 @@
// @ts-nocheck
"use client";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import {
ChartConfig,
ChartContainer,
ChartTooltip,
ChartTooltipContent,
} from "@basango/ui/components/chart";
import { useQuery } from "@tanstack/react-query";
import { Cell, Pie, PieChart } from "recharts";
import { useTRPC } from "#dashboard/trpc/client";
import { getColorFromName } from "#dashboard/utils/categories";
import { formatNumber } from "#dashboard/utils/utils";
const chartConfig = {} satisfies ChartConfig;
export function SourceDistributionChart() {
const trpc = useTRPC();
const { data } = useQuery(
trpc.articles.getSourceDistribution.queryOptions({
limit: 10,
}),
);
return (
<Card className="pt-0">
<CardHeader className="flex items-start gap-2 space-y-0 border-b py-5 sm:flex-row sm:items-center">
<div className="grid flex-1 gap-1">
<CardTitle>Source distribution</CardTitle>
<CardDescription>Share of articles by source</CardDescription>
</div>
</CardHeader>
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
<ChartContainer className="mx-auto aspect-square max-h-80 w-full" config={chartConfig}>
<PieChart>
<ChartTooltip content={<ChartTooltipContent nameKey="name" />} />
<Pie
data={data?.items}
dataKey="count"
innerRadius={70}
nameKey="name"
outerRadius={110}
paddingAngle={2}
strokeWidth={0}
>
{data?.items.map((item) => (
<Cell fill={getColorFromName(item.name)} key={item.id} />
))}
</Pie>
</PieChart>
</ChartContainer>
<ul className="mt-4 space-y-2">
{data?.items.map((item) => (
<li className="flex items-center justify-between text-sm" key={item.id}>
<span className="flex items-center gap-2">
<span
className="h-2.5 w-2.5 rounded-full"
style={{ backgroundColor: getColorFromName(item.name) }}
/>
<span className="font-medium leading-none">{item.name}</span>
</span>
<span className="text-muted-foreground">
{formatNumber(item.count)} ({item.percentage}%)
</span>
</li>
))}
</ul>
</CardContent>
</Card>
);
}
@@ -0,0 +1,28 @@
"use client";
import { ChartTooltip, ChartTooltipContent } from "@basango/ui/components/chart";
import { Bar, BarChart as BaseBarChart, CartesianGrid, XAxis } from "recharts";
import { formatDate } from "#dashboard/utils/utils";
type BarChartProps = {
data: unknown;
};
export function BarChart({ data }: BarChartProps) {
return (
<BaseBarChart accessibilityLayer data={data}>
<CartesianGrid vertical={false} />
<XAxis
axisLine={false}
dataKey="date"
minTickGap={32}
tickFormatter={(value) => formatDate(value)}
tickLine={false}
tickMargin={8}
/>
<ChartTooltip content={<ChartTooltipContent indicator="dashed" />} cursor={false} />
<Bar dataKey="count" fill="var(--color-count)" radius={4} />
</BaseBarChart>
);
}
@@ -0,0 +1,259 @@
"use client";
import { Button } from "@basango/ui/components/button";
import { Calendar } from "@basango/ui/components/calendar";
import { Popover, PopoverContent, PopoverTrigger } from "@basango/ui/components/popover";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@basango/ui/components/select";
import { ToggleGroup, ToggleGroupItem } from "@basango/ui/components/toggle-group";
import { differenceInCalendarDays, format, subDays } from "date-fns";
import { CalendarIcon, ChevronDown } from "lucide-react";
import { parseAsInteger, parseAsIsoDate, useQueryStates } from "nuqs";
import { useMemo, useState } from "react";
import { DateRange } from "react-day-picker";
const DEFAULT_PERIOD_OPTIONS = [
{ label: "Last 7 days", value: 7 },
{ label: "Last 30 days", value: 30 },
{ label: "Last 3 months", value: 90 },
{ label: "Last 6 months", value: 180 },
{ label: "Last 12 months", value: 365 },
] as const;
type DateInput = number | Date | null | undefined;
const createRangeFromDays = (days: number): DateRange => {
const end = new Date();
return {
from: subDays(end, Math.max(days - 1, 0)),
to: end,
};
};
const DEFAULT_LIMIT_OPTIONS = [
{ label: "Top 10", value: 10 },
{ label: "Top 20", value: 20 },
{ label: "Top 50", value: 50 },
] as const;
type ChartPeriodFilterOptions = {
defaultDays?: number;
paramKey?: string;
};
type ChartLimitFilterOptions = {
defaultValue?: number;
paramKey?: string;
};
export function useChartPeriodFilter(options: ChartPeriodFilterOptions = {}) {
const { defaultDays = 30, paramKey = "chartPeriod" } = options;
const fromKey = `${paramKey}From`;
const toKey = `${paramKey}To`;
const defaultRange = useMemo(() => createRangeFromDays(defaultDays), [defaultDays]);
const [state, setState] = useQueryStates({
[fromKey]: parseAsIsoDate,
[toKey]: parseAsIsoDate,
});
const from = state[fromKey] ?? undefined;
const to = state[toKey] ?? undefined;
const selectedRange = useMemo(() => {
if (from || to) {
return { from, to };
}
return undefined;
}, [from, to]);
const range = useMemo(() => {
if (from && to) {
return { from, to };
}
return defaultRange;
}, [defaultRange, from, to]);
return {
defaultDays,
keys: { fromKey, toKey },
range,
selectedRange,
setState,
};
}
export function useChartLimitFilter(options: ChartLimitFilterOptions = {}) {
const { defaultValue = 10, paramKey = "chartLimit" } = options;
const [state, setState] = useQueryStates({
[paramKey]: parseAsInteger.withDefault(defaultValue),
});
const limit = state[paramKey];
return {
limit,
setLimit: (value: number) => {
setState({ [paramKey]: value });
},
};
}
type ChartPeriodPickerProps = ChartPeriodFilterOptions & {
options?: ReadonlyArray<{ label: string; value: number }>;
};
export function ChartPeriodPicker({
defaultDays = 30,
options = DEFAULT_PERIOD_OPTIONS,
paramKey = "chartPeriod",
disabled,
}: ChartPeriodPickerProps & { disabled?: boolean }) {
const { range, selectedRange, keys, setState } = useChartPeriodFilter({ defaultDays, paramKey });
const [open, setOpen] = useState(false);
const selectValue = useMemo(() => {
if (!range?.from || !range?.to) {
return "custom";
}
const diff = differenceInCalendarDays(range.to, range.from) + 1;
const match = options.find((option) => option.value === diff);
return match ? String(match.value) : "custom";
}, [options, range]);
const handlePresetChange = (value: string) => {
if (value === "custom") {
return;
}
const presetRange = createRangeFromDays(Number(value));
setState({
[keys.fromKey]: presetRange.from ?? null,
[keys.toKey]: presetRange.to ?? null,
});
};
const handleCalendarSelect = (value: DateRange | undefined) => {
if (value?.from && value?.to) {
setState({
[keys.fromKey]: value.from,
[keys.toKey]: value.to,
});
} else {
setState({
[keys.fromKey]: null,
[keys.toKey]: null,
});
}
};
const displayLabel =
formatDateRange(range) ??
options.find((option) => String(option.value) === selectValue)?.label ??
"Select range";
return (
<Popover onOpenChange={setOpen} open={open}>
<PopoverTrigger asChild disabled={disabled}>
<Button
className="w-full justify-start gap-2 text-left font-medium sm:w-72"
type="button"
variant="outline"
>
<CalendarIcon className="h-4 w-4 text-muted-foreground" />
<span className="flex-1 truncate">{displayLabel}</span>
<ChevronDown className="h-4 w-4 text-muted-foreground" />
</Button>
</PopoverTrigger>
<PopoverContent align="end" className="w-screen space-y-4 p-4 sm:w-[520px]" sideOffset={8}>
<Select onValueChange={handlePresetChange} value={selectValue}>
<SelectTrigger>
<SelectValue placeholder="Quick range" />
</SelectTrigger>
<SelectContent>
{options.map((option) => (
<SelectItem key={option.value} value={String(option.value)}>
{option.label}
</SelectItem>
))}
<SelectItem value="custom">Custom range</SelectItem>
</SelectContent>
</Select>
<Calendar
mode="range"
numberOfMonths={2}
onSelect={handleCalendarSelect}
selected={(selectedRange ?? range) as DateRange | undefined}
/>
<div className="flex justify-end gap-2">
<Button
onClick={() =>
setState({
[keys.fromKey]: null,
[keys.toKey]: null,
})
}
type="button"
variant="ghost"
>
Reset
</Button>
<Button onClick={() => setOpen(false)} type="button">
Done
</Button>
</div>
</PopoverContent>
</Popover>
);
}
type ChartLimitToggleProps = ChartLimitFilterOptions & {
options?: ReadonlyArray<{ label: string; value: number }>;
};
export function ChartLimitToggle({
defaultValue = 10,
options = DEFAULT_LIMIT_OPTIONS,
paramKey = "chartLimit",
}: ChartLimitToggleProps) {
const { limit, setLimit } = useChartLimitFilter({ defaultValue, paramKey });
return (
<ToggleGroup
onValueChange={(value) => {
if (value) {
setLimit(Number(value));
}
}}
type="single"
value={String(limit)}
variant="outline"
>
{options.map((option) => (
<ToggleGroupItem key={option.value} value={String(option.value)}>
{option.label}
</ToggleGroupItem>
))}
</ToggleGroup>
);
}
function formatDateRange(range?: { from?: DateInput; to?: DateInput }) {
if (!range?.from || !range?.to) return null;
return `${format(range.from, "MMM d, yyyy")} - ${format(range.to, "MMM d, yyyy")}`;
}
@@ -1,109 +0,0 @@
"use client";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import {
ChartConfig,
ChartContainer,
ChartTooltip,
ChartTooltipContent,
} from "@basango/ui/components/chart";
import {
Select,
SelectContent,
SelectItem,
SelectTrigger,
SelectValue,
} from "@basango/ui/components/select";
import { useQuery } from "@tanstack/react-query";
import * as React from "react";
import { Bar, BarChart, CartesianGrid, XAxis } from "recharts";
import { useTRPC } from "#dashboard/trpc/client";
import { formatDate } from "#dashboard/utils/utils";
const chartConfig = {
count: {
color: "var(--chart-2)",
label: "Articles",
},
views: {
label: "Articles",
},
} satisfies ChartConfig;
type Props = {
sourceId: string;
};
export function SourcePublicationgGraphChart({ sourceId }: Props) {
const trpc = useTRPC();
const [timeRange, setTimeRange] = React.useState("30");
const { data } = useQuery(
trpc.sources.getPublicationGraph.queryOptions({
days: Number(timeRange),
id: sourceId,
}),
);
return (
<Card className="pt-0">
<CardHeader className="flex items-center gap-2 space-y-0 border-b py-5 sm:flex-row">
<div className="grid flex-1 gap-1">
<CardTitle>Publication Graph</CardTitle>
<CardDescription>
Showing total crawled articles for the last {timeRange} days
</CardDescription>
</div>
<Select onValueChange={setTimeRange} value={timeRange}>
<SelectTrigger
aria-label="Select a value"
className="hidden w-40 rounded-lg sm:ml-auto sm:flex"
>
<SelectValue placeholder="Last 3 months" />
</SelectTrigger>
<SelectContent className="rounded-xl">
<SelectItem className="rounded-lg" value="7">
Last 7 days
</SelectItem>
<SelectItem className="rounded-lg" value="30">
Last 30 days
</SelectItem>
<SelectItem className="rounded-lg" value="90">
Last 3 months
</SelectItem>
<SelectItem className="rounded-lg" value="180">
Last 6 months
</SelectItem>
<SelectItem className="rounded-lg" value="365">
Last 12 months
</SelectItem>
</SelectContent>
</Select>
</CardHeader>
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
<ChartContainer className="aspect-auto h-[250px] w-full" config={chartConfig}>
<BarChart accessibilityLayer data={data?.items}>
<CartesianGrid vertical={false} />
<XAxis
axisLine={false}
dataKey="date"
minTickGap={32}
tickFormatter={(value) => formatDate(value)}
tickLine={false}
tickMargin={8}
/>
<ChartTooltip content={<ChartTooltipContent indicator="dashed" />} cursor={false} />
<Bar dataKey="count" fill="var(--color-count)" radius={4} />
</BarChart>
</ChartContainer>
</CardContent>
</Card>
);
}
@@ -1,3 +1,4 @@
// @ts-nocheck
"use client";
import {
@@ -7,11 +8,10 @@ import {
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import { ToggleGroup, ToggleGroupItem } from "@basango/ui/components/toggle-group";
import { useQuery } from "@tanstack/react-query";
import { useState } from "react";
import { Bar, BarChart, Legend, ResponsiveContainer, XAxis, YAxis } from "recharts";
import { ChartLimitToggle, useChartLimitFilter } from "#dashboard/components/charts/chart-filters";
import { useTRPC } from "#dashboard/trpc/client";
import { getColorFromName } from "#dashboard/utils/categories";
@@ -19,30 +19,24 @@ type Props = {
sourceId: string;
};
export function SourceCategorySharesChart({ sourceId }: Props) {
export function CategorySharesChart({ sourceId }: Props) {
const trpc = useTRPC();
const [limit, setLimit] = useState(10);
const { limit } = useChartLimitFilter();
const { data } = useQuery(
trpc.sources.getCategoryShares.queryOptions({
id: sourceId,
limit: limit,
limit,
}),
);
const items = data?.items ?? [];
const chartData = [
{
name: "Total",
...Object.fromEntries(items.map((item) => [item.category, item.count])),
...Object.fromEntries(data?.items.map((item) => [item.category, item.count])),
},
];
const barData = items.map((item) => ({
fill: getColorFromName(item.category),
name: item.category,
}));
return (
<Card className="pt-0">
<CardHeader className="flex items-center gap-2 space-y-0 border-b py-5 sm:flex-row">
@@ -50,17 +44,7 @@ export function SourceCategorySharesChart({ sourceId }: Props) {
<CardTitle>Category Shares</CardTitle>
<CardDescription>showing top {limit} categories for this source</CardDescription>
</div>
<ToggleGroup
className="*:data-[slot=toggle-group-item]:px-4! @[767px]/card:flex"
onValueChange={(v) => setLimit(Number(v))}
type="single"
value={String(limit)}
variant="outline"
>
<ToggleGroupItem value="10">Top 10</ToggleGroupItem>
<ToggleGroupItem value="20">Top 20</ToggleGroupItem>
<ToggleGroupItem value="50">Top 50</ToggleGroupItem>
</ToggleGroup>
<ChartLimitToggle paramKey={`categoryLimit-${sourceId}`} />
</CardHeader>
<CardContent>
<div className="-ml-1 h-20">
@@ -80,15 +64,15 @@ export function SourceCategorySharesChart({ sourceId }: Props) {
/>
<XAxis axisLine={false} fontSize={12} hide tickLine={false} type="number" />
<Legend align="left" iconSize={8} iconType="circle" />
{barData.map((entry, index) => (
{data?.items.map((entry, index) => (
<Bar
barSize={16}
className="transition-all delay-75"
dataKey={entry.name}
fill={entry.fill}
dataKey={entry.category}
fill={getColorFromName(entry.category)}
key={`bar-${index}`}
radius={
index === 0 ? [4, 0, 0, 4] : index === barData.length - 1 ? [0, 4, 4, 0] : 0
index === 0 ? [4, 0, 0, 4] : index === data?.items.length - 1 ? [0, 4, 4, 0] : 0
}
stackId="category"
/>
@@ -0,0 +1,62 @@
// @ts-nocheck
"use client";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import { ChartConfig, ChartContainer } from "@basango/ui/components/chart";
import { useQuery } from "@tanstack/react-query";
import { AreaChart } from "#dashboard/components/charts/area-chart";
import {
ChartPeriodPicker,
useChartPeriodFilter,
} from "#dashboard/components/charts/chart-filters";
import { useTRPC } from "#dashboard/trpc/client";
const chartConfig = {
count: {
color: "var(--chart-2)",
label: "Articles",
},
views: {
label: "Articles",
},
} satisfies ChartConfig;
type Props = {
sourceId: string;
};
export function PublicationGraphChart({ sourceId }: Props) {
const trpc = useTRPC();
const period = useChartPeriodFilter();
const { data } = useQuery(
trpc.sources.getPublicationGraph.queryOptions({
id: sourceId,
range: period.range,
}),
);
return (
<Card className="pt-0">
<CardHeader className="flex items-center gap-2 space-y-0 border-b py-5 sm:flex-row">
<div className="grid flex-1 gap-1">
<CardTitle>Publication Graph</CardTitle>
<CardDescription>Showing total crawled articles for the selected period</CardDescription>
</div>
<ChartPeriodPicker defaultDays={period.defaultDays} paramKey={`sourcePeriod-${sourceId}`} />
</CardHeader>
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
<ChartContainer className="aspect-auto h-[250px] w-full" config={chartConfig}>
<AreaChart data={data?.items} />
</ChartContainer>
</CardContent>
</Card>
);
}
@@ -0,0 +1,38 @@
import { Delta } from "@basango/domain/models";
import { cn } from "@basango/ui/lib/utils";
import { ArrowDownRightIcon, ArrowUpRightIcon } from "lucide-react";
import { formatNumber } from "#dashboard/utils/utils";
type StatusProps = {
value: Delta | undefined;
percentage?: boolean;
icons?: boolean;
sign?: boolean;
};
export function Status({ value, percentage = false, icons = true, sign = true }: StatusProps) {
if (value === undefined) {
return <span className="text-muted-foreground">0</span>;
}
const color = value.delta >= 0 ? "text-emerald-600" : "text-rose-600";
const icon =
value.delta >= 0 ? (
<ArrowUpRightIcon className={cn("size-4", color)} />
) : (
<ArrowDownRightIcon className={cn("size-4", color)} />
);
return (
<>
{icons && icon}
<span className={cn("font-semibold", color)}>
{sign && value.sign}
{percentage
? `${formatNumber(Math.abs(value.percentage))}%`
: formatNumber(Math.abs(value.delta))}
</span>
</>
);
}
@@ -0,0 +1,176 @@
"use client";
import type { RouterOutputs } from "@basango/api/trpc/routers/_app";
import { updateSourceSchema } from "@basango/domain/models/sources";
import {
Field,
FieldDescription,
FieldError,
FieldGroup,
FieldLabel,
} from "@basango/ui/components/field";
import { Input } from "@basango/ui/components/input";
import { SubmitButton } from "@basango/ui/components/submit-button";
import { Textarea } from "@basango/ui/components/textarea";
import { useMutation, useQueryClient } from "@tanstack/react-query";
import { useCallback, useEffect } from "react";
import { Controller } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
import { useZodForm } from "#dashboard/hooks/use-zod-form";
import { useTRPC } from "#dashboard/trpc/client";
const baseSchema = updateSourceSchema.pick({
description: true,
displayName: true,
id: true,
name: true,
});
const sourceEditSchema = z.object({
description: z
.string()
.optional()
.transform((value) => {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
})
.pipe(baseSchema.shape.description),
displayName: z
.string()
.optional()
.transform((value) => {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
})
.pipe(baseSchema.shape.displayName),
id: baseSchema.shape.id,
name: z.string().trim().pipe(baseSchema.shape.name),
});
type SourceEditValues = z.infer<typeof sourceEditSchema>;
type Props = {
source: RouterOutputs["sources"]["getById"];
};
export function SourceEditForm({ source }: Props) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const form = useZodForm(sourceEditSchema, {
defaultValues: {
description: source.description ?? "",
displayName: source.displayName ?? "",
id: source.id,
name: source.name,
},
mode: "onChange",
});
useEffect(() => {
form.reset({
description: source.description ?? "",
displayName: source.displayName ?? "",
id: source.id,
name: source.name,
});
}, [form, source.description, source.displayName, source.id, source.name]);
const mutation = useMutation(
trpc.sources.update.mutationOptions({
onError(error) {
toast.error(error.message ?? "Unable to update source.");
},
onSuccess() {
toast.success("Source updated successfully.");
void Promise.all([
queryClient.invalidateQueries({
queryKey: trpc.sources.list.queryKey(),
}),
queryClient.invalidateQueries({
queryKey: trpc.sources.getById.queryKey({ id: source.id }),
}),
]);
},
}),
);
const handleSubmit = useCallback(
(values: SourceEditValues) => {
mutation.mutate(values);
},
[mutation],
);
return (
<form className="space-y-6" onSubmit={form.handleSubmit(handleSubmit)}>
<FieldGroup>
<Controller
control={form.control}
name="name"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Name</FieldLabel>
<Input
{...field}
aria-invalid={fieldState.invalid}
autoComplete="off"
disabled={mutation.isPending}
id={field.name}
placeholder="radiookapi.com"
/>
<FieldDescription>Internal identifier of the source.</FieldDescription>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
<Controller
control={form.control}
name="displayName"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Display name</FieldLabel>
<Input
{...field}
aria-invalid={fieldState.invalid}
autoComplete="off"
disabled={mutation.isPending}
id={field.name}
placeholder="Radio Okapi"
/>
<FieldDescription>Optional friendly label shown in the dashboard.</FieldDescription>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
<Controller
control={form.control}
name="description"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Description</FieldLabel>
<Textarea
{...field}
aria-invalid={fieldState.invalid}
disabled={mutation.isPending}
id={field.name}
placeholder="Short summary about the source..."
rows={4}
/>
<FieldDescription>Optional summary shown across the product.</FieldDescription>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
</FieldGroup>
<SubmitButton className="w-full" isSubmitting={mutation.isPending} type="submit">
Save changes
</SubmitButton>
</form>
);
}
@@ -0,0 +1,186 @@
"use client";
import { createSourceSchema } from "@basango/domain/models/sources";
import {
Field,
FieldDescription,
FieldError,
FieldGroup,
FieldLabel,
} from "@basango/ui/components/field";
import { Input } from "@basango/ui/components/input";
import { SubmitButton } from "@basango/ui/components/submit-button";
import { Textarea } from "@basango/ui/components/textarea";
import { useMutation, useQueryClient } from "@tanstack/react-query";
import { useCallback } from "react";
import { Controller } from "react-hook-form";
import { toast } from "sonner";
import { z } from "zod";
import { useZodForm } from "#dashboard/hooks/use-zod-form";
import { useTRPC } from "#dashboard/trpc/client";
const baseSchema = createSourceSchema.pick({
description: true,
displayName: true,
name: true,
url: true,
});
const sourceFormSchema = z.object({
description: z
.string()
.optional()
.transform((value) => {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
})
.pipe(baseSchema.shape.description),
displayName: z
.string()
.optional()
.transform((value) => {
const trimmed = value?.trim();
return trimmed ? trimmed : undefined;
})
.pipe(baseSchema.shape.displayName),
name: z.string().trim().pipe(baseSchema.shape.name),
url: z.string().trim().pipe(baseSchema.shape.url),
});
type SourceFormValues = z.infer<typeof sourceFormSchema>;
type SourceFormProps = {
onSuccess?: () => void;
};
export function SourceForm({ onSuccess }: SourceFormProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const form = useZodForm(sourceFormSchema, {
defaultValues: {
description: "",
displayName: "",
name: "",
url: "",
},
});
const mutation = useMutation(
trpc.sources.create.mutationOptions({
onError(error) {
toast.error(error.message ?? "Unable to create source.");
},
onSuccess() {
toast.success("Source created successfully.");
queryClient.invalidateQueries({
queryKey: trpc.sources.list.queryKey(),
});
form.reset();
onSuccess?.();
},
}),
);
const handleSubmit = useCallback(
(values: SourceFormValues) => {
mutation.mutate({
...values,
});
},
[mutation],
);
return (
<form className="space-y-6" onSubmit={form.handleSubmit(handleSubmit)}>
<FieldGroup>
<Controller
control={form.control}
name="name"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Name</FieldLabel>
<Input
{...field}
aria-invalid={fieldState.invalid}
autoComplete="off"
disabled={mutation.isPending}
id={field.name}
placeholder="radiookapi.com"
/>
<FieldDescription>
This should match the unique identifier of the source.
</FieldDescription>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
<Controller
control={form.control}
name="displayName"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Display name</FieldLabel>
<Input
{...field}
aria-invalid={fieldState.invalid}
autoComplete="off"
disabled={mutation.isPending}
id={field.name}
placeholder="Radio Okapi"
/>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
<Controller
control={form.control}
name="url"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Website URL</FieldLabel>
<Input
{...field}
aria-invalid={fieldState.invalid}
autoComplete="off"
disabled={mutation.isPending}
id={field.name}
placeholder="https://techcrunch.com"
type="url"
/>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
<Controller
control={form.control}
name="description"
render={({ field, fieldState }) => (
<Field data-invalid={fieldState.invalid}>
<FieldLabel htmlFor={field.name}>Description</FieldLabel>
<Textarea
{...field}
aria-invalid={fieldState.invalid}
disabled={mutation.isPending}
id={field.name}
placeholder="Short summary about the source..."
rows={4}
/>
<FieldDescription>
Optional brief description (up to 1024 characters).
</FieldDescription>
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
</Field>
)}
/>
</FieldGroup>
<SubmitButton className="w-full" isSubmitting={mutation.isPending} type="submit">
Create source
</SubmitButton>
</form>
);
}
@@ -0,0 +1,47 @@
"use client";
import {
Dialog,
DialogContent,
DialogDescription,
DialogHeader,
DialogTitle,
} from "@basango/ui/components/dialog";
import { useCallback } from "react";
import { SourceForm } from "#dashboard/components/forms/source-form";
import { useSourceParams } from "#dashboard/hooks/use-source-params";
export function SourceCreateModal() {
const { createSource, setParams } = useSourceParams();
const isOpen = Boolean(createSource);
const openDialog = useCallback(() => {
void setParams({ createSource: true });
}, [setParams]);
const closeDialog = useCallback(() => {
void setParams(null);
}, [setParams]);
return (
<Dialog
onOpenChange={(open) => {
if (open) {
openDialog();
} else {
closeDialog();
}
}}
open={isOpen}
>
<DialogContent>
<DialogHeader>
<DialogTitle>Create a new source</DialogTitle>
<DialogDescription>Add a news outlet to start tracking its articles.</DialogDescription>
</DialogHeader>
<SourceForm onSuccess={closeDialog} />
</DialogContent>
</Dialog>
);
}
@@ -11,7 +11,7 @@ export const PageLayout = (props: React.PropsWithChildren<PageProps>) => {
return (
<div className="flex flex-1 flex-col gap-4 p-4">
<div className="container mx-auto">
<div className="container mx-auto space-y-4">
{title && (
<div className="mb-8 flex items-center justify-between space-y-4">
<div>
@@ -7,7 +7,7 @@ import {
SidebarHeader,
SidebarRail,
} from "@basango/ui/components/sidebar";
import { SquareTerminal } from "lucide-react";
import { LayoutDashboard, SquareTerminal } from "lucide-react";
import * as React from "react";
import { AppSidebarContent } from "./app-sidebar-content";
@@ -16,6 +16,18 @@ import { AppSidebarUser } from "./app-sidebar-user";
const data = {
main: [
{
icon: LayoutDashboard,
isActive: true,
items: [
{
title: "Dashboard",
url: "/dashboard",
},
],
title: "Overview",
url: "#",
},
{
icon: SquareTerminal,
isActive: true,
@@ -0,0 +1,86 @@
"use client";
import { Source } from "@basango/domain/models/sources";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import {
ChartConfig,
ChartContainer,
ChartTooltip,
ChartTooltipContent,
} from "@basango/ui/components/chart";
import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from "recharts";
import { formatDate, formatNumber } from "#dashboard/utils/utils";
const chartConfig = {
count: {
color: "var(--chart-2)",
label: "Articles",
},
views: {
label: "Articles",
},
} satisfies ChartConfig;
export function SourceCard({ source }: { source: Source }) {
return (
<Card>
<CardHeader>
<CardTitle>{source.name}</CardTitle>
<CardDescription>{source.id}</CardDescription>
</CardHeader>
<CardContent>
<ChartContainer config={chartConfig}>
<AreaChart accessibilityLayer data={source.publications?.items}>
<CartesianGrid strokeDasharray="3 3" vertical={false} />
<XAxis
axisLine={false}
dataKey="date"
minTickGap={32}
tickFormatter={(value) => formatDate(String(value))}
tickLine={false}
tickMargin={8}
/>
<YAxis
allowDecimals={false}
axisLine={false}
tickFormatter={(value) => formatNumber(Number(value))}
tickLine={false}
width={48}
/>
<ChartTooltip
content={
<ChartTooltipContent
labelFormatter={(value) => formatDate(String(value), "PP")}
nameKey="count"
/>
}
cursor={{ stroke: "var(--border)", strokeDasharray: "4 4" }}
/>
<Area
dataKey="count"
fill="var(--color-count)"
fillOpacity={0.15}
stroke="var(--color-count)"
strokeWidth={2}
type="monotone"
/>
</AreaChart>
</ChartContainer>
</CardContent>
<CardFooter className="flex-col items-start gap-2 text-sm">
<div className="flex gap-2 leading-none font-medium">
{formatNumber(source.articles)} articles crawled
</div>
<div className="text-muted-foreground leading-none">Showing last 30 days</div>
</CardFooter>
</Card>
);
}
@@ -0,0 +1,72 @@
"use client";
import { Source } from "@basango/domain/models";
import {
Card,
CardContent,
CardDescription,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import Link from "next/link";
import type { ReactNode } from "react";
import { SourceEditForm } from "#dashboard/components/forms/source-edit-form";
export function SourceDetailsTab({ source }: { source: Source }) {
const credibility = source.credibility;
return (
<div className="grid gap-4 lg:grid-cols-[2fr_1fr]">
<Card>
<CardHeader>
<CardTitle>Source details</CardTitle>
<CardDescription>Key properties of this publication.</CardDescription>
</CardHeader>
<CardContent>
<dl className="grid gap-6 sm:grid-cols-2">
<DetailItem label="Name" value={source.name} />
<DetailItem label="Display name" value={source.displayName ?? "—"} />
<DetailItem
label="Website"
value={
<Link
className="text-primary underline underline-offset-4"
href={source.url}
target="_blank"
>
{source.url}
</Link>
}
/>
<DetailItem label="Description" value={source.description ?? "Not provided"} />
<DetailItem label="Bias" value={credibility?.bias ?? "Unknown"} />
<DetailItem label="Reliability" value={credibility?.reliability ?? "Unknown"} />
<DetailItem label="Transparency" value={credibility?.transparency ?? "Unknown"} />
</dl>
</CardContent>
</Card>
<Card>
<CardHeader>
<CardTitle>Edit source</CardTitle>
<CardDescription>
Update the name or description shown inside the dashboard.
</CardDescription>
</CardHeader>
<CardContent>
<SourceEditForm source={source} />
</CardContent>
</Card>
</div>
);
}
function DetailItem({ label, value }: { label: string; value: ReactNode }) {
return (
<div className="space-y-1">
<dt className="text-muted-foreground text-sm">{label}</dt>
<dd className="text-base font-medium wrap-break-word">{value}</dd>
</div>
);
}
@@ -1,83 +0,0 @@
"use client";
import {
Card,
CardContent,
CardDescription,
CardFooter,
CardHeader,
CardTitle,
} from "@basango/ui/components/card";
import {
ChartConfig,
ChartContainer,
ChartTooltip,
ChartTooltipContent,
} from "@basango/ui/components/chart";
import { Bar, BarChart, CartesianGrid, XAxis } from "recharts";
import { formatDate, formatNumber } from "#dashboard/utils/utils";
const chartConfig = {
count: {
color: "var(--chart-2)",
label: "Articles",
},
views: {
label: "Articles",
},
} satisfies ChartConfig;
type SourceDetails = {
id: string;
name: string;
displayName: string | null;
url: string;
description: string;
publicationGraph: {
items: { date: string; count: number }[];
total: number;
};
credibility: {
bias: string;
reliability: string;
transparency: string;
};
articles: number;
};
export function SourceCard({ source }: { source: SourceDetails }) {
return (
<Card>
<CardHeader>
<CardTitle>{source.name}</CardTitle>
<CardDescription>{source.id}</CardDescription>
</CardHeader>
<CardContent>
<ChartContainer config={chartConfig}>
<BarChart accessibilityLayer data={source.publicationGraph.items}>
<CartesianGrid vertical={false} />
<XAxis
axisLine={false}
dataKey="date"
minTickGap={32}
tickFormatter={(value) => formatDate(value)}
tickLine={false}
tickMargin={8}
/>
<ChartTooltip content={<ChartTooltipContent indicator="dashed" />} cursor={false} />
<Bar dataKey="count" fill="var(--color-count)" radius={4} />
</BarChart>
</ChartContainer>
</CardContent>
<CardFooter className="flex-col items-start gap-2 text-sm">
<div className="flex gap-2 leading-none font-medium">
{formatNumber(source.articles)} articles crawled
</div>
<div className="text-muted-foreground leading-none">
Showing last {source.publicationGraph.total} days
</div>
</CardFooter>
</Card>
);
}
@@ -0,0 +1,12 @@
import { parseAsBoolean, useQueryStates } from "nuqs";
export function useSourceParams() {
const [params, setParams] = useQueryStates({
createSource: parseAsBoolean,
});
return {
...params,
setParams,
};
}
+11 -3
View File
@@ -4,7 +4,11 @@ import "server-only";
import type { AppRouter } from "@basango/api/trpc/routers/_app";
import { HydrationBoundary, dehydrate } from "@tanstack/react-query";
import { createTRPCClient, httpBatchLink, loggerLink } from "@trpc/client";
import { type TRPCQueryOptions, createTRPCOptionsProxy } from "@trpc/tanstack-react-query";
import {
type TRPCInfiniteQueryOptions,
type TRPCQueryOptions,
createTRPCOptionsProxy,
} from "@trpc/tanstack-react-query";
import { cache } from "react";
import superjson from "superjson";
@@ -46,7 +50,11 @@ export function HydrateClient(props: { children: React.ReactNode }) {
return <HydrationBoundary state={dehydrate(queryClient)}>{props.children}</HydrationBoundary>;
}
export function prefetch<T extends ReturnType<TRPCQueryOptions<any>>>(queryOptions: T) {
type AnyQueryOptions =
| ReturnType<TRPCQueryOptions<any>>
| ReturnType<TRPCInfiniteQueryOptions<any>>;
export function prefetch<T extends AnyQueryOptions>(queryOptions: T) {
const queryClient = getQueryClient();
if (queryOptions.queryKey[1]?.type === "infinite") {
void queryClient.prefetchInfiniteQuery(queryOptions as any);
@@ -55,7 +63,7 @@ export function prefetch<T extends ReturnType<TRPCQueryOptions<any>>>(queryOptio
}
}
export function batchPrefetch<T extends ReturnType<TRPCQueryOptions<any>>>(queryOptionsArray: T[]) {
export function batchPrefetch<T extends AnyQueryOptions>(queryOptionsArray: T[]) {
const queryClient = getQueryClient();
for (const queryOptions of queryOptionsArray) {
+2 -2
View File
@@ -42,8 +42,8 @@ export function formatDate(date: string, dateFormat?: string | null, checkYear =
return format(new Date(date), dateFormat ?? "P");
}
export function formatNumber(value: number): string {
return Intl.NumberFormat("en-US").format(value);
export function formatNumber(value: number | undefined): string {
return Intl.NumberFormat("en-US").format(value ?? 0);
}
export function getInitials(value: string) {
+1 -12
View File
@@ -1,11 +1,5 @@
{
"compilerOptions": {
"baseUrl": ".",
"paths": {
"@basango/api/*": ["../api/src/*"],
"@basango/ui/*": ["../../packages/ui/src/*"],
"#dashboard/*": ["./src/*"]
},
"plugins": [
{
"name": "next"
@@ -14,10 +8,5 @@
},
"exclude": ["node_modules"],
"extends": "@basango/tsconfig/nextjs.json",
"include": ["next-env.d.ts", "next.config.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
"references": [
{
"path": "../api"
}
]
"include": ["next-env.d.ts", "next.config.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"]
}