feat(domain): centralize data definition
This commit is contained in:
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@basango/db": "workspace:*",
|
||||
"@basango/domain": "workspace:*",
|
||||
"@basango/encryption": "workspace:*",
|
||||
"@basango/logger": "workspace:*",
|
||||
"@hono/node-server": "^1.19.6",
|
||||
@@ -19,6 +20,9 @@
|
||||
"exports": {
|
||||
"./trpc/routers/_app": "./src/trpc/routers/_app.ts"
|
||||
},
|
||||
"imports": {
|
||||
"#api/*": "./src/*"
|
||||
},
|
||||
"name": "@basango/api",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { createArticle } from "@basango/db/queries";
|
||||
import { createArticleResponseSchema, createArticleSchema } from "@basango/domain/models";
|
||||
import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
|
||||
|
||||
import type { Context } from "#api/rest/init";
|
||||
import { withCrawlerAuth } from "#api/rest/middlewares/crawler";
|
||||
import { withDatabase } from "#api/rest/middlewares/db";
|
||||
import type { Context } from "#api/rest/types";
|
||||
import { createArticleResponseSchema, createArticleSchema } from "#api/schemas/articles";
|
||||
import { validateResponse } from "#api/utils/response";
|
||||
|
||||
const app = new OpenAPIHono<Context>();
|
||||
@@ -44,10 +44,7 @@ app.openapi(
|
||||
const input = c.req.valid("json");
|
||||
const result = await createArticle(db, input);
|
||||
|
||||
return c.json(
|
||||
validateResponse(result, createArticleResponseSchema) as { id: string; sourceId: string },
|
||||
201,
|
||||
);
|
||||
return c.json(validateResponse(result, createArticleResponseSchema), 201);
|
||||
},
|
||||
);
|
||||
|
||||
|
||||
@@ -1,73 +0,0 @@
|
||||
import { z } from "@hono/zod-openapi";
|
||||
|
||||
const metadataSchema = z.object({
|
||||
description: z.string().optional().openapi({
|
||||
description: "A brief description or summary of the article.",
|
||||
example: "This article discusses the latest advancements in AI technology.",
|
||||
}),
|
||||
image: z.url().optional().openapi({
|
||||
description: "The URL of the main image associated with the article.",
|
||||
example: "https://example.com/image.jpg",
|
||||
}),
|
||||
title: z.string().optional().openapi({
|
||||
description: "The title of the article for metadata purposes.",
|
||||
example: "The Rise of AI",
|
||||
}),
|
||||
});
|
||||
|
||||
export const createArticleSchema = z
|
||||
.object({
|
||||
body: z.string().min(1).openapi({
|
||||
description: "The main content of the article.",
|
||||
example: "This is the body of the article...",
|
||||
}),
|
||||
categories: z
|
||||
.array(z.string())
|
||||
.openapi({
|
||||
description: "The categories or tags associated with the article.",
|
||||
example: ["Technology", "AI"],
|
||||
})
|
||||
.optional()
|
||||
.default([]),
|
||||
hash: z.string().min(1).openapi({
|
||||
description: "The unique hash of the article link.",
|
||||
example: "d41d8cd98f00b204e9800998ecf8427e",
|
||||
}),
|
||||
link: z.url().openapi({
|
||||
description: "The URL of the article.",
|
||||
example: "https://example.com/article",
|
||||
}),
|
||||
metadata: metadataSchema.optional(),
|
||||
publishedAt: z
|
||||
.string()
|
||||
.refine((value) => !Number.isNaN(Date.parse(value)), {
|
||||
message: "Invalid date format",
|
||||
})
|
||||
.transform((value) => new Date(value))
|
||||
.openapi({
|
||||
description: "The publication date of the article in ISO 8601 format.",
|
||||
example: "2023-01-01T00:00:00Z",
|
||||
}),
|
||||
sourceId: z.string().openapi({
|
||||
description: "The unique identifier of the source from which the article was crawled.",
|
||||
example: "radiookapi.net",
|
||||
}),
|
||||
title: z.string().min(1).openapi({
|
||||
description: "The title of the article.",
|
||||
example: "The Rise of AI",
|
||||
}),
|
||||
})
|
||||
.openapi("CreateArticle");
|
||||
|
||||
export const createArticleResponseSchema = z
|
||||
.object({
|
||||
id: z.uuid().openapi({
|
||||
description: "The unique identifier of the article.",
|
||||
example: "b3b7c8e2-1f2a-4c3d-9e4f-5a6b7c8d9e0f",
|
||||
}),
|
||||
sourceId: z.uuid().openapi({
|
||||
description: "The unique identifier of the source associated with the article.",
|
||||
example: "a1a2b3c4-d5e6-7f8g-9h0i-j1k2l3m4n5o6",
|
||||
}),
|
||||
})
|
||||
.openapi("CreateArticleResponse");
|
||||
@@ -1,106 +0,0 @@
|
||||
import { z } from "zod";
|
||||
|
||||
const idSchema = z.uuid().openapi({
|
||||
description: "The unique identifier of the source.",
|
||||
example: "b3e1c8f4-5d6a-4c9e-8f1e-2d3c4b5a6f7g",
|
||||
});
|
||||
|
||||
const biasSchema = z.enum(["neutral", "slightly", "partisan", "extreme"]).openapi({
|
||||
description: "The bias level of the source.",
|
||||
example: "neutral",
|
||||
});
|
||||
const reliabilitySchema = z
|
||||
.enum(["trusted", "reliable", "average", "low_trust", "unreliable"])
|
||||
.openapi({
|
||||
description: "The reliability level of the source.",
|
||||
example: "trusted",
|
||||
});
|
||||
|
||||
const transparencySchema = z.enum(["high", "medium", "low"]).openapi({
|
||||
description: "The transparency level of the source.",
|
||||
example: "high",
|
||||
});
|
||||
|
||||
const credibilitySchema = z
|
||||
.object({
|
||||
bias: biasSchema.default("neutral"),
|
||||
reliability: reliabilitySchema.default("average"),
|
||||
transparency: transparencySchema.default("medium"),
|
||||
})
|
||||
.openapi({
|
||||
description: "Credibility information about the source.",
|
||||
});
|
||||
|
||||
export const createSourceSchema = z.object({
|
||||
credibility: credibilitySchema.optional(),
|
||||
description: z.string().max(1024).optional().openapi({
|
||||
description: "A brief description of the source.",
|
||||
example: "Radio Okapi is a Congolese radio station that provides news and information.",
|
||||
}),
|
||||
displayName: z.string().min(1).max(255).optional().openapi({
|
||||
description: "The display name of the source.",
|
||||
example: "Radio Okapi",
|
||||
}),
|
||||
name: z.string().min(1).max(255).openapi({
|
||||
description: "The name of the source.",
|
||||
example: "radiookapi.com",
|
||||
}),
|
||||
url: z.url().openapi({
|
||||
description: "The URL of the source.",
|
||||
example: "https://techcrunch.com",
|
||||
}),
|
||||
});
|
||||
|
||||
export const getSourceSchema = z.object({
|
||||
id: idSchema,
|
||||
});
|
||||
|
||||
export const getSourcePublicationGraphSchema = z.object({
|
||||
days: z
|
||||
.number()
|
||||
.optional()
|
||||
.openapi({
|
||||
default: 60,
|
||||
description: "",
|
||||
example: 60,
|
||||
})
|
||||
.openapi({
|
||||
description: "The number of days to include in the publication graph.",
|
||||
}),
|
||||
id: idSchema,
|
||||
range: z
|
||||
.object({
|
||||
from: z.date().openapi({
|
||||
description: "The start date of the range.",
|
||||
}),
|
||||
to: z.date().openapi({
|
||||
description: "The end date of the range.",
|
||||
}),
|
||||
})
|
||||
.optional()
|
||||
.openapi({
|
||||
description: "The date range for the publication graph.",
|
||||
}),
|
||||
});
|
||||
|
||||
export const getSourceCategorySharesSchema = z.object({
|
||||
id: idSchema,
|
||||
limit: z.number().int().min(1).max(100).optional().openapi({
|
||||
default: 10,
|
||||
description: "The maximum number of categories to return.",
|
||||
example: 10,
|
||||
}),
|
||||
});
|
||||
|
||||
export const updateSourceSchema = z.object({
|
||||
credibility: credibilitySchema.optional(),
|
||||
description: createSourceSchema.shape.description,
|
||||
displayName: createSourceSchema.shape.displayName,
|
||||
id: idSchema,
|
||||
name: createSourceSchema.shape.name.optional(),
|
||||
});
|
||||
|
||||
export const createSourceResponseSchema = z.object({
|
||||
id: idSchema,
|
||||
...createSourceSchema.shape,
|
||||
});
|
||||
@@ -1,10 +1,34 @@
|
||||
import { createArticle } from "@basango/db/queries";
|
||||
import {
|
||||
createArticle,
|
||||
getArticles,
|
||||
getArticlesPublicationGraph,
|
||||
getArticlesSourceDistribution,
|
||||
} from "@basango/db/queries";
|
||||
import {
|
||||
createArticleSchema,
|
||||
getArticlesSchema,
|
||||
getDistributionsSchema,
|
||||
getPublicationsSchema,
|
||||
} from "@basango/domain/models";
|
||||
|
||||
import { createArticleSchema } from "#api/schemas/articles";
|
||||
import { createTRPCRouter, protectedProcedure } from "#api/trpc/init";
|
||||
|
||||
export const articlesRouter = createTRPCRouter({
|
||||
create: protectedProcedure.input(createArticleSchema).mutation(async ({ ctx, input }) => {
|
||||
return createArticle(ctx.db, input);
|
||||
}),
|
||||
|
||||
getPublications: protectedProcedure.input(getPublicationsSchema).query(async ({ ctx, input }) => {
|
||||
return getArticlesPublicationGraph(ctx.db, input);
|
||||
}),
|
||||
|
||||
getSourceDistribution: protectedProcedure
|
||||
.input(getDistributionsSchema)
|
||||
.query(async ({ ctx, input }) => {
|
||||
return getArticlesSourceDistribution(ctx.db, input);
|
||||
}),
|
||||
|
||||
list: protectedProcedure.input(getArticlesSchema).query(async ({ ctx, input }) => {
|
||||
return getArticles(ctx.db, input);
|
||||
}),
|
||||
});
|
||||
|
||||
@@ -6,14 +6,14 @@ import {
|
||||
getSources,
|
||||
updateSource,
|
||||
} from "@basango/db/queries";
|
||||
|
||||
import {
|
||||
createSourceSchema,
|
||||
getSourceCategorySharesSchema,
|
||||
getSourcePublicationGraphSchema,
|
||||
getCategorySharesSchema,
|
||||
getPublicationsSchema,
|
||||
getSourceSchema,
|
||||
updateSourceSchema,
|
||||
} from "#api/schemas/sources";
|
||||
} from "@basango/domain/models";
|
||||
|
||||
import { createTRPCRouter, protectedProcedure } from "#api/trpc/init";
|
||||
|
||||
export const sourcesRouter = createTRPCRouter({
|
||||
@@ -21,24 +21,24 @@ export const sourcesRouter = createTRPCRouter({
|
||||
return createSource(ctx.db, input);
|
||||
}),
|
||||
|
||||
get: protectedProcedure.query(async ({ ctx }) => getSources(ctx.db)),
|
||||
|
||||
getById: protectedProcedure.input(getSourceSchema).query(async ({ ctx, input }) => {
|
||||
return getSourceById(ctx.db, input.id);
|
||||
}),
|
||||
|
||||
getCategoryShares: protectedProcedure
|
||||
.input(getSourceCategorySharesSchema)
|
||||
.input(getCategorySharesSchema)
|
||||
.query(async ({ ctx, input }) => {
|
||||
return getSourceCategoryShares(ctx.db, input);
|
||||
}),
|
||||
|
||||
getPublicationGraph: protectedProcedure
|
||||
.input(getSourcePublicationGraphSchema)
|
||||
.input(getPublicationsSchema)
|
||||
.query(async ({ ctx, input }) => {
|
||||
return getSourcePublicationGraph(ctx.db, input);
|
||||
}),
|
||||
|
||||
list: protectedProcedure.query(async ({ ctx }) => getSources(ctx.db)),
|
||||
|
||||
update: protectedProcedure.input(updateSourceSchema).mutation(async ({ ctx, input }) => {
|
||||
return updateSource(ctx.db, input);
|
||||
}),
|
||||
|
||||
@@ -1,4 +0,0 @@
|
||||
export function parseInputValue(value?: string | null) {
|
||||
if (value === null) return null;
|
||||
return value ? JSON.parse(value) : undefined;
|
||||
}
|
||||
@@ -1,20 +1,19 @@
|
||||
import { logger } from "@basango/logger";
|
||||
import { z } from "zod";
|
||||
|
||||
export function validateResponse(data: unknown, schema: z.ZodSchema) {
|
||||
type ValidationSuccess<T> = z.infer<T>;
|
||||
|
||||
export function validateResponse<T extends z.ZodTypeAny>(
|
||||
data: unknown,
|
||||
schema: T,
|
||||
): ValidationSuccess<T> {
|
||||
const result = schema.safeParse(data);
|
||||
|
||||
if (!result.success) {
|
||||
const cause = z.treeifyError(result.error);
|
||||
logger.error({ cause }, "Response validation failed");
|
||||
|
||||
logger.error(cause);
|
||||
|
||||
return {
|
||||
data: null,
|
||||
details: cause,
|
||||
error: "Response validation failed",
|
||||
success: false,
|
||||
};
|
||||
throw new Error("Response validation failed");
|
||||
}
|
||||
|
||||
return result.data;
|
||||
|
||||
@@ -1,14 +1,4 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"composite": true,
|
||||
"incremental": true,
|
||||
"paths": {
|
||||
"@basango/db": ["../../packages/db/src/*"],
|
||||
"#api/*": ["./src/*"],
|
||||
"#db/*": ["../../packages/db/src/*"]
|
||||
}
|
||||
},
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"include": ["src"]
|
||||
}
|
||||
|
||||
@@ -166,8 +166,8 @@ basango/apps/crawler/
|
||||
│ │ ├── crawler.ts # Main crawler interface
|
||||
│ │ └── persistence.ts # Data persistence layer
|
||||
│ ├── scripts/ # CLI entry points
|
||||
│ │ ├── crawl.ts # Sync crawling script
|
||||
│ │ ├── queue.ts # Async job scheduling
|
||||
│ │ ├── sync.ts # Sync crawling script
|
||||
│ │ ├── async.ts # Async job scheduling
|
||||
│ │ ├── worker.ts # Worker process
|
||||
│ │ └── utils.ts # CLI utilities
|
||||
│ └── __tests__/ # Test files
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@basango/domain": "workspace:*",
|
||||
"@basango/encryption": "workspace:*",
|
||||
"@basango/logger": "workspace:*",
|
||||
"bullmq": "^4.18.3",
|
||||
@@ -13,13 +14,16 @@
|
||||
"@types/turndown": "^5.0.6",
|
||||
"vitest": "^4.0.7"
|
||||
},
|
||||
"imports": {
|
||||
"#crawler/*": "./src/*"
|
||||
},
|
||||
"name": "@basango/crawler",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"clean": "rm -rf .turbo node_modules",
|
||||
"crawler:async": "bun run src/scripts/queue.ts",
|
||||
"crawler:push": "bun run src/scripts/sync.ts",
|
||||
"crawler:sync": "bun run src/scripts/crawl.ts",
|
||||
"crawler:async": "bun run src/scripts/async.ts",
|
||||
"crawler:push": "bun run src/scripts/push.ts",
|
||||
"crawler:sync": "bun run src/scripts/sync.ts",
|
||||
"crawler:worker": "bun run src/scripts/worker.ts",
|
||||
"dev": "bun run src/scripts/worker.ts",
|
||||
"test": "vitest --run",
|
||||
|
||||
@@ -1,15 +1,14 @@
|
||||
import path from "node:path";
|
||||
|
||||
import { loadConfig as defineConfig } from "@devscast/config";
|
||||
import { z } from "zod";
|
||||
|
||||
import {
|
||||
DateRangeSchema,
|
||||
HtmlSourceConfigSchema,
|
||||
PageRangeSchema,
|
||||
TimestampRangeSchema,
|
||||
UpdateDirectionSchema,
|
||||
WordPressSourceConfigSchema,
|
||||
} from "#crawler/schema";
|
||||
} from "@basango/domain/crawler";
|
||||
import { loadConfig as defineConfig } from "@devscast/config";
|
||||
import { z } from "zod";
|
||||
|
||||
export const PROJECT_DIR = path.resolve(__dirname, "../");
|
||||
|
||||
@@ -43,7 +42,7 @@ export const PipelineConfigSchema = z.object({
|
||||
}),
|
||||
crawler: z.object({
|
||||
category: z.string().optional(),
|
||||
dateRange: DateRangeSchema.optional(),
|
||||
dateRange: TimestampRangeSchema.optional(),
|
||||
direction: UpdateDirectionSchema.default("forward"),
|
||||
isUpdate: z.boolean().default(false),
|
||||
maxWorkers: z.number().int().positive().default(5),
|
||||
|
||||
@@ -1,29 +0,0 @@
|
||||
/**
|
||||
* Default date format used for parsing and formatting dates.
|
||||
* Follows the "yyyy-LL-dd" pattern (e.g., "2024-06-15").
|
||||
*/
|
||||
export const DEFAULT_DATE_FORMAT = "yyyy-LL-dd";
|
||||
|
||||
/**
|
||||
* Default User-Agent string for HTTP requests made by the crawler.
|
||||
* Some websites may block requests with missing or generic User-Agent headers.
|
||||
*/
|
||||
export const DEFAULT_USER_AGENT = "Basango/0.1 (+https://github.com/bernard-ng/basango)";
|
||||
|
||||
/**
|
||||
* User-Agent string used for Open Graph requests.
|
||||
* Some services require a specific User-Agent to return Open Graph data.
|
||||
*/
|
||||
export const OPEN_GRAPH_USER_AGENT = "facebookexternalhit/1.1";
|
||||
|
||||
/**
|
||||
* HTTP status codes considered transient errors.
|
||||
* Used for retry logic in HTTP clients.
|
||||
*/
|
||||
export const TRANSIENT_HTTP_STATUSES = [429, 500, 502, 503, 504];
|
||||
|
||||
/**
|
||||
* Default header name for Retry-After responses.
|
||||
* Used when handling rate limiting.
|
||||
*/
|
||||
export const DEFAULT_RETRY_AFTER_HEADER = "retry-after";
|
||||
@@ -1,11 +1,12 @@
|
||||
import { setTimeout as delay } from "node:timers/promises";
|
||||
|
||||
import { FetchClientConfig } from "#crawler/config";
|
||||
import {
|
||||
DEFAULT_RETRY_AFTER_HEADER,
|
||||
DEFAULT_TRANSIENT_HTTP_STATUSES,
|
||||
DEFAULT_USER_AGENT,
|
||||
TRANSIENT_HTTP_STATUSES,
|
||||
} from "#crawler/constants";
|
||||
} from "@basango/domain/constants";
|
||||
|
||||
import { FetchClientConfig } from "#crawler/config";
|
||||
import { UserAgents } from "#crawler/http/user-agent";
|
||||
|
||||
export type HttpHeaders = Record<string, string>;
|
||||
@@ -187,7 +188,7 @@ export class SyncHttpClient extends BaseHttpClient {
|
||||
const response = await this.fetchImpl(target, init);
|
||||
|
||||
if (
|
||||
TRANSIENT_HTTP_STATUSES.includes(response.status as number) &&
|
||||
DEFAULT_TRANSIENT_HTTP_STATUSES.includes(response.status as number) &&
|
||||
attempt < this.config.maxRetries
|
||||
) {
|
||||
await this.maybeDelay(attempt, response, retryAfterHeader);
|
||||
|
||||
@@ -1,10 +1,10 @@
|
||||
import { DEFAULT_OPEN_GRAPH_USER_AGENT } from "@basango/domain/constants";
|
||||
import { ArticleMetadata } from "@basango/domain/models";
|
||||
import { parse } from "node-html-parser";
|
||||
|
||||
import { config } from "#crawler/config";
|
||||
import { OPEN_GRAPH_USER_AGENT } from "#crawler/constants";
|
||||
import { SyncHttpClient } from "#crawler/http/http-client";
|
||||
import { UserAgents } from "#crawler/http/user-agent";
|
||||
import { ArticleMetadata } from "#crawler/schema";
|
||||
import { createAbsoluteUrl } from "#crawler/utils";
|
||||
|
||||
/**
|
||||
@@ -45,7 +45,7 @@ export class OpenGraph {
|
||||
|
||||
constructor() {
|
||||
const settings = config.fetch.client;
|
||||
const provider = new UserAgents(true, OPEN_GRAPH_USER_AGENT);
|
||||
const provider = new UserAgents(true, DEFAULT_OPEN_GRAPH_USER_AGENT);
|
||||
|
||||
this.client = new SyncHttpClient(settings, {
|
||||
defaultHeaders: { "User-Agent": provider.og() },
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
import { DEFAULT_USER_AGENT, OPEN_GRAPH_USER_AGENT } from "#crawler/constants";
|
||||
import { DEFAULT_OPEN_GRAPH_USER_AGENT, DEFAULT_USER_AGENT } from "@basango/domain/constants";
|
||||
|
||||
/**
|
||||
* User agent provider with optional rotation.
|
||||
@@ -30,7 +30,7 @@ export class UserAgents {
|
||||
}
|
||||
|
||||
og(): string {
|
||||
return OPEN_GRAPH_USER_AGENT;
|
||||
return DEFAULT_OPEN_GRAPH_USER_AGENT;
|
||||
}
|
||||
|
||||
get(): string {
|
||||
|
||||
@@ -1,8 +1,8 @@
|
||||
import type { HtmlSourceConfig, WordPressSourceConfig } from "@basango/domain/crawler";
|
||||
import { Article } from "@basango/domain/models";
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { config, env } from "#crawler/config";
|
||||
import { UnsupportedSourceKindError } from "#crawler/errors";
|
||||
import { SyncHttpClient } from "#crawler/http/http-client";
|
||||
import { QueueManager, createQueueManager } from "#crawler/process/async/queue";
|
||||
import {
|
||||
DetailsTaskPayload,
|
||||
@@ -12,11 +12,11 @@ import {
|
||||
import { createPersistors, resolveCrawlerConfig } from "#crawler/process/crawler";
|
||||
import { HtmlCrawler } from "#crawler/process/parsers/html";
|
||||
import { WordPressCrawler } from "#crawler/process/parsers/wordpress";
|
||||
import { Article, HtmlSourceConfig, WordPressSourceConfig } from "#crawler/schema";
|
||||
import { forward } from "#crawler/process/persistence";
|
||||
import {
|
||||
createDateRange,
|
||||
formatDateRange,
|
||||
createTimestampRange,
|
||||
formatPageRange,
|
||||
formatTimestampRange,
|
||||
resolveSourceConfig,
|
||||
} from "#crawler/utils";
|
||||
|
||||
@@ -45,7 +45,7 @@ export const collectHtmlListing = async (
|
||||
|
||||
await manager.enqueueArticle({
|
||||
category: payload.category,
|
||||
dateRange: createDateRange(payload.dateRange),
|
||||
dateRange: createTimestampRange(payload.dateRange),
|
||||
sourceId: payload.sourceId,
|
||||
url,
|
||||
} as DetailsTaskPayload);
|
||||
@@ -85,7 +85,7 @@ export const collectWordPressListing = async (
|
||||
await manager.enqueueArticle({
|
||||
category: payload.category,
|
||||
data,
|
||||
dateRange: createDateRange(payload.dateRange),
|
||||
dateRange: createTimestampRange(payload.dateRange),
|
||||
sourceId: payload.sourceId,
|
||||
url,
|
||||
} as DetailsTaskPayload);
|
||||
@@ -106,7 +106,7 @@ export const collectArticle = async (
|
||||
const source = resolveSourceConfig(payload.sourceId);
|
||||
const settings = resolveCrawlerConfig(source, {
|
||||
category: payload.category,
|
||||
dateRange: payload.dateRange ? formatDateRange(payload.dateRange) : undefined,
|
||||
dateRange: payload.dateRange ? formatTimestampRange(payload.dateRange) : undefined,
|
||||
pageRange: payload.pageRange ? formatPageRange(payload.pageRange) : undefined,
|
||||
sourceId: payload.sourceId,
|
||||
});
|
||||
@@ -141,19 +141,7 @@ export const forwardForProcessing = async (payload: ProcessingTaskPayload): Prom
|
||||
|
||||
try {
|
||||
logger.info({ article: payload.article.title }, "Forwarding article to API");
|
||||
|
||||
const client = new SyncHttpClient(config.fetch.client);
|
||||
const response = await client.post(env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT"), {
|
||||
headers: {
|
||||
Authorization: `${env("BASANGO_CRAWLER_TOKEN")}`,
|
||||
},
|
||||
json: payload.article,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
logger.info({ ...data }, "Article successfully forwarded to API");
|
||||
}
|
||||
await forward(payload.article);
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Failed to forward article to API");
|
||||
}
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
import { PageRangeSchema, TimestampRangeSchema } from "@basango/domain/crawler";
|
||||
import { articleSchema } from "@basango/domain/models";
|
||||
import { z } from "zod";
|
||||
|
||||
import { ArticleSchema, DateRangeSchema, PageRangeSchema } from "#crawler/schema";
|
||||
|
||||
export const ListingTaskPayloadSchema = z.object({
|
||||
category: z.string().optional(),
|
||||
dateRange: z.string().optional(),
|
||||
@@ -12,7 +12,7 @@ export const ListingTaskPayloadSchema = z.object({
|
||||
export const DetailsTaskPayloadSchema = z.object({
|
||||
category: z.string().optional(),
|
||||
data: z.any().optional(),
|
||||
dateRange: DateRangeSchema.optional(),
|
||||
dateRange: TimestampRangeSchema.optional(),
|
||||
page: z.number().int().nonnegative().optional(),
|
||||
pageRange: PageRangeSchema.optional(),
|
||||
sourceId: z.string(),
|
||||
@@ -20,7 +20,7 @@ export const DetailsTaskPayloadSchema = z.object({
|
||||
});
|
||||
|
||||
export const ProcessingTaskPayloadSchema = z.object({
|
||||
article: ArticleSchema,
|
||||
article: articleSchema,
|
||||
sourceId: z.string(),
|
||||
});
|
||||
|
||||
|
||||
@@ -1,9 +1,9 @@
|
||||
import type { AnySourceConfig } from "@basango/domain/crawler";
|
||||
import logger from "@basango/logger";
|
||||
|
||||
import { FetchCrawlerConfig, config } from "#crawler/config";
|
||||
import { JsonlPersistor, Persistor } from "#crawler/process/persistence";
|
||||
import { AnySourceConfig } from "#crawler/schema";
|
||||
import { createDateRange, createPageRange } from "#crawler/utils";
|
||||
import { createPageRange, createTimestampRange } from "#crawler/utils";
|
||||
|
||||
export interface CrawlingOptions {
|
||||
sourceId: string;
|
||||
@@ -19,7 +19,7 @@ export const resolveCrawlerConfig = (
|
||||
return {
|
||||
...config.fetch.crawler,
|
||||
category: options.category,
|
||||
dateRange: createDateRange(options.dateRange),
|
||||
dateRange: createTimestampRange(options.dateRange),
|
||||
pageRange: createPageRange(options.pageRange),
|
||||
source,
|
||||
};
|
||||
|
||||
@@ -1,10 +1,11 @@
|
||||
import type { AnySourceConfig } from "@basango/domain/crawler";
|
||||
import { Article } from "@basango/domain/models";
|
||||
import { HTMLElement, parse as parseHtml } from "node-html-parser";
|
||||
|
||||
import { FetchCrawlerConfig, config } from "#crawler/config";
|
||||
import { SyncHttpClient } from "#crawler/http/http-client";
|
||||
import { OpenGraph } from "#crawler/http/open-graph";
|
||||
import type { Persistor } from "#crawler/process/persistence";
|
||||
import { AnySourceConfig, Article } from "#crawler/schema";
|
||||
|
||||
export interface CrawlerOptions {
|
||||
persistors?: Persistor[];
|
||||
@@ -97,7 +98,10 @@ export abstract class BaseCrawler {
|
||||
* @param record - The article record
|
||||
* @param url - The URL to fetch Open Graph data from
|
||||
*/
|
||||
protected async enrichWithOpenGraph(record: Article, url?: string): Promise<Article> {
|
||||
protected async enrichWithOpenGraph(
|
||||
record: Partial<Article>,
|
||||
url?: string,
|
||||
): Promise<Partial<Article>> {
|
||||
try {
|
||||
const metadata = url ? await this.openGraph.consumeUrl(url) : undefined;
|
||||
return { ...record, metadata };
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { HtmlSourceConfig, TimestampRange } from "@basango/domain/crawler";
|
||||
import { Article } from "@basango/domain/models";
|
||||
import { logger } from "@basango/logger";
|
||||
import { fromUnixTime, getUnixTime, isMatch as isDateMatch, parse } from "date-fns";
|
||||
import { HTMLElement } from "node-html-parser";
|
||||
@@ -12,7 +14,6 @@ import {
|
||||
} from "#crawler/errors";
|
||||
import { BaseCrawler } from "#crawler/process/parsers/base";
|
||||
import { Persistor, persist } from "#crawler/process/persistence";
|
||||
import { Article, DateRange, HtmlSourceConfig } from "#crawler/schema";
|
||||
import { createAbsoluteUrl, isTimestampInRange } from "#crawler/utils";
|
||||
|
||||
const md = new TurndownService({
|
||||
@@ -106,7 +107,7 @@ export class HtmlCrawler extends BaseCrawler {
|
||||
* @param html - The HTML content of the article
|
||||
* @param dateRange - Optional date range for filtering
|
||||
*/
|
||||
async fetchOne(html: string, dateRange?: DateRange | null): Promise<Article> {
|
||||
async fetchOne(html: string, dateRange?: TimestampRange | null): Promise<Partial<Article>> {
|
||||
const root = this.parseHtml(html);
|
||||
const selectors = this.source.sourceSelectors;
|
||||
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
import type { PageRange, TimestampRange, WordPressSourceConfig } from "@basango/domain/crawler";
|
||||
import { Article } from "@basango/domain/models";
|
||||
import { logger } from "@basango/logger";
|
||||
import { fromUnixTime } from "date-fns";
|
||||
import TurndownService from "turndown";
|
||||
@@ -10,7 +12,6 @@ import {
|
||||
} from "#crawler/errors";
|
||||
import { BaseCrawler } from "#crawler/process/parsers/base";
|
||||
import { Persistor, persist } from "#crawler/process/persistence";
|
||||
import { Article, DateRange, PageRange, WordPressSourceConfig } from "#crawler/schema";
|
||||
import { isTimestampInRange } from "#crawler/utils";
|
||||
|
||||
const md = new TurndownService({
|
||||
@@ -107,7 +108,7 @@ export class WordPressCrawler extends BaseCrawler {
|
||||
* @param input - Decoded JSON object or raw JSON string
|
||||
* @param dateRange - Optional date range for filtering
|
||||
*/
|
||||
async fetchOne(input: unknown, dateRange?: DateRange | null): Promise<Article> {
|
||||
async fetchOne(input: unknown, dateRange?: TimestampRange | null): Promise<Article> {
|
||||
// input can be the decoded JSON object or a raw JSON string
|
||||
let data: WordPressPost | null = null;
|
||||
try {
|
||||
|
||||
@@ -1,13 +1,15 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
|
||||
import type { Article } from "@basango/domain/models";
|
||||
import { md5 } from "@basango/encryption";
|
||||
import logger from "@basango/logger";
|
||||
|
||||
import { Article } from "#crawler/schema";
|
||||
import { config, env } from "#crawler/config";
|
||||
import { HttpError, SyncHttpClient } from "#crawler/http/http-client";
|
||||
|
||||
export interface Persistor {
|
||||
persist(record: Article): Promise<void> | void;
|
||||
persist(record: Partial<Article>): Promise<void> | void;
|
||||
close: () => Promise<void> | void;
|
||||
}
|
||||
|
||||
@@ -35,17 +37,20 @@ const sanitize = (text: string): string => {
|
||||
return s.trim();
|
||||
};
|
||||
|
||||
export const persist = async (payload: Article, persistors: Persistor[]): Promise<Article> => {
|
||||
export const persist = async (
|
||||
payload: Partial<Article>,
|
||||
persistors: Persistor[],
|
||||
): Promise<Article> => {
|
||||
const data = {
|
||||
...payload,
|
||||
body: sanitize(payload.body),
|
||||
categories: payload.categories.map(sanitize),
|
||||
title: sanitize(payload.title),
|
||||
body: sanitize(payload.body!),
|
||||
categories: payload.categories!.map(sanitize),
|
||||
title: sanitize(payload.title!),
|
||||
};
|
||||
|
||||
const article = {
|
||||
...data,
|
||||
hash: md5(data.link),
|
||||
hash: md5(data.link!),
|
||||
} as Article;
|
||||
|
||||
for (const persistor of persistors) {
|
||||
@@ -60,6 +65,37 @@ export const persist = async (payload: Article, persistors: Persistor[]): Promis
|
||||
return article;
|
||||
};
|
||||
|
||||
export const forward = async (payload: Partial<Article>): Promise<void> => {
|
||||
const client = new SyncHttpClient(config.fetch.client);
|
||||
const endpoint = env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT");
|
||||
const token = env("BASANGO_CRAWLER_TOKEN");
|
||||
|
||||
try {
|
||||
const response = await client.post(endpoint, {
|
||||
headers: {
|
||||
Authorization: `${token}`,
|
||||
},
|
||||
json: payload,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
logger.info({ ...data }, "Article forwarded");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error({ status: response.status, url: payload.link }, "Forwarding failed");
|
||||
} catch (error) {
|
||||
if (error instanceof HttpError) {
|
||||
const data = await error.response.json();
|
||||
logger.error({ ...data, url: payload.link }, "Error forwarding article");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error({ error, url: payload.link }, "Error forwarding article");
|
||||
}
|
||||
};
|
||||
|
||||
export class JsonlPersistor implements Persistor {
|
||||
private readonly filePath: string;
|
||||
private readonly encoding: BufferEncoding;
|
||||
@@ -78,15 +114,15 @@ export class JsonlPersistor implements Persistor {
|
||||
}
|
||||
}
|
||||
|
||||
persist(record: Article): Promise<void> {
|
||||
persist(payload: Partial<Article>): Promise<void> {
|
||||
if (this.closed) {
|
||||
return Promise.reject(new Error("Persistor has been closed"));
|
||||
}
|
||||
|
||||
const payload = `${JSON.stringify(record)}\n`;
|
||||
const record = `${JSON.stringify(payload)}\n`;
|
||||
|
||||
this.pending = this.pending.then(async () => {
|
||||
fs.appendFileSync(this.filePath, payload, { encoding: this.encoding });
|
||||
fs.appendFileSync(this.filePath, record, { encoding: this.encoding });
|
||||
});
|
||||
|
||||
return this.pending;
|
||||
|
||||
@@ -1,130 +0,0 @@
|
||||
import { z } from "zod";
|
||||
|
||||
export const UpdateDirectionSchema = z.enum(["forward", "backward"]);
|
||||
export const SourceKindSchema = z.enum(["wordpress", "html"]);
|
||||
|
||||
export const DateRangeSchema = z
|
||||
.object({
|
||||
end: z.number().int(),
|
||||
start: z.number().int(),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.start === 0 || value.end === 0) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "Timestamp cannot be zero",
|
||||
});
|
||||
}
|
||||
if (value.end < value.start) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "End timestamp must be greater than or equal to start",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export const PageRangeSchema = z
|
||||
.object({
|
||||
end: z.number().int().min(0),
|
||||
start: z.number().int().min(0),
|
||||
})
|
||||
.superRefine((value, ctx) => {
|
||||
if (value.end < value.start) {
|
||||
ctx.addIssue({
|
||||
code: "custom",
|
||||
message: "End page must be greater than or equal to start page",
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
export const PageRangeSpecSchema = z
|
||||
.string()
|
||||
.regex(/^[0-9]+:[0-9]+$/, "Invalid page range format. Use start:end")
|
||||
.transform((spec) => {
|
||||
const [startText, endText] = spec.split(":");
|
||||
return {
|
||||
end: Number.parseInt(String(endText), 10),
|
||||
start: Number.parseInt(String(startText), 10),
|
||||
};
|
||||
});
|
||||
|
||||
export const DateRangeSpecSchema = z
|
||||
.string()
|
||||
.regex(/.+:.+/, "Expected start:end format")
|
||||
.transform((spec) => {
|
||||
const [startRaw, endRaw] = spec.split(":");
|
||||
return { endRaw: String(endRaw), startRaw: String(startRaw) };
|
||||
});
|
||||
|
||||
export const SourceDateSchema = z.object({
|
||||
format: z.string().default("yyyy-LL-dd HH:mm"),
|
||||
});
|
||||
|
||||
const BaseSourceSchema = z.object({
|
||||
categories: z.array(z.string()).default([]),
|
||||
requiresDetails: z.boolean().default(false),
|
||||
requiresRateLimit: z.boolean().default(false),
|
||||
sourceDate: SourceDateSchema,
|
||||
sourceId: z.string(),
|
||||
sourceKind: SourceKindSchema,
|
||||
sourceUrl: z.url(),
|
||||
supportsCategories: z.boolean().default(false),
|
||||
});
|
||||
|
||||
export const HtmlSourceConfigSchema = BaseSourceSchema.extend({
|
||||
paginationTemplate: z.string(),
|
||||
sourceKind: z.literal("html"),
|
||||
sourceSelectors: z.object({
|
||||
articleBody: z.string(),
|
||||
articleCategories: z.string().optional(),
|
||||
articleDate: z.string(),
|
||||
articleLink: z.string(),
|
||||
articles: z.string(),
|
||||
articleTitle: z.string(),
|
||||
pagination: z.string().default("ul.pagination > li a"),
|
||||
}),
|
||||
});
|
||||
|
||||
export const WordPressSourceConfigSchema = BaseSourceSchema.extend({
|
||||
sourceDate: SourceDateSchema.default(SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" })),
|
||||
sourceKind: z.literal("wordpress"),
|
||||
});
|
||||
|
||||
export const ArticleMetadataSchema = z.object({
|
||||
description: z.string().optional(),
|
||||
image: z.string().optional(),
|
||||
title: z.string().optional(),
|
||||
url: z.url().optional(),
|
||||
});
|
||||
|
||||
export const ArticleTokenStatisticsSchema = z.object({
|
||||
body: z.number().int().nonnegative().default(0),
|
||||
categories: z.number().int().nonnegative().default(0),
|
||||
excerpt: z.number().int().nonnegative().default(0),
|
||||
title: z.number().int().nonnegative().default(0),
|
||||
});
|
||||
|
||||
export const ArticleSchema = z.object({
|
||||
body: z.string(),
|
||||
categories: z.array(z.string()).default([]),
|
||||
hash: z.string().optional(),
|
||||
link: z.url(),
|
||||
metadata: ArticleMetadataSchema.optional(),
|
||||
publishedAt: z.date(),
|
||||
sourceId: z.string(),
|
||||
title: z.string(),
|
||||
tokenStatistics: ArticleTokenStatisticsSchema.optional(),
|
||||
});
|
||||
|
||||
export type ArticleMetadata = z.infer<typeof ArticleMetadataSchema>;
|
||||
export type Article = z.infer<typeof ArticleSchema>;
|
||||
export type DateRange = z.infer<typeof DateRangeSchema>;
|
||||
export type PageRange = z.infer<typeof PageRangeSchema>;
|
||||
export type HtmlSourceConfig = z.infer<typeof HtmlSourceConfigSchema>;
|
||||
export type WordPressSourceConfig = z.infer<typeof WordPressSourceConfigSchema>;
|
||||
export type AnySourceConfig = HtmlSourceConfig | WordPressSourceConfig;
|
||||
|
||||
export interface CreateDateRangeOptions {
|
||||
format?: string;
|
||||
separator?: string;
|
||||
}
|
||||
@@ -1,3 +1,5 @@
|
||||
#! /usr/bin/env bun
|
||||
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { scheduleAsyncCrawl } from "#crawler/process/async/tasks";
|
||||
@@ -1,23 +0,0 @@
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { runSyncCrawl } from "#crawler/process/sync/tasks";
|
||||
import { CRAWLING_USAGE, parseCrawlingCliArgs } from "#crawler/scripts/utils";
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const options = parseCrawlingCliArgs();
|
||||
|
||||
if (options.sourceId === undefined) {
|
||||
console.log(CRAWLING_USAGE);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
await runSyncCrawl({ ...options });
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Synchronous crawl failed");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
};
|
||||
|
||||
void main();
|
||||
@@ -0,0 +1,79 @@
|
||||
#! /usr/bin/env bun
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { parseArgs } from "node:util";
|
||||
|
||||
import type { Article } from "@basango/domain/models";
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { config } from "#crawler/config";
|
||||
import { forward } from "#crawler/process/persistence";
|
||||
|
||||
const USAGE = `
|
||||
Usage: bun run crawler:sync -- --sourceId <id>
|
||||
`;
|
||||
|
||||
const parseCliArgs = (): { sourceId?: string } => {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
sourceId: { type: "string" },
|
||||
},
|
||||
});
|
||||
return values as { sourceId?: string };
|
||||
};
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const { sourceId } = parseCliArgs();
|
||||
if (!sourceId) {
|
||||
console.log(USAGE);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const filePath = path.join(config.paths.data, `${sourceId}.jsonl`);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL not found");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const stat = fs.statSync(filePath);
|
||||
if (stat.size === 0) {
|
||||
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL is empty");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ filePath, sourceId }, "Syncing articles from JSONL to backend");
|
||||
|
||||
const stream = fs.createReadStream(filePath, { encoding: "utf-8" });
|
||||
const rl = createInterface({ crlfDelay: Infinity, input: stream });
|
||||
|
||||
let count = 0;
|
||||
try {
|
||||
for await (const raw of rl) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
|
||||
try {
|
||||
const article = JSON.parse(line) as Article & { publishedAt: string };
|
||||
await forward({
|
||||
...article,
|
||||
publishedAt: new Date(article.publishedAt),
|
||||
});
|
||||
|
||||
count += 1;
|
||||
} catch (error) {
|
||||
logger.error({ error, linePreview: line.slice(0, 100) }, "Invalid JSONL line");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
|
||||
logger.info({ forwarded: count, sourceId }, "Sync completed");
|
||||
};
|
||||
|
||||
void main();
|
||||
@@ -1,109 +1,25 @@
|
||||
import fs from "node:fs";
|
||||
import path from "node:path";
|
||||
import { createInterface } from "node:readline";
|
||||
import { parseArgs } from "node:util";
|
||||
#! /usr/bin/env bun
|
||||
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { config, env } from "#crawler/config";
|
||||
import { HttpError, SyncHttpClient } from "#crawler/http/http-client";
|
||||
import type { Article } from "#crawler/schema";
|
||||
|
||||
const USAGE = `
|
||||
Usage: bun run crawler:sync -- --sourceId <id>
|
||||
`;
|
||||
|
||||
const parseCliArgs = (): { sourceId?: string } => {
|
||||
const { values } = parseArgs({
|
||||
options: {
|
||||
sourceId: { type: "string" },
|
||||
},
|
||||
});
|
||||
return values as { sourceId?: string };
|
||||
};
|
||||
|
||||
const forwardArticle = async (article: Article): Promise<void> => {
|
||||
const client = new SyncHttpClient(config.fetch.client);
|
||||
const endpoint = env("BASANGO_CRAWLER_BACKEND_API_ENDPOINT");
|
||||
const token = env("BASANGO_CRAWLER_TOKEN");
|
||||
|
||||
try {
|
||||
const response = await client.post(endpoint, {
|
||||
headers: {
|
||||
Authorization: `${token}`,
|
||||
},
|
||||
json: article,
|
||||
});
|
||||
|
||||
if (response.ok) {
|
||||
const data = await response.json();
|
||||
logger.info({ ...data }, "Article forwarded");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error({ link: article.link, status: response.status }, "Forwarding failed");
|
||||
} catch (error) {
|
||||
if (error instanceof HttpError) {
|
||||
const data = await error.response.json();
|
||||
logger.error({ ...data, link: article.link }, "Error forwarding article");
|
||||
return;
|
||||
}
|
||||
|
||||
logger.error({ error, link: article.link }, "Error forwarding article");
|
||||
}
|
||||
};
|
||||
import { runSyncCrawl } from "#crawler/process/sync/tasks";
|
||||
import { CRAWLING_USAGE, parseCrawlingCliArgs } from "#crawler/scripts/utils";
|
||||
|
||||
const main = async (): Promise<void> => {
|
||||
const { sourceId } = parseCliArgs();
|
||||
if (!sourceId) {
|
||||
console.log(USAGE);
|
||||
const options = parseCrawlingCliArgs();
|
||||
|
||||
if (options.sourceId === undefined) {
|
||||
console.log(CRAWLING_USAGE);
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const filePath = path.join(config.paths.data, `${sourceId}.jsonl`);
|
||||
|
||||
if (!fs.existsSync(filePath)) {
|
||||
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL not found");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
const stat = fs.statSync(filePath);
|
||||
if (stat.size === 0) {
|
||||
logger.error({ filePath, sourceId }, "Source must be crawled first; JSONL is empty");
|
||||
process.exitCode = 1;
|
||||
return;
|
||||
}
|
||||
|
||||
logger.info({ filePath, sourceId }, "Syncing articles from JSONL to backend");
|
||||
|
||||
const stream = fs.createReadStream(filePath, { encoding: "utf-8" });
|
||||
const rl = createInterface({ crlfDelay: Infinity, input: stream });
|
||||
|
||||
let count = 0;
|
||||
try {
|
||||
for await (const raw of rl) {
|
||||
const line = raw.trim();
|
||||
if (!line) continue;
|
||||
|
||||
try {
|
||||
const article = JSON.parse(line) as Article & { publishedAt: string };
|
||||
await forwardArticle({
|
||||
...article,
|
||||
publishedAt: new Date(article.publishedAt),
|
||||
});
|
||||
|
||||
count += 1;
|
||||
} catch (error) {
|
||||
logger.error({ error, linePreview: line.slice(0, 100) }, "Invalid JSONL line");
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
rl.close();
|
||||
await runSyncCrawl({ ...options });
|
||||
} catch (error) {
|
||||
logger.error({ error }, "Synchronous crawl failed");
|
||||
process.exitCode = 1;
|
||||
}
|
||||
|
||||
logger.info({ forwarded: count, sourceId }, "Sync completed");
|
||||
};
|
||||
|
||||
void main();
|
||||
|
||||
@@ -1,3 +1,5 @@
|
||||
#! /usr/bin/env bun
|
||||
|
||||
import { logger } from "@basango/logger";
|
||||
|
||||
import { createQueueManager } from "#crawler/process/async/queue";
|
||||
|
||||
+23
-35
@@ -1,20 +1,19 @@
|
||||
import { format, getUnixTime, isMatch, parse } from "date-fns";
|
||||
import type { RedisOptions } from "ioredis";
|
||||
|
||||
import { config } from "#crawler/config";
|
||||
import { DEFAULT_DATE_FORMAT } from "#crawler/constants";
|
||||
import { DEFAULT_DATE_FORMAT } from "@basango/domain/constants";
|
||||
import {
|
||||
AnySourceConfig,
|
||||
CreateDateRangeOptions,
|
||||
DateRange,
|
||||
DateRangeSchema,
|
||||
DateRangeSpecSchema,
|
||||
DateSpecSchema,
|
||||
HtmlSourceConfig,
|
||||
PageRange,
|
||||
PageRangeSchema,
|
||||
PageRangeSpecSchema,
|
||||
PageSpecSchema,
|
||||
TimestampRange,
|
||||
TimestampRangeSchema,
|
||||
WordPressSourceConfig,
|
||||
} from "#crawler/schema";
|
||||
} from "@basango/domain/crawler";
|
||||
import { format, fromUnixTime, getUnixTime, isMatch, parse } from "date-fns";
|
||||
import type { RedisOptions } from "ioredis";
|
||||
|
||||
import { config } from "#crawler/config";
|
||||
|
||||
/**
|
||||
* Resolve a source configuration by its ID.
|
||||
@@ -71,7 +70,7 @@ const parseDate = (value: string, format: string): Date => {
|
||||
*/
|
||||
export const createPageRange = (spec: string | undefined): PageRange | undefined => {
|
||||
if (!spec) return undefined;
|
||||
const parsed = PageRangeSpecSchema.parse(spec);
|
||||
const parsed = PageSpecSchema.parse(spec);
|
||||
return PageRangeSchema.parse(parsed);
|
||||
};
|
||||
|
||||
@@ -80,10 +79,13 @@ export const createPageRange = (spec: string | undefined): PageRange | undefined
|
||||
* @param spec - The date range specification (e.g., "2023-01-01:2023-12-31")
|
||||
* @param options - Options for date range creation
|
||||
*/
|
||||
export const createDateRange = (
|
||||
export const createTimestampRange = (
|
||||
spec: string | undefined,
|
||||
options: CreateDateRangeOptions = {},
|
||||
): DateRange | undefined => {
|
||||
options: {
|
||||
format?: string;
|
||||
separator?: string;
|
||||
} = {},
|
||||
): TimestampRange | undefined => {
|
||||
if (!spec) return undefined;
|
||||
const { format = DEFAULT_DATE_FORMAT, separator = ":" } = options;
|
||||
if (!separator) {
|
||||
@@ -91,7 +93,7 @@ export const createDateRange = (
|
||||
}
|
||||
|
||||
const normalized = spec.replace(separator, ":");
|
||||
const parsedSpec = DateRangeSpecSchema.parse(normalized);
|
||||
const parsedSpec = DateSpecSchema.parse(normalized);
|
||||
|
||||
const startDate = parseDate(parsedSpec.startRaw, format);
|
||||
const endDate = parseDate(parsedSpec.endRaw, format);
|
||||
@@ -101,7 +103,7 @@ export const createDateRange = (
|
||||
start: getUnixTime(startDate),
|
||||
};
|
||||
|
||||
return DateRangeSchema.parse(range);
|
||||
return TimestampRangeSchema.parse(range);
|
||||
};
|
||||
|
||||
/**
|
||||
@@ -109,9 +111,9 @@ export const createDateRange = (
|
||||
* @param range - The date range
|
||||
* @param fmt - The date format (default: DEFAULT_DATE_FORMAT)
|
||||
*/
|
||||
export const formatDateRange = (range: DateRange, fmt = DEFAULT_DATE_FORMAT): string => {
|
||||
const start = format(new Date(range.start * 1000), fmt);
|
||||
const end = format(new Date(range.end * 1000), fmt);
|
||||
export const formatTimestampRange = (range: TimestampRange, fmt = DEFAULT_DATE_FORMAT): string => {
|
||||
const start = format(fromUnixTime(range.start), fmt);
|
||||
const end = format(fromUnixTime(range.end), fmt);
|
||||
return `${start}:${end}`;
|
||||
};
|
||||
|
||||
@@ -128,7 +130,7 @@ export const formatPageRange = (range: PageRange): string => {
|
||||
* @param range - The date range
|
||||
* @param timestamp - The timestamp to check
|
||||
*/
|
||||
export const isTimestampInRange = (range: DateRange, timestamp: number): boolean => {
|
||||
export const isTimestampInRange = (range: TimestampRange, timestamp: number): boolean => {
|
||||
return range.start <= timestamp && timestamp <= range.end;
|
||||
};
|
||||
|
||||
@@ -145,17 +147,3 @@ export const createAbsoluteUrl = (base: string, href: string): string => {
|
||||
return href;
|
||||
}
|
||||
};
|
||||
|
||||
/**
|
||||
* extract the domain name from a URL.
|
||||
* @param url - The URL string
|
||||
* @returns The domain name or null if invalid URL
|
||||
*/
|
||||
export const extractDomainName = (url: string): string | null => {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return parsed.hostname;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
};
|
||||
|
||||
@@ -1,11 +1,4 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"#crawler/*": ["./src/*"]
|
||||
}
|
||||
},
|
||||
"extends": "@basango/tsconfig/base.json",
|
||||
"include": ["src"],
|
||||
"references": []
|
||||
"include": ["src"]
|
||||
}
|
||||
|
||||
@@ -16,7 +16,7 @@ const nextConfig = {
|
||||
},
|
||||
poweredByHeader: false,
|
||||
reactStrictMode: true,
|
||||
transpilePackages: ["@basango/ui", "@basango/api"],
|
||||
transpilePackages: ["@basango/ui", "@basango/api", "@basango/domain"],
|
||||
};
|
||||
|
||||
export default nextConfig;
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
{
|
||||
"dependencies": {
|
||||
"@basango/api": "workspace:*",
|
||||
"@basango/domain": "workspace:*",
|
||||
"@basango/ui": "workspace:*",
|
||||
"@date-fns/tz": "^1.4.1",
|
||||
"@hookform/resolvers": "^5.2.2",
|
||||
@@ -18,10 +19,12 @@
|
||||
"next-themes": "^0.4.6",
|
||||
"nuqs": "^2.7.3",
|
||||
"react": "catalog:",
|
||||
"react-day-picker": "^9.11.1",
|
||||
"react-dom": "catalog:",
|
||||
"react-hook-form": "^7.66.0",
|
||||
"recharts": "^3.4.1",
|
||||
"server-only": "^0.0.1",
|
||||
"sonner": "^2.0.7",
|
||||
"superjson": "^2.2.5",
|
||||
"zod": "^4.1.12",
|
||||
"zustand": "^5.0.8"
|
||||
@@ -34,12 +37,15 @@
|
||||
"@types/react-dom": "catalog:",
|
||||
"typescript": "catalog:"
|
||||
},
|
||||
"imports": {
|
||||
"#dashboard/*": "./src/*"
|
||||
},
|
||||
"name": "@basango/dashboard",
|
||||
"private": true,
|
||||
"scripts": {
|
||||
"build": "next build",
|
||||
"clean": "rm -rf .next node_modules",
|
||||
"dev": "next dev",
|
||||
"lint": "eslint",
|
||||
"start": "next start"
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,22 +1,21 @@
|
||||
import { Metadata } from "next";
|
||||
|
||||
import { ArticlesFeed } from "#dashboard/components/articles-feed";
|
||||
import { PageLayout } from "#dashboard/components/shell/page-layout";
|
||||
import { HydrateClient, batchPrefetch, trpc } from "#dashboard/trpc/server";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Articles | Basango Dashboard",
|
||||
};
|
||||
|
||||
export default function Page() {
|
||||
batchPrefetch([trpc.articles.list.infiniteQueryOptions({ limit: 12 })]);
|
||||
|
||||
return (
|
||||
<PageLayout leading="Manage your articles" title="Articles">
|
||||
<div className="flex flex-1 flex-col gap-4">
|
||||
<div className="grid auto-rows-min gap-4 md:grid-cols-3">
|
||||
<div className="bg-muted/50 aspect-video rounded-xl" />
|
||||
<div className="bg-muted/50 aspect-video rounded-xl" />
|
||||
<div className="bg-muted/50 aspect-video rounded-xl" />
|
||||
</div>
|
||||
<div className="bg-muted/50 min-h-screen flex-1 rounded-xl md:min-h-min" />
|
||||
</div>
|
||||
</PageLayout>
|
||||
<HydrateClient>
|
||||
<PageLayout leading="Track crawled content and trends" title="Articles">
|
||||
<ArticlesFeed />
|
||||
</PageLayout>
|
||||
</HydrateClient>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,30 @@
|
||||
import { Metadata } from "next";
|
||||
|
||||
import { PublicationGraphChart } from "#dashboard/components/charts/articles/publication-graph-chart";
|
||||
import { SourceDistributionChart } from "#dashboard/components/charts/articles/source-distribution-chart";
|
||||
import { PageLayout } from "#dashboard/components/shell/page-layout";
|
||||
import { HydrateClient, batchPrefetch, trpc } from "#dashboard/trpc/server";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Dashboard | Basango",
|
||||
};
|
||||
|
||||
export default async function Page() {
|
||||
batchPrefetch([
|
||||
trpc.articles.getPublicationGraph.queryOptions({}),
|
||||
trpc.articles.getSourceDistribution.queryOptions({ limit: 8 }),
|
||||
]);
|
||||
|
||||
return (
|
||||
<HydrateClient>
|
||||
<PageLayout leading="Keep track of article volume and source coverage" title="Dashboard">
|
||||
<div className="grid grid-cols-1 gap-4 lg:grid-cols-4">
|
||||
<div className="lg:col-span-3">
|
||||
<PublicationGraphChart />
|
||||
</div>
|
||||
<SourceDistributionChart />
|
||||
</div>
|
||||
</PageLayout>
|
||||
</HydrateClient>
|
||||
);
|
||||
}
|
||||
@@ -1,9 +1,12 @@
|
||||
import { Source } from "@basango/domain/models/sources";
|
||||
import { Tabs, TabsContent, TabsList, TabsTrigger } from "@basango/ui/components/tabs";
|
||||
import { Metadata } from "next";
|
||||
|
||||
import { SourceCategorySharesChart } from "#dashboard/components/charts/source-category-shares-chart";
|
||||
import { SourcePublicationgGraphChart } from "#dashboard/components/charts/source-publication-graph-chart";
|
||||
import { ArticlesFeed } from "#dashboard/components/articles-feed";
|
||||
import { CategorySharesChart } from "#dashboard/components/charts/sources/category-shares-chart";
|
||||
import { PublicationGraphChart } from "#dashboard/components/charts/sources/publication-graph-chart";
|
||||
import { PageLayout } from "#dashboard/components/shell/page-layout";
|
||||
import { SourceDetailsTab } from "#dashboard/components/source-details-tab";
|
||||
import { HydrateClient, batchPrefetch, getQueryClient, trpc } from "#dashboard/trpc/server";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
@@ -16,11 +19,12 @@ export default async function Page({ params }: { params: Promise<{ id: string }>
|
||||
|
||||
batchPrefetch([
|
||||
trpc.sources.getById.queryOptions({ id }),
|
||||
trpc.sources.getCategoryShares.queryOptions({ id }),
|
||||
trpc.sources.getCategoryShares.queryOptions({ id, limit: 10 }),
|
||||
trpc.sources.getPublicationGraph.queryOptions({ id }),
|
||||
trpc.articles.list.infiniteQueryOptions({ limit: 12, sourceId: id }),
|
||||
]);
|
||||
|
||||
const source = await queryClient.fetchQuery(trpc.sources.getById.queryOptions({ id }));
|
||||
const source: Source = await queryClient.fetchQuery(trpc.sources.getById.queryOptions({ id }));
|
||||
|
||||
return (
|
||||
<HydrateClient>
|
||||
@@ -29,20 +33,17 @@ export default async function Page({ params }: { params: Promise<{ id: string }>
|
||||
<TabsList>
|
||||
<TabsTrigger value="overview">Overview</TabsTrigger>
|
||||
<TabsTrigger value="articles">Articles</TabsTrigger>
|
||||
<TabsTrigger value="details">Details</TabsTrigger>
|
||||
</TabsList>
|
||||
<TabsContent className="space-y-4" value="overview">
|
||||
<SourceCategorySharesChart sourceId={source.id} />
|
||||
<SourcePublicationgGraphChart sourceId={source.id} />
|
||||
<CategorySharesChart sourceId={source.id} />
|
||||
<PublicationGraphChart sourceId={source.id} />
|
||||
</TabsContent>
|
||||
<TabsContent value="articles">
|
||||
<div className="flex flex-1 flex-col gap-4">
|
||||
<div className="grid auto-rows-min gap-4 md:grid-cols-3">
|
||||
<div className="bg-muted/50 aspect-video rounded-xl" />
|
||||
<div className="bg-muted/50 aspect-video rounded-xl" />
|
||||
<div className="bg-muted/50 aspect-video rounded-xl" />
|
||||
</div>
|
||||
<div className="bg-muted/50 min-h-screen flex-1 rounded-xl md:min-h-min" />
|
||||
</div>
|
||||
<ArticlesFeed sourceId={source.id} />
|
||||
</TabsContent>
|
||||
<TabsContent value="details">
|
||||
<SourceDetailsTab source={source} />
|
||||
</TabsContent>
|
||||
</Tabs>
|
||||
</PageLayout>
|
||||
|
||||
@@ -1,33 +1,44 @@
|
||||
import { RouterOutputs } from "@basango/api/trpc/routers/_app";
|
||||
import { Source } from "@basango/domain/models/sources";
|
||||
import { Button } from "@basango/ui/components/button";
|
||||
import { PlusIcon } from "lucide-react";
|
||||
import { Metadata } from "next";
|
||||
import Link from "next/link";
|
||||
|
||||
import { SourceCreateModal } from "#dashboard/components/modals/source-create-modal";
|
||||
import { PageLayout } from "#dashboard/components/shell/page-layout";
|
||||
import { SourceCard } from "#dashboard/components/widgets/source-card";
|
||||
import { SourceCard } from "#dashboard/components/source-card";
|
||||
import { HydrateClient, getQueryClient, prefetch, trpc } from "#dashboard/trpc/server";
|
||||
|
||||
export const metadata: Metadata = {
|
||||
title: "Sources | Basango Dashboard",
|
||||
};
|
||||
|
||||
type SourceDetails = RouterOutputs["sources"]["get"][number];
|
||||
|
||||
export default async function Page() {
|
||||
const queryClient = getQueryClient();
|
||||
|
||||
prefetch(trpc.sources.get.queryOptions());
|
||||
const sources = await queryClient.fetchQuery(trpc.sources.get.queryOptions());
|
||||
const sources: Source[] = await queryClient.fetchQuery(trpc.sources.get.queryOptions());
|
||||
|
||||
return (
|
||||
<HydrateClient>
|
||||
<PageLayout leading="Manage your news sources" title="Sources">
|
||||
<div className="mb-6 flex justify-end">
|
||||
<Link href="?createSource=true">
|
||||
<Button type="button">
|
||||
<PlusIcon className="mr-2 size-4" />
|
||||
Add source
|
||||
</Button>
|
||||
</Link>
|
||||
</div>
|
||||
<div className="grid grid-cols-1 gap-4 sm:grid-cols-3">
|
||||
{sources.map((source: SourceDetails) => (
|
||||
{sources.map((source: Source) => (
|
||||
<Link href={`/sources/${source.id}`} key={source.id}>
|
||||
<SourceCard source={source} />
|
||||
</Link>
|
||||
))}
|
||||
</div>
|
||||
|
||||
<SourceCreateModal />
|
||||
</PageLayout>
|
||||
</HydrateClient>
|
||||
);
|
||||
|
||||
@@ -0,0 +1,138 @@
|
||||
"use client";
|
||||
|
||||
import type { RouterOutputs } from "@basango/api/trpc/routers/_app";
|
||||
import { Badge } from "@basango/ui/components/badge";
|
||||
import { Button } from "@basango/ui/components/button";
|
||||
import { Card, CardContent, CardFooter, CardHeader, CardTitle } from "@basango/ui/components/card";
|
||||
import {
|
||||
DropdownMenu,
|
||||
DropdownMenuContent,
|
||||
DropdownMenuItem,
|
||||
DropdownMenuTrigger,
|
||||
} from "@basango/ui/components/dropdown-menu";
|
||||
import { Skeleton } from "@basango/ui/components/skeleton";
|
||||
import { ExternalLink, Link2, MoreHorizontal } from "lucide-react";
|
||||
import Link from "next/link";
|
||||
import * as React from "react";
|
||||
|
||||
import { formatDate, formatRelativeTime } from "#dashboard/utils/utils";
|
||||
|
||||
type Article = RouterOutputs["articles"]["list"]["items"][number];
|
||||
|
||||
type ArticleCardProps = {
|
||||
article: Article;
|
||||
};
|
||||
|
||||
function getDescription(article: Article) {
|
||||
return (
|
||||
article.metadata?.description ??
|
||||
article.excerpt ??
|
||||
"No description was provided for this article."
|
||||
);
|
||||
}
|
||||
|
||||
export function ArticleCard({ article }: ArticleCardProps) {
|
||||
const [copied, setCopied] = React.useState(false);
|
||||
const description = getDescription(article);
|
||||
const imageUrl = article.image ?? undefined;
|
||||
|
||||
const copyLink = React.useCallback(async () => {
|
||||
try {
|
||||
await navigator.clipboard.writeText(article.link);
|
||||
setCopied(true);
|
||||
setTimeout(() => setCopied(false), 1500);
|
||||
} catch {
|
||||
setCopied(false);
|
||||
}
|
||||
}, [article.link]);
|
||||
|
||||
return (
|
||||
<Card className="flex h-full flex-col overflow-hidden border border-border/80 p-0">
|
||||
<CardHeader className="relative h-40 overflow-hidden p-0">
|
||||
<div className="relative h-full w-full bg-muted">
|
||||
{imageUrl ? (
|
||||
<img
|
||||
alt={article.title}
|
||||
className="h-full w-full object-cover transition duration-200 hover:scale-105"
|
||||
loading="lazy"
|
||||
src={imageUrl}
|
||||
/>
|
||||
) : (
|
||||
<div className="flex h-full w-full items-center justify-center text-xs text-muted-foreground">
|
||||
No image available
|
||||
</div>
|
||||
)}
|
||||
<div className="absolute left-3 top-3">
|
||||
<Badge variant="secondary">{article.sourceName}</Badge>
|
||||
</div>
|
||||
<div className="absolute right-3 top-3">
|
||||
<DropdownMenu>
|
||||
<DropdownMenuTrigger asChild>
|
||||
<Button
|
||||
className="size-8 rounded-full bg-background/80 backdrop-blur"
|
||||
size="icon"
|
||||
variant="ghost"
|
||||
>
|
||||
<MoreHorizontal className="h-4 w-4" />
|
||||
</Button>
|
||||
</DropdownMenuTrigger>
|
||||
<DropdownMenuContent align="end">
|
||||
<DropdownMenuItem asChild>
|
||||
<Link href={article.link} rel="noreferrer" target="_blank">
|
||||
<ExternalLink className="mr-2 h-4 w-4" />
|
||||
Open original
|
||||
</Link>
|
||||
</DropdownMenuItem>
|
||||
<DropdownMenuItem onClick={copyLink}>
|
||||
<Link2 className="mr-2 h-4 w-4" />
|
||||
{copied ? "Copied!" : "Copy link"}
|
||||
</DropdownMenuItem>
|
||||
</DropdownMenuContent>
|
||||
</DropdownMenu>
|
||||
</div>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="flex flex-1 flex-col gap-3 p-4">
|
||||
<CardTitle className="text-base leading-tight">
|
||||
<Link
|
||||
className="transition hover:text-primary hover:underline"
|
||||
href={article.link}
|
||||
rel="noreferrer"
|
||||
target="_blank"
|
||||
>
|
||||
{article.title}
|
||||
</Link>
|
||||
</CardTitle>
|
||||
<p className="text-sm text-muted-foreground line-clamp-3">{description}</p>
|
||||
</CardContent>
|
||||
<CardFooter className="flex items-center justify-between gap-2 px-4 py-3 text-xs text-muted-foreground">
|
||||
<div className="flex flex-col">
|
||||
<span className="font-medium text-foreground">
|
||||
{formatDate(article.publishedAt.toISOString(), "PP", false)}
|
||||
</span>
|
||||
<span>{formatRelativeTime(new Date(article.publishedAt))}</span>
|
||||
</div>
|
||||
<span>{article.readingTime} min</span>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
|
||||
export function ArticleCardSkeleton() {
|
||||
return (
|
||||
<Card className="flex h-full flex-col overflow-hidden p-0">
|
||||
<div className="h-60 w-full bg-muted">
|
||||
<Skeleton className="h-full w-full" />
|
||||
</div>
|
||||
<CardContent className="flex flex-1 flex-col gap-3 p-4">
|
||||
<Skeleton className="h-4 w-3/4" />
|
||||
<Skeleton className="h-4 w-full" />
|
||||
<Skeleton className="h-4 w-5/6" />
|
||||
</CardContent>
|
||||
<CardFooter className="flex items-center justify-between px-4 py-3">
|
||||
<Skeleton className="h-4 w-1/3" />
|
||||
<Skeleton className="h-8 w-16" />
|
||||
</CardFooter>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,94 @@
|
||||
"use client";
|
||||
|
||||
import { Alert, AlertDescription, AlertTitle } from "@basango/ui/components/alert";
|
||||
import { Button } from "@basango/ui/components/button";
|
||||
import { useInfiniteQuery } from "@tanstack/react-query";
|
||||
import { Loader2 } from "lucide-react";
|
||||
import * as React from "react";
|
||||
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
|
||||
import { ArticleCard, ArticleCardSkeleton } from "./article-card";
|
||||
|
||||
type ArticlesTableProps = {
|
||||
sourceId?: string;
|
||||
};
|
||||
|
||||
const PLACEHOLDER_COUNT = 8;
|
||||
|
||||
export function ArticlesFeed({ sourceId }: ArticlesTableProps) {
|
||||
const trpc = useTRPC();
|
||||
|
||||
const query = useInfiniteQuery(
|
||||
trpc.articles.list.infiniteQueryOptions(
|
||||
{
|
||||
limit: 12,
|
||||
sourceId,
|
||||
},
|
||||
{
|
||||
getNextPageParam: (lastPage) => (lastPage.meta.hasNext ? lastPage.meta.nextCursor : null),
|
||||
initialCursor: null,
|
||||
},
|
||||
),
|
||||
);
|
||||
|
||||
const articles = React.useMemo(
|
||||
() => query.data?.pages.flatMap((page) => page.items) ?? [],
|
||||
[query.data],
|
||||
);
|
||||
|
||||
const isInitialLoading = query.isLoading && !query.data;
|
||||
|
||||
return (
|
||||
<div className="space-y-4">
|
||||
{query.isError && (
|
||||
<Alert variant="destructive">
|
||||
<AlertTitle>Unable to load articles</AlertTitle>
|
||||
<AlertDescription>
|
||||
{query.error.message ?? "An unexpected error occurred while fetching articles."}
|
||||
</AlertDescription>
|
||||
</Alert>
|
||||
)}
|
||||
|
||||
{isInitialLoading ? (
|
||||
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-4">
|
||||
{Array.from({ length: PLACEHOLDER_COUNT }).map((_, index) => (
|
||||
<ArticleCardSkeleton key={index} />
|
||||
))}
|
||||
</div>
|
||||
) : articles.length > 0 ? (
|
||||
<div className="grid gap-4 md:grid-cols-2 xl:grid-cols-4">
|
||||
{articles.map((article) => (
|
||||
<ArticleCard article={article} key={article.id} />
|
||||
))}
|
||||
</div>
|
||||
) : (
|
||||
<div className="rounded-lg border border-dashed px-6 py-12 text-center text-sm text-muted-foreground">
|
||||
No articles match your filters yet.
|
||||
</div>
|
||||
)}
|
||||
|
||||
<div className="flex items-center justify-center">
|
||||
{query.hasNextPage ? (
|
||||
<Button
|
||||
disabled={query.isFetchingNextPage}
|
||||
onClick={() => query.fetchNextPage()}
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
{query.isFetchingNextPage ? (
|
||||
<>
|
||||
<Loader2 className="mr-2 h-4 w-4 animate-spin" />
|
||||
Loading…
|
||||
</>
|
||||
) : (
|
||||
"Load more"
|
||||
)}
|
||||
</Button>
|
||||
) : articles.length > 0 ? (
|
||||
<p className="text-xs text-muted-foreground">You're all caught up.</p>
|
||||
) : null}
|
||||
</div>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,50 @@
|
||||
"use client";
|
||||
|
||||
import { ChartTooltip, ChartTooltipContent } from "@basango/ui/components/chart";
|
||||
import { Area, AreaChart as BaseAreachart, CartesianGrid, XAxis, YAxis } from "recharts";
|
||||
|
||||
import { formatDate, formatNumber } from "#dashboard/utils/utils";
|
||||
|
||||
type AreaChartProps = {
|
||||
data: unknown;
|
||||
};
|
||||
|
||||
export function AreaChart({ data }: AreaChartProps) {
|
||||
return (
|
||||
<BaseAreachart accessibilityLayer data={data}>
|
||||
<CartesianGrid strokeDasharray="3 3" vertical={false} />
|
||||
<XAxis
|
||||
axisLine={false}
|
||||
dataKey="date"
|
||||
minTickGap={32}
|
||||
tickFormatter={(value) => formatDate(String(value))}
|
||||
tickLine={false}
|
||||
tickMargin={8}
|
||||
/>
|
||||
<YAxis
|
||||
allowDecimals={false}
|
||||
axisLine={false}
|
||||
tickFormatter={(value) => formatNumber(Number(value))}
|
||||
tickLine={false}
|
||||
width={48}
|
||||
/>
|
||||
<ChartTooltip
|
||||
content={
|
||||
<ChartTooltipContent
|
||||
labelFormatter={(value) => formatDate(String(value), "PP")}
|
||||
nameKey="count"
|
||||
/>
|
||||
}
|
||||
cursor={{ stroke: "var(--border)", strokeDasharray: "4 4" }}
|
||||
/>
|
||||
<Area
|
||||
dataKey="count"
|
||||
fill="var(--color-count)"
|
||||
fillOpacity={0.15}
|
||||
stroke="var(--color-count)"
|
||||
strokeWidth={2}
|
||||
type="monotone"
|
||||
/>
|
||||
</BaseAreachart>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,81 @@
|
||||
// @ts-nocheck
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardFooter,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import { ChartConfig, ChartContainer } from "@basango/ui/components/chart";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
import { AreaChart } from "#dashboard/components/charts/area-chart";
|
||||
import {
|
||||
ChartPeriodPicker,
|
||||
useChartPeriodFilter,
|
||||
} from "#dashboard/components/charts/chart-filters";
|
||||
import { Status } from "#dashboard/components/charts/status";
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
import { formatNumber } from "#dashboard/utils/utils";
|
||||
|
||||
const chartConfig = {
|
||||
count: {
|
||||
color: "var(--chart-1)",
|
||||
label: "Articles",
|
||||
},
|
||||
} satisfies ChartConfig;
|
||||
|
||||
export function PublicationGraphChart() {
|
||||
const trpc = useTRPC();
|
||||
const period = useChartPeriodFilter();
|
||||
|
||||
const { data } = useQuery(
|
||||
trpc.articles.getPublicationGraph.queryOptions({
|
||||
range: period.range,
|
||||
}),
|
||||
);
|
||||
|
||||
return (
|
||||
<Card className="pt-0">
|
||||
<CardHeader className="flex items-start gap-2 space-y-0 border-b py-5 sm:flex-row sm:items-center">
|
||||
<div className="grid flex-1 gap-1">
|
||||
<CardTitle>{formatNumber(data?.meta?.current)} articles</CardTitle>
|
||||
<CardDescription>
|
||||
<div className="flex items-center justify-start gap-1 text-xs">
|
||||
<Status value={data?.delta} />
|
||||
<span className="text-muted-foreground">vs previous</span>
|
||||
</div>
|
||||
</CardDescription>
|
||||
</div>
|
||||
<div className="flex items-center gap-3">
|
||||
<ChartPeriodPicker defaultDays={period.defaultDays} paramKey="articlesPeriod" />
|
||||
</div>
|
||||
</CardHeader>
|
||||
|
||||
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
|
||||
<ChartContainer className="aspect-auto h-[250px] w-full" config={chartConfig}>
|
||||
<AreaChart data={data?.items} />
|
||||
</ChartContainer>
|
||||
</CardContent>
|
||||
|
||||
<CardFooter>
|
||||
<CardDescription>
|
||||
Showing total crawled articles for the selected period,
|
||||
<div className="flex flex-wrap items-center gap-2 text-xs sm:text-sm">
|
||||
<span className="font-semibold text-foreground">
|
||||
{formatNumber(data?.meta?.current)} vs {formatNumber(data?.meta?.previous)} articles
|
||||
</span>
|
||||
<Status icons={false} percentage={true} value={data?.deltaPercentage} />
|
||||
<span className="text-muted-foreground">period</span>
|
||||
{data?.meta?.previous === 0 && data?.meta?.current === 0 && (
|
||||
<span className="text-muted-foreground">(no articles yet)</span>
|
||||
)}
|
||||
</div>
|
||||
</CardDescription>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,82 @@
|
||||
// @ts-nocheck
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import {
|
||||
ChartConfig,
|
||||
ChartContainer,
|
||||
ChartTooltip,
|
||||
ChartTooltipContent,
|
||||
} from "@basango/ui/components/chart";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { Cell, Pie, PieChart } from "recharts";
|
||||
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
import { getColorFromName } from "#dashboard/utils/categories";
|
||||
import { formatNumber } from "#dashboard/utils/utils";
|
||||
|
||||
const chartConfig = {} satisfies ChartConfig;
|
||||
|
||||
export function SourceDistributionChart() {
|
||||
const trpc = useTRPC();
|
||||
|
||||
const { data } = useQuery(
|
||||
trpc.articles.getSourceDistribution.queryOptions({
|
||||
limit: 10,
|
||||
}),
|
||||
);
|
||||
|
||||
return (
|
||||
<Card className="pt-0">
|
||||
<CardHeader className="flex items-start gap-2 space-y-0 border-b py-5 sm:flex-row sm:items-center">
|
||||
<div className="grid flex-1 gap-1">
|
||||
<CardTitle>Source distribution</CardTitle>
|
||||
<CardDescription>Share of articles by source</CardDescription>
|
||||
</div>
|
||||
</CardHeader>
|
||||
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
|
||||
<ChartContainer className="mx-auto aspect-square max-h-80 w-full" config={chartConfig}>
|
||||
<PieChart>
|
||||
<ChartTooltip content={<ChartTooltipContent nameKey="name" />} />
|
||||
<Pie
|
||||
data={data?.items}
|
||||
dataKey="count"
|
||||
innerRadius={70}
|
||||
nameKey="name"
|
||||
outerRadius={110}
|
||||
paddingAngle={2}
|
||||
strokeWidth={0}
|
||||
>
|
||||
{data?.items.map((item) => (
|
||||
<Cell fill={getColorFromName(item.name)} key={item.id} />
|
||||
))}
|
||||
</Pie>
|
||||
</PieChart>
|
||||
</ChartContainer>
|
||||
|
||||
<ul className="mt-4 space-y-2">
|
||||
{data?.items.map((item) => (
|
||||
<li className="flex items-center justify-between text-sm" key={item.id}>
|
||||
<span className="flex items-center gap-2">
|
||||
<span
|
||||
className="h-2.5 w-2.5 rounded-full"
|
||||
style={{ backgroundColor: getColorFromName(item.name) }}
|
||||
/>
|
||||
<span className="font-medium leading-none">{item.name}</span>
|
||||
</span>
|
||||
<span className="text-muted-foreground">
|
||||
{formatNumber(item.count)} ({item.percentage}%)
|
||||
</span>
|
||||
</li>
|
||||
))}
|
||||
</ul>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
"use client";
|
||||
|
||||
import { ChartTooltip, ChartTooltipContent } from "@basango/ui/components/chart";
|
||||
import { Bar, BarChart as BaseBarChart, CartesianGrid, XAxis } from "recharts";
|
||||
|
||||
import { formatDate } from "#dashboard/utils/utils";
|
||||
|
||||
type BarChartProps = {
|
||||
data: unknown;
|
||||
};
|
||||
|
||||
export function BarChart({ data }: BarChartProps) {
|
||||
return (
|
||||
<BaseBarChart accessibilityLayer data={data}>
|
||||
<CartesianGrid vertical={false} />
|
||||
<XAxis
|
||||
axisLine={false}
|
||||
dataKey="date"
|
||||
minTickGap={32}
|
||||
tickFormatter={(value) => formatDate(value)}
|
||||
tickLine={false}
|
||||
tickMargin={8}
|
||||
/>
|
||||
<ChartTooltip content={<ChartTooltipContent indicator="dashed" />} cursor={false} />
|
||||
<Bar dataKey="count" fill="var(--color-count)" radius={4} />
|
||||
</BaseBarChart>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,259 @@
|
||||
"use client";
|
||||
|
||||
import { Button } from "@basango/ui/components/button";
|
||||
import { Calendar } from "@basango/ui/components/calendar";
|
||||
import { Popover, PopoverContent, PopoverTrigger } from "@basango/ui/components/popover";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@basango/ui/components/select";
|
||||
import { ToggleGroup, ToggleGroupItem } from "@basango/ui/components/toggle-group";
|
||||
import { differenceInCalendarDays, format, subDays } from "date-fns";
|
||||
import { CalendarIcon, ChevronDown } from "lucide-react";
|
||||
import { parseAsInteger, parseAsIsoDate, useQueryStates } from "nuqs";
|
||||
import { useMemo, useState } from "react";
|
||||
import { DateRange } from "react-day-picker";
|
||||
|
||||
const DEFAULT_PERIOD_OPTIONS = [
|
||||
{ label: "Last 7 days", value: 7 },
|
||||
{ label: "Last 30 days", value: 30 },
|
||||
{ label: "Last 3 months", value: 90 },
|
||||
{ label: "Last 6 months", value: 180 },
|
||||
{ label: "Last 12 months", value: 365 },
|
||||
] as const;
|
||||
|
||||
type DateInput = number | Date | null | undefined;
|
||||
|
||||
const createRangeFromDays = (days: number): DateRange => {
|
||||
const end = new Date();
|
||||
|
||||
return {
|
||||
from: subDays(end, Math.max(days - 1, 0)),
|
||||
to: end,
|
||||
};
|
||||
};
|
||||
|
||||
const DEFAULT_LIMIT_OPTIONS = [
|
||||
{ label: "Top 10", value: 10 },
|
||||
{ label: "Top 20", value: 20 },
|
||||
{ label: "Top 50", value: 50 },
|
||||
] as const;
|
||||
|
||||
type ChartPeriodFilterOptions = {
|
||||
defaultDays?: number;
|
||||
paramKey?: string;
|
||||
};
|
||||
|
||||
type ChartLimitFilterOptions = {
|
||||
defaultValue?: number;
|
||||
paramKey?: string;
|
||||
};
|
||||
|
||||
export function useChartPeriodFilter(options: ChartPeriodFilterOptions = {}) {
|
||||
const { defaultDays = 30, paramKey = "chartPeriod" } = options;
|
||||
const fromKey = `${paramKey}From`;
|
||||
const toKey = `${paramKey}To`;
|
||||
|
||||
const defaultRange = useMemo(() => createRangeFromDays(defaultDays), [defaultDays]);
|
||||
|
||||
const [state, setState] = useQueryStates({
|
||||
[fromKey]: parseAsIsoDate,
|
||||
[toKey]: parseAsIsoDate,
|
||||
});
|
||||
|
||||
const from = state[fromKey] ?? undefined;
|
||||
const to = state[toKey] ?? undefined;
|
||||
|
||||
const selectedRange = useMemo(() => {
|
||||
if (from || to) {
|
||||
return { from, to };
|
||||
}
|
||||
|
||||
return undefined;
|
||||
}, [from, to]);
|
||||
|
||||
const range = useMemo(() => {
|
||||
if (from && to) {
|
||||
return { from, to };
|
||||
}
|
||||
|
||||
return defaultRange;
|
||||
}, [defaultRange, from, to]);
|
||||
|
||||
return {
|
||||
defaultDays,
|
||||
keys: { fromKey, toKey },
|
||||
range,
|
||||
selectedRange,
|
||||
setState,
|
||||
};
|
||||
}
|
||||
|
||||
export function useChartLimitFilter(options: ChartLimitFilterOptions = {}) {
|
||||
const { defaultValue = 10, paramKey = "chartLimit" } = options;
|
||||
const [state, setState] = useQueryStates({
|
||||
[paramKey]: parseAsInteger.withDefault(defaultValue),
|
||||
});
|
||||
|
||||
const limit = state[paramKey];
|
||||
|
||||
return {
|
||||
limit,
|
||||
setLimit: (value: number) => {
|
||||
setState({ [paramKey]: value });
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
type ChartPeriodPickerProps = ChartPeriodFilterOptions & {
|
||||
options?: ReadonlyArray<{ label: string; value: number }>;
|
||||
};
|
||||
|
||||
export function ChartPeriodPicker({
|
||||
defaultDays = 30,
|
||||
options = DEFAULT_PERIOD_OPTIONS,
|
||||
paramKey = "chartPeriod",
|
||||
disabled,
|
||||
}: ChartPeriodPickerProps & { disabled?: boolean }) {
|
||||
const { range, selectedRange, keys, setState } = useChartPeriodFilter({ defaultDays, paramKey });
|
||||
const [open, setOpen] = useState(false);
|
||||
|
||||
const selectValue = useMemo(() => {
|
||||
if (!range?.from || !range?.to) {
|
||||
return "custom";
|
||||
}
|
||||
|
||||
const diff = differenceInCalendarDays(range.to, range.from) + 1;
|
||||
const match = options.find((option) => option.value === diff);
|
||||
|
||||
return match ? String(match.value) : "custom";
|
||||
}, [options, range]);
|
||||
|
||||
const handlePresetChange = (value: string) => {
|
||||
if (value === "custom") {
|
||||
return;
|
||||
}
|
||||
|
||||
const presetRange = createRangeFromDays(Number(value));
|
||||
|
||||
setState({
|
||||
[keys.fromKey]: presetRange.from ?? null,
|
||||
[keys.toKey]: presetRange.to ?? null,
|
||||
});
|
||||
};
|
||||
|
||||
const handleCalendarSelect = (value: DateRange | undefined) => {
|
||||
if (value?.from && value?.to) {
|
||||
setState({
|
||||
[keys.fromKey]: value.from,
|
||||
[keys.toKey]: value.to,
|
||||
});
|
||||
} else {
|
||||
setState({
|
||||
[keys.fromKey]: null,
|
||||
[keys.toKey]: null,
|
||||
});
|
||||
}
|
||||
};
|
||||
|
||||
const displayLabel =
|
||||
formatDateRange(range) ??
|
||||
options.find((option) => String(option.value) === selectValue)?.label ??
|
||||
"Select range";
|
||||
|
||||
return (
|
||||
<Popover onOpenChange={setOpen} open={open}>
|
||||
<PopoverTrigger asChild disabled={disabled}>
|
||||
<Button
|
||||
className="w-full justify-start gap-2 text-left font-medium sm:w-72"
|
||||
type="button"
|
||||
variant="outline"
|
||||
>
|
||||
<CalendarIcon className="h-4 w-4 text-muted-foreground" />
|
||||
<span className="flex-1 truncate">{displayLabel}</span>
|
||||
<ChevronDown className="h-4 w-4 text-muted-foreground" />
|
||||
</Button>
|
||||
</PopoverTrigger>
|
||||
<PopoverContent align="end" className="w-screen space-y-4 p-4 sm:w-[520px]" sideOffset={8}>
|
||||
<Select onValueChange={handlePresetChange} value={selectValue}>
|
||||
<SelectTrigger>
|
||||
<SelectValue placeholder="Quick range" />
|
||||
</SelectTrigger>
|
||||
<SelectContent>
|
||||
{options.map((option) => (
|
||||
<SelectItem key={option.value} value={String(option.value)}>
|
||||
{option.label}
|
||||
</SelectItem>
|
||||
))}
|
||||
<SelectItem value="custom">Custom range</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
|
||||
<Calendar
|
||||
mode="range"
|
||||
numberOfMonths={2}
|
||||
onSelect={handleCalendarSelect}
|
||||
selected={(selectedRange ?? range) as DateRange | undefined}
|
||||
/>
|
||||
|
||||
<div className="flex justify-end gap-2">
|
||||
<Button
|
||||
onClick={() =>
|
||||
setState({
|
||||
[keys.fromKey]: null,
|
||||
[keys.toKey]: null,
|
||||
})
|
||||
}
|
||||
type="button"
|
||||
variant="ghost"
|
||||
>
|
||||
Reset
|
||||
</Button>
|
||||
<Button onClick={() => setOpen(false)} type="button">
|
||||
Done
|
||||
</Button>
|
||||
</div>
|
||||
</PopoverContent>
|
||||
</Popover>
|
||||
);
|
||||
}
|
||||
|
||||
type ChartLimitToggleProps = ChartLimitFilterOptions & {
|
||||
options?: ReadonlyArray<{ label: string; value: number }>;
|
||||
};
|
||||
|
||||
export function ChartLimitToggle({
|
||||
defaultValue = 10,
|
||||
options = DEFAULT_LIMIT_OPTIONS,
|
||||
paramKey = "chartLimit",
|
||||
}: ChartLimitToggleProps) {
|
||||
const { limit, setLimit } = useChartLimitFilter({ defaultValue, paramKey });
|
||||
|
||||
return (
|
||||
<ToggleGroup
|
||||
onValueChange={(value) => {
|
||||
if (value) {
|
||||
setLimit(Number(value));
|
||||
}
|
||||
}}
|
||||
type="single"
|
||||
value={String(limit)}
|
||||
variant="outline"
|
||||
>
|
||||
{options.map((option) => (
|
||||
<ToggleGroupItem key={option.value} value={String(option.value)}>
|
||||
{option.label}
|
||||
</ToggleGroupItem>
|
||||
))}
|
||||
</ToggleGroup>
|
||||
);
|
||||
}
|
||||
|
||||
function formatDateRange(range?: { from?: DateInput; to?: DateInput }) {
|
||||
if (!range?.from || !range?.to) return null;
|
||||
|
||||
return `${format(range.from, "MMM d, yyyy")} - ${format(range.to, "MMM d, yyyy")}`;
|
||||
}
|
||||
@@ -1,109 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import {
|
||||
ChartConfig,
|
||||
ChartContainer,
|
||||
ChartTooltip,
|
||||
ChartTooltipContent,
|
||||
} from "@basango/ui/components/chart";
|
||||
import {
|
||||
Select,
|
||||
SelectContent,
|
||||
SelectItem,
|
||||
SelectTrigger,
|
||||
SelectValue,
|
||||
} from "@basango/ui/components/select";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import * as React from "react";
|
||||
import { Bar, BarChart, CartesianGrid, XAxis } from "recharts";
|
||||
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
import { formatDate } from "#dashboard/utils/utils";
|
||||
|
||||
const chartConfig = {
|
||||
count: {
|
||||
color: "var(--chart-2)",
|
||||
label: "Articles",
|
||||
},
|
||||
views: {
|
||||
label: "Articles",
|
||||
},
|
||||
} satisfies ChartConfig;
|
||||
|
||||
type Props = {
|
||||
sourceId: string;
|
||||
};
|
||||
|
||||
export function SourcePublicationgGraphChart({ sourceId }: Props) {
|
||||
const trpc = useTRPC();
|
||||
const [timeRange, setTimeRange] = React.useState("30");
|
||||
|
||||
const { data } = useQuery(
|
||||
trpc.sources.getPublicationGraph.queryOptions({
|
||||
days: Number(timeRange),
|
||||
id: sourceId,
|
||||
}),
|
||||
);
|
||||
|
||||
return (
|
||||
<Card className="pt-0">
|
||||
<CardHeader className="flex items-center gap-2 space-y-0 border-b py-5 sm:flex-row">
|
||||
<div className="grid flex-1 gap-1">
|
||||
<CardTitle>Publication Graph</CardTitle>
|
||||
<CardDescription>
|
||||
Showing total crawled articles for the last {timeRange} days
|
||||
</CardDescription>
|
||||
</div>
|
||||
<Select onValueChange={setTimeRange} value={timeRange}>
|
||||
<SelectTrigger
|
||||
aria-label="Select a value"
|
||||
className="hidden w-40 rounded-lg sm:ml-auto sm:flex"
|
||||
>
|
||||
<SelectValue placeholder="Last 3 months" />
|
||||
</SelectTrigger>
|
||||
<SelectContent className="rounded-xl">
|
||||
<SelectItem className="rounded-lg" value="7">
|
||||
Last 7 days
|
||||
</SelectItem>
|
||||
<SelectItem className="rounded-lg" value="30">
|
||||
Last 30 days
|
||||
</SelectItem>
|
||||
<SelectItem className="rounded-lg" value="90">
|
||||
Last 3 months
|
||||
</SelectItem>
|
||||
<SelectItem className="rounded-lg" value="180">
|
||||
Last 6 months
|
||||
</SelectItem>
|
||||
<SelectItem className="rounded-lg" value="365">
|
||||
Last 12 months
|
||||
</SelectItem>
|
||||
</SelectContent>
|
||||
</Select>
|
||||
</CardHeader>
|
||||
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
|
||||
<ChartContainer className="aspect-auto h-[250px] w-full" config={chartConfig}>
|
||||
<BarChart accessibilityLayer data={data?.items}>
|
||||
<CartesianGrid vertical={false} />
|
||||
<XAxis
|
||||
axisLine={false}
|
||||
dataKey="date"
|
||||
minTickGap={32}
|
||||
tickFormatter={(value) => formatDate(value)}
|
||||
tickLine={false}
|
||||
tickMargin={8}
|
||||
/>
|
||||
<ChartTooltip content={<ChartTooltipContent indicator="dashed" />} cursor={false} />
|
||||
<Bar dataKey="count" fill="var(--color-count)" radius={4} />
|
||||
</BarChart>
|
||||
</ChartContainer>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
+11
-27
@@ -1,3 +1,4 @@
|
||||
// @ts-nocheck
|
||||
"use client";
|
||||
|
||||
import {
|
||||
@@ -7,11 +8,10 @@ import {
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import { ToggleGroup, ToggleGroupItem } from "@basango/ui/components/toggle-group";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
import { useState } from "react";
|
||||
import { Bar, BarChart, Legend, ResponsiveContainer, XAxis, YAxis } from "recharts";
|
||||
|
||||
import { ChartLimitToggle, useChartLimitFilter } from "#dashboard/components/charts/chart-filters";
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
import { getColorFromName } from "#dashboard/utils/categories";
|
||||
|
||||
@@ -19,30 +19,24 @@ type Props = {
|
||||
sourceId: string;
|
||||
};
|
||||
|
||||
export function SourceCategorySharesChart({ sourceId }: Props) {
|
||||
export function CategorySharesChart({ sourceId }: Props) {
|
||||
const trpc = useTRPC();
|
||||
const [limit, setLimit] = useState(10);
|
||||
const { limit } = useChartLimitFilter();
|
||||
|
||||
const { data } = useQuery(
|
||||
trpc.sources.getCategoryShares.queryOptions({
|
||||
id: sourceId,
|
||||
limit: limit,
|
||||
limit,
|
||||
}),
|
||||
);
|
||||
const items = data?.items ?? [];
|
||||
|
||||
const chartData = [
|
||||
{
|
||||
name: "Total",
|
||||
...Object.fromEntries(items.map((item) => [item.category, item.count])),
|
||||
...Object.fromEntries(data?.items.map((item) => [item.category, item.count])),
|
||||
},
|
||||
];
|
||||
|
||||
const barData = items.map((item) => ({
|
||||
fill: getColorFromName(item.category),
|
||||
name: item.category,
|
||||
}));
|
||||
|
||||
return (
|
||||
<Card className="pt-0">
|
||||
<CardHeader className="flex items-center gap-2 space-y-0 border-b py-5 sm:flex-row">
|
||||
@@ -50,17 +44,7 @@ export function SourceCategorySharesChart({ sourceId }: Props) {
|
||||
<CardTitle>Category Shares</CardTitle>
|
||||
<CardDescription>showing top {limit} categories for this source</CardDescription>
|
||||
</div>
|
||||
<ToggleGroup
|
||||
className="*:data-[slot=toggle-group-item]:px-4! @[767px]/card:flex"
|
||||
onValueChange={(v) => setLimit(Number(v))}
|
||||
type="single"
|
||||
value={String(limit)}
|
||||
variant="outline"
|
||||
>
|
||||
<ToggleGroupItem value="10">Top 10</ToggleGroupItem>
|
||||
<ToggleGroupItem value="20">Top 20</ToggleGroupItem>
|
||||
<ToggleGroupItem value="50">Top 50</ToggleGroupItem>
|
||||
</ToggleGroup>
|
||||
<ChartLimitToggle paramKey={`categoryLimit-${sourceId}`} />
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<div className="-ml-1 h-20">
|
||||
@@ -80,15 +64,15 @@ export function SourceCategorySharesChart({ sourceId }: Props) {
|
||||
/>
|
||||
<XAxis axisLine={false} fontSize={12} hide tickLine={false} type="number" />
|
||||
<Legend align="left" iconSize={8} iconType="circle" />
|
||||
{barData.map((entry, index) => (
|
||||
{data?.items.map((entry, index) => (
|
||||
<Bar
|
||||
barSize={16}
|
||||
className="transition-all delay-75"
|
||||
dataKey={entry.name}
|
||||
fill={entry.fill}
|
||||
dataKey={entry.category}
|
||||
fill={getColorFromName(entry.category)}
|
||||
key={`bar-${index}`}
|
||||
radius={
|
||||
index === 0 ? [4, 0, 0, 4] : index === barData.length - 1 ? [0, 4, 4, 0] : 0
|
||||
index === 0 ? [4, 0, 0, 4] : index === data?.items.length - 1 ? [0, 4, 4, 0] : 0
|
||||
}
|
||||
stackId="category"
|
||||
/>
|
||||
@@ -0,0 +1,62 @@
|
||||
// @ts-nocheck
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import { ChartConfig, ChartContainer } from "@basango/ui/components/chart";
|
||||
import { useQuery } from "@tanstack/react-query";
|
||||
|
||||
import { AreaChart } from "#dashboard/components/charts/area-chart";
|
||||
import {
|
||||
ChartPeriodPicker,
|
||||
useChartPeriodFilter,
|
||||
} from "#dashboard/components/charts/chart-filters";
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
|
||||
const chartConfig = {
|
||||
count: {
|
||||
color: "var(--chart-2)",
|
||||
label: "Articles",
|
||||
},
|
||||
views: {
|
||||
label: "Articles",
|
||||
},
|
||||
} satisfies ChartConfig;
|
||||
|
||||
type Props = {
|
||||
sourceId: string;
|
||||
};
|
||||
|
||||
export function PublicationGraphChart({ sourceId }: Props) {
|
||||
const trpc = useTRPC();
|
||||
const period = useChartPeriodFilter();
|
||||
|
||||
const { data } = useQuery(
|
||||
trpc.sources.getPublicationGraph.queryOptions({
|
||||
id: sourceId,
|
||||
range: period.range,
|
||||
}),
|
||||
);
|
||||
|
||||
return (
|
||||
<Card className="pt-0">
|
||||
<CardHeader className="flex items-center gap-2 space-y-0 border-b py-5 sm:flex-row">
|
||||
<div className="grid flex-1 gap-1">
|
||||
<CardTitle>Publication Graph</CardTitle>
|
||||
<CardDescription>Showing total crawled articles for the selected period</CardDescription>
|
||||
</div>
|
||||
<ChartPeriodPicker defaultDays={period.defaultDays} paramKey={`sourcePeriod-${sourceId}`} />
|
||||
</CardHeader>
|
||||
<CardContent className="px-2 pt-4 sm:px-6 sm:pt-6">
|
||||
<ChartContainer className="aspect-auto h-[250px] w-full" config={chartConfig}>
|
||||
<AreaChart data={data?.items} />
|
||||
</ChartContainer>
|
||||
</CardContent>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,38 @@
|
||||
import { Delta } from "@basango/domain/models";
|
||||
import { cn } from "@basango/ui/lib/utils";
|
||||
import { ArrowDownRightIcon, ArrowUpRightIcon } from "lucide-react";
|
||||
|
||||
import { formatNumber } from "#dashboard/utils/utils";
|
||||
|
||||
type StatusProps = {
|
||||
value: Delta | undefined;
|
||||
percentage?: boolean;
|
||||
icons?: boolean;
|
||||
sign?: boolean;
|
||||
};
|
||||
|
||||
export function Status({ value, percentage = false, icons = true, sign = true }: StatusProps) {
|
||||
if (value === undefined) {
|
||||
return <span className="text-muted-foreground">0</span>;
|
||||
}
|
||||
|
||||
const color = value.delta >= 0 ? "text-emerald-600" : "text-rose-600";
|
||||
const icon =
|
||||
value.delta >= 0 ? (
|
||||
<ArrowUpRightIcon className={cn("size-4", color)} />
|
||||
) : (
|
||||
<ArrowDownRightIcon className={cn("size-4", color)} />
|
||||
);
|
||||
|
||||
return (
|
||||
<>
|
||||
{icons && icon}
|
||||
<span className={cn("font-semibold", color)}>
|
||||
{sign && value.sign}
|
||||
{percentage
|
||||
? `${formatNumber(Math.abs(value.percentage))}%`
|
||||
: formatNumber(Math.abs(value.delta))}
|
||||
</span>
|
||||
</>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,176 @@
|
||||
"use client";
|
||||
|
||||
import type { RouterOutputs } from "@basango/api/trpc/routers/_app";
|
||||
import { updateSourceSchema } from "@basango/domain/models/sources";
|
||||
import {
|
||||
Field,
|
||||
FieldDescription,
|
||||
FieldError,
|
||||
FieldGroup,
|
||||
FieldLabel,
|
||||
} from "@basango/ui/components/field";
|
||||
import { Input } from "@basango/ui/components/input";
|
||||
import { SubmitButton } from "@basango/ui/components/submit-button";
|
||||
import { Textarea } from "@basango/ui/components/textarea";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { useCallback, useEffect } from "react";
|
||||
import { Controller } from "react-hook-form";
|
||||
import { toast } from "sonner";
|
||||
import { z } from "zod";
|
||||
|
||||
import { useZodForm } from "#dashboard/hooks/use-zod-form";
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
|
||||
const baseSchema = updateSourceSchema.pick({
|
||||
description: true,
|
||||
displayName: true,
|
||||
id: true,
|
||||
name: true,
|
||||
});
|
||||
|
||||
const sourceEditSchema = z.object({
|
||||
description: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((value) => {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
})
|
||||
.pipe(baseSchema.shape.description),
|
||||
displayName: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((value) => {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
})
|
||||
.pipe(baseSchema.shape.displayName),
|
||||
id: baseSchema.shape.id,
|
||||
name: z.string().trim().pipe(baseSchema.shape.name),
|
||||
});
|
||||
|
||||
type SourceEditValues = z.infer<typeof sourceEditSchema>;
|
||||
|
||||
type Props = {
|
||||
source: RouterOutputs["sources"]["getById"];
|
||||
};
|
||||
|
||||
export function SourceEditForm({ source }: Props) {
|
||||
const trpc = useTRPC();
|
||||
const queryClient = useQueryClient();
|
||||
|
||||
const form = useZodForm(sourceEditSchema, {
|
||||
defaultValues: {
|
||||
description: source.description ?? "",
|
||||
displayName: source.displayName ?? "",
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
},
|
||||
mode: "onChange",
|
||||
});
|
||||
|
||||
useEffect(() => {
|
||||
form.reset({
|
||||
description: source.description ?? "",
|
||||
displayName: source.displayName ?? "",
|
||||
id: source.id,
|
||||
name: source.name,
|
||||
});
|
||||
}, [form, source.description, source.displayName, source.id, source.name]);
|
||||
|
||||
const mutation = useMutation(
|
||||
trpc.sources.update.mutationOptions({
|
||||
onError(error) {
|
||||
toast.error(error.message ?? "Unable to update source.");
|
||||
},
|
||||
onSuccess() {
|
||||
toast.success("Source updated successfully.");
|
||||
void Promise.all([
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: trpc.sources.list.queryKey(),
|
||||
}),
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: trpc.sources.getById.queryKey({ id: source.id }),
|
||||
}),
|
||||
]);
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
(values: SourceEditValues) => {
|
||||
mutation.mutate(values);
|
||||
},
|
||||
[mutation],
|
||||
);
|
||||
|
||||
return (
|
||||
<form className="space-y-6" onSubmit={form.handleSubmit(handleSubmit)}>
|
||||
<FieldGroup>
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="name"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Name</FieldLabel>
|
||||
<Input
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
autoComplete="off"
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="radiookapi.com"
|
||||
/>
|
||||
<FieldDescription>Internal identifier of the source.</FieldDescription>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="displayName"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Display name</FieldLabel>
|
||||
<Input
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
autoComplete="off"
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="Radio Okapi"
|
||||
/>
|
||||
<FieldDescription>Optional friendly label shown in the dashboard.</FieldDescription>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="description"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Description</FieldLabel>
|
||||
<Textarea
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="Short summary about the source..."
|
||||
rows={4}
|
||||
/>
|
||||
<FieldDescription>Optional summary shown across the product.</FieldDescription>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
</FieldGroup>
|
||||
|
||||
<SubmitButton className="w-full" isSubmitting={mutation.isPending} type="submit">
|
||||
Save changes
|
||||
</SubmitButton>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,186 @@
|
||||
"use client";
|
||||
|
||||
import { createSourceSchema } from "@basango/domain/models/sources";
|
||||
import {
|
||||
Field,
|
||||
FieldDescription,
|
||||
FieldError,
|
||||
FieldGroup,
|
||||
FieldLabel,
|
||||
} from "@basango/ui/components/field";
|
||||
import { Input } from "@basango/ui/components/input";
|
||||
import { SubmitButton } from "@basango/ui/components/submit-button";
|
||||
import { Textarea } from "@basango/ui/components/textarea";
|
||||
import { useMutation, useQueryClient } from "@tanstack/react-query";
|
||||
import { useCallback } from "react";
|
||||
import { Controller } from "react-hook-form";
|
||||
import { toast } from "sonner";
|
||||
import { z } from "zod";
|
||||
|
||||
import { useZodForm } from "#dashboard/hooks/use-zod-form";
|
||||
import { useTRPC } from "#dashboard/trpc/client";
|
||||
|
||||
const baseSchema = createSourceSchema.pick({
|
||||
description: true,
|
||||
displayName: true,
|
||||
name: true,
|
||||
url: true,
|
||||
});
|
||||
|
||||
const sourceFormSchema = z.object({
|
||||
description: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((value) => {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
})
|
||||
.pipe(baseSchema.shape.description),
|
||||
displayName: z
|
||||
.string()
|
||||
.optional()
|
||||
.transform((value) => {
|
||||
const trimmed = value?.trim();
|
||||
return trimmed ? trimmed : undefined;
|
||||
})
|
||||
.pipe(baseSchema.shape.displayName),
|
||||
name: z.string().trim().pipe(baseSchema.shape.name),
|
||||
url: z.string().trim().pipe(baseSchema.shape.url),
|
||||
});
|
||||
|
||||
type SourceFormValues = z.infer<typeof sourceFormSchema>;
|
||||
|
||||
type SourceFormProps = {
|
||||
onSuccess?: () => void;
|
||||
};
|
||||
|
||||
export function SourceForm({ onSuccess }: SourceFormProps) {
|
||||
const trpc = useTRPC();
|
||||
const queryClient = useQueryClient();
|
||||
const form = useZodForm(sourceFormSchema, {
|
||||
defaultValues: {
|
||||
description: "",
|
||||
displayName: "",
|
||||
name: "",
|
||||
url: "",
|
||||
},
|
||||
});
|
||||
|
||||
const mutation = useMutation(
|
||||
trpc.sources.create.mutationOptions({
|
||||
onError(error) {
|
||||
toast.error(error.message ?? "Unable to create source.");
|
||||
},
|
||||
onSuccess() {
|
||||
toast.success("Source created successfully.");
|
||||
queryClient.invalidateQueries({
|
||||
queryKey: trpc.sources.list.queryKey(),
|
||||
});
|
||||
form.reset();
|
||||
onSuccess?.();
|
||||
},
|
||||
}),
|
||||
);
|
||||
|
||||
const handleSubmit = useCallback(
|
||||
(values: SourceFormValues) => {
|
||||
mutation.mutate({
|
||||
...values,
|
||||
});
|
||||
},
|
||||
[mutation],
|
||||
);
|
||||
|
||||
return (
|
||||
<form className="space-y-6" onSubmit={form.handleSubmit(handleSubmit)}>
|
||||
<FieldGroup>
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="name"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Name</FieldLabel>
|
||||
<Input
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
autoComplete="off"
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="radiookapi.com"
|
||||
/>
|
||||
<FieldDescription>
|
||||
This should match the unique identifier of the source.
|
||||
</FieldDescription>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="displayName"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Display name</FieldLabel>
|
||||
<Input
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
autoComplete="off"
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="Radio Okapi"
|
||||
/>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="url"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Website URL</FieldLabel>
|
||||
<Input
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
autoComplete="off"
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="https://techcrunch.com"
|
||||
type="url"
|
||||
/>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
|
||||
<Controller
|
||||
control={form.control}
|
||||
name="description"
|
||||
render={({ field, fieldState }) => (
|
||||
<Field data-invalid={fieldState.invalid}>
|
||||
<FieldLabel htmlFor={field.name}>Description</FieldLabel>
|
||||
<Textarea
|
||||
{...field}
|
||||
aria-invalid={fieldState.invalid}
|
||||
disabled={mutation.isPending}
|
||||
id={field.name}
|
||||
placeholder="Short summary about the source..."
|
||||
rows={4}
|
||||
/>
|
||||
<FieldDescription>
|
||||
Optional brief description (up to 1024 characters).
|
||||
</FieldDescription>
|
||||
{fieldState.invalid && <FieldError errors={[fieldState.error]} />}
|
||||
</Field>
|
||||
)}
|
||||
/>
|
||||
</FieldGroup>
|
||||
|
||||
<SubmitButton className="w-full" isSubmitting={mutation.isPending} type="submit">
|
||||
Create source
|
||||
</SubmitButton>
|
||||
</form>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,47 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Dialog,
|
||||
DialogContent,
|
||||
DialogDescription,
|
||||
DialogHeader,
|
||||
DialogTitle,
|
||||
} from "@basango/ui/components/dialog";
|
||||
import { useCallback } from "react";
|
||||
|
||||
import { SourceForm } from "#dashboard/components/forms/source-form";
|
||||
import { useSourceParams } from "#dashboard/hooks/use-source-params";
|
||||
|
||||
export function SourceCreateModal() {
|
||||
const { createSource, setParams } = useSourceParams();
|
||||
const isOpen = Boolean(createSource);
|
||||
|
||||
const openDialog = useCallback(() => {
|
||||
void setParams({ createSource: true });
|
||||
}, [setParams]);
|
||||
|
||||
const closeDialog = useCallback(() => {
|
||||
void setParams(null);
|
||||
}, [setParams]);
|
||||
|
||||
return (
|
||||
<Dialog
|
||||
onOpenChange={(open) => {
|
||||
if (open) {
|
||||
openDialog();
|
||||
} else {
|
||||
closeDialog();
|
||||
}
|
||||
}}
|
||||
open={isOpen}
|
||||
>
|
||||
<DialogContent>
|
||||
<DialogHeader>
|
||||
<DialogTitle>Create a new source</DialogTitle>
|
||||
<DialogDescription>Add a news outlet to start tracking its articles.</DialogDescription>
|
||||
</DialogHeader>
|
||||
<SourceForm onSuccess={closeDialog} />
|
||||
</DialogContent>
|
||||
</Dialog>
|
||||
);
|
||||
}
|
||||
@@ -11,7 +11,7 @@ export const PageLayout = (props: React.PropsWithChildren<PageProps>) => {
|
||||
|
||||
return (
|
||||
<div className="flex flex-1 flex-col gap-4 p-4">
|
||||
<div className="container mx-auto">
|
||||
<div className="container mx-auto space-y-4">
|
||||
{title && (
|
||||
<div className="mb-8 flex items-center justify-between space-y-4">
|
||||
<div>
|
||||
|
||||
@@ -7,7 +7,7 @@ import {
|
||||
SidebarHeader,
|
||||
SidebarRail,
|
||||
} from "@basango/ui/components/sidebar";
|
||||
import { SquareTerminal } from "lucide-react";
|
||||
import { LayoutDashboard, SquareTerminal } from "lucide-react";
|
||||
import * as React from "react";
|
||||
|
||||
import { AppSidebarContent } from "./app-sidebar-content";
|
||||
@@ -16,6 +16,18 @@ import { AppSidebarUser } from "./app-sidebar-user";
|
||||
|
||||
const data = {
|
||||
main: [
|
||||
{
|
||||
icon: LayoutDashboard,
|
||||
isActive: true,
|
||||
items: [
|
||||
{
|
||||
title: "Dashboard",
|
||||
url: "/dashboard",
|
||||
},
|
||||
],
|
||||
title: "Overview",
|
||||
url: "#",
|
||||
},
|
||||
{
|
||||
icon: SquareTerminal,
|
||||
isActive: true,
|
||||
|
||||
@@ -0,0 +1,86 @@
|
||||
"use client";
|
||||
|
||||
import { Source } from "@basango/domain/models/sources";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardFooter,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import {
|
||||
ChartConfig,
|
||||
ChartContainer,
|
||||
ChartTooltip,
|
||||
ChartTooltipContent,
|
||||
} from "@basango/ui/components/chart";
|
||||
import { Area, AreaChart, CartesianGrid, XAxis, YAxis } from "recharts";
|
||||
|
||||
import { formatDate, formatNumber } from "#dashboard/utils/utils";
|
||||
|
||||
const chartConfig = {
|
||||
count: {
|
||||
color: "var(--chart-2)",
|
||||
label: "Articles",
|
||||
},
|
||||
views: {
|
||||
label: "Articles",
|
||||
},
|
||||
} satisfies ChartConfig;
|
||||
|
||||
export function SourceCard({ source }: { source: Source }) {
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{source.name}</CardTitle>
|
||||
<CardDescription>{source.id}</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<ChartContainer config={chartConfig}>
|
||||
<AreaChart accessibilityLayer data={source.publications?.items}>
|
||||
<CartesianGrid strokeDasharray="3 3" vertical={false} />
|
||||
<XAxis
|
||||
axisLine={false}
|
||||
dataKey="date"
|
||||
minTickGap={32}
|
||||
tickFormatter={(value) => formatDate(String(value))}
|
||||
tickLine={false}
|
||||
tickMargin={8}
|
||||
/>
|
||||
<YAxis
|
||||
allowDecimals={false}
|
||||
axisLine={false}
|
||||
tickFormatter={(value) => formatNumber(Number(value))}
|
||||
tickLine={false}
|
||||
width={48}
|
||||
/>
|
||||
<ChartTooltip
|
||||
content={
|
||||
<ChartTooltipContent
|
||||
labelFormatter={(value) => formatDate(String(value), "PP")}
|
||||
nameKey="count"
|
||||
/>
|
||||
}
|
||||
cursor={{ stroke: "var(--border)", strokeDasharray: "4 4" }}
|
||||
/>
|
||||
<Area
|
||||
dataKey="count"
|
||||
fill="var(--color-count)"
|
||||
fillOpacity={0.15}
|
||||
stroke="var(--color-count)"
|
||||
strokeWidth={2}
|
||||
type="monotone"
|
||||
/>
|
||||
</AreaChart>
|
||||
</ChartContainer>
|
||||
</CardContent>
|
||||
<CardFooter className="flex-col items-start gap-2 text-sm">
|
||||
<div className="flex gap-2 leading-none font-medium">
|
||||
{formatNumber(source.articles)} articles crawled
|
||||
</div>
|
||||
<div className="text-muted-foreground leading-none">Showing last 30 days</div>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,72 @@
|
||||
"use client";
|
||||
|
||||
import { Source } from "@basango/domain/models";
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import Link from "next/link";
|
||||
import type { ReactNode } from "react";
|
||||
|
||||
import { SourceEditForm } from "#dashboard/components/forms/source-edit-form";
|
||||
|
||||
export function SourceDetailsTab({ source }: { source: Source }) {
|
||||
const credibility = source.credibility;
|
||||
|
||||
return (
|
||||
<div className="grid gap-4 lg:grid-cols-[2fr_1fr]">
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Source details</CardTitle>
|
||||
<CardDescription>Key properties of this publication.</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<dl className="grid gap-6 sm:grid-cols-2">
|
||||
<DetailItem label="Name" value={source.name} />
|
||||
<DetailItem label="Display name" value={source.displayName ?? "—"} />
|
||||
<DetailItem
|
||||
label="Website"
|
||||
value={
|
||||
<Link
|
||||
className="text-primary underline underline-offset-4"
|
||||
href={source.url}
|
||||
target="_blank"
|
||||
>
|
||||
{source.url}
|
||||
</Link>
|
||||
}
|
||||
/>
|
||||
<DetailItem label="Description" value={source.description ?? "Not provided"} />
|
||||
<DetailItem label="Bias" value={credibility?.bias ?? "Unknown"} />
|
||||
<DetailItem label="Reliability" value={credibility?.reliability ?? "Unknown"} />
|
||||
<DetailItem label="Transparency" value={credibility?.transparency ?? "Unknown"} />
|
||||
</dl>
|
||||
</CardContent>
|
||||
</Card>
|
||||
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>Edit source</CardTitle>
|
||||
<CardDescription>
|
||||
Update the name or description shown inside the dashboard.
|
||||
</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<SourceEditForm source={source} />
|
||||
</CardContent>
|
||||
</Card>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
|
||||
function DetailItem({ label, value }: { label: string; value: ReactNode }) {
|
||||
return (
|
||||
<div className="space-y-1">
|
||||
<dt className="text-muted-foreground text-sm">{label}</dt>
|
||||
<dd className="text-base font-medium wrap-break-word">{value}</dd>
|
||||
</div>
|
||||
);
|
||||
}
|
||||
@@ -1,83 +0,0 @@
|
||||
"use client";
|
||||
|
||||
import {
|
||||
Card,
|
||||
CardContent,
|
||||
CardDescription,
|
||||
CardFooter,
|
||||
CardHeader,
|
||||
CardTitle,
|
||||
} from "@basango/ui/components/card";
|
||||
import {
|
||||
ChartConfig,
|
||||
ChartContainer,
|
||||
ChartTooltip,
|
||||
ChartTooltipContent,
|
||||
} from "@basango/ui/components/chart";
|
||||
import { Bar, BarChart, CartesianGrid, XAxis } from "recharts";
|
||||
|
||||
import { formatDate, formatNumber } from "#dashboard/utils/utils";
|
||||
|
||||
const chartConfig = {
|
||||
count: {
|
||||
color: "var(--chart-2)",
|
||||
label: "Articles",
|
||||
},
|
||||
views: {
|
||||
label: "Articles",
|
||||
},
|
||||
} satisfies ChartConfig;
|
||||
|
||||
type SourceDetails = {
|
||||
id: string;
|
||||
name: string;
|
||||
displayName: string | null;
|
||||
url: string;
|
||||
description: string;
|
||||
publicationGraph: {
|
||||
items: { date: string; count: number }[];
|
||||
total: number;
|
||||
};
|
||||
credibility: {
|
||||
bias: string;
|
||||
reliability: string;
|
||||
transparency: string;
|
||||
};
|
||||
articles: number;
|
||||
};
|
||||
|
||||
export function SourceCard({ source }: { source: SourceDetails }) {
|
||||
return (
|
||||
<Card>
|
||||
<CardHeader>
|
||||
<CardTitle>{source.name}</CardTitle>
|
||||
<CardDescription>{source.id}</CardDescription>
|
||||
</CardHeader>
|
||||
<CardContent>
|
||||
<ChartContainer config={chartConfig}>
|
||||
<BarChart accessibilityLayer data={source.publicationGraph.items}>
|
||||
<CartesianGrid vertical={false} />
|
||||
<XAxis
|
||||
axisLine={false}
|
||||
dataKey="date"
|
||||
minTickGap={32}
|
||||
tickFormatter={(value) => formatDate(value)}
|
||||
tickLine={false}
|
||||
tickMargin={8}
|
||||
/>
|
||||
<ChartTooltip content={<ChartTooltipContent indicator="dashed" />} cursor={false} />
|
||||
<Bar dataKey="count" fill="var(--color-count)" radius={4} />
|
||||
</BarChart>
|
||||
</ChartContainer>
|
||||
</CardContent>
|
||||
<CardFooter className="flex-col items-start gap-2 text-sm">
|
||||
<div className="flex gap-2 leading-none font-medium">
|
||||
{formatNumber(source.articles)} articles crawled
|
||||
</div>
|
||||
<div className="text-muted-foreground leading-none">
|
||||
Showing last {source.publicationGraph.total} days
|
||||
</div>
|
||||
</CardFooter>
|
||||
</Card>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,12 @@
|
||||
import { parseAsBoolean, useQueryStates } from "nuqs";
|
||||
|
||||
export function useSourceParams() {
|
||||
const [params, setParams] = useQueryStates({
|
||||
createSource: parseAsBoolean,
|
||||
});
|
||||
|
||||
return {
|
||||
...params,
|
||||
setParams,
|
||||
};
|
||||
}
|
||||
@@ -4,7 +4,11 @@ import "server-only";
|
||||
import type { AppRouter } from "@basango/api/trpc/routers/_app";
|
||||
import { HydrationBoundary, dehydrate } from "@tanstack/react-query";
|
||||
import { createTRPCClient, httpBatchLink, loggerLink } from "@trpc/client";
|
||||
import { type TRPCQueryOptions, createTRPCOptionsProxy } from "@trpc/tanstack-react-query";
|
||||
import {
|
||||
type TRPCInfiniteQueryOptions,
|
||||
type TRPCQueryOptions,
|
||||
createTRPCOptionsProxy,
|
||||
} from "@trpc/tanstack-react-query";
|
||||
import { cache } from "react";
|
||||
import superjson from "superjson";
|
||||
|
||||
@@ -46,7 +50,11 @@ export function HydrateClient(props: { children: React.ReactNode }) {
|
||||
return <HydrationBoundary state={dehydrate(queryClient)}>{props.children}</HydrationBoundary>;
|
||||
}
|
||||
|
||||
export function prefetch<T extends ReturnType<TRPCQueryOptions<any>>>(queryOptions: T) {
|
||||
type AnyQueryOptions =
|
||||
| ReturnType<TRPCQueryOptions<any>>
|
||||
| ReturnType<TRPCInfiniteQueryOptions<any>>;
|
||||
|
||||
export function prefetch<T extends AnyQueryOptions>(queryOptions: T) {
|
||||
const queryClient = getQueryClient();
|
||||
if (queryOptions.queryKey[1]?.type === "infinite") {
|
||||
void queryClient.prefetchInfiniteQuery(queryOptions as any);
|
||||
@@ -55,7 +63,7 @@ export function prefetch<T extends ReturnType<TRPCQueryOptions<any>>>(queryOptio
|
||||
}
|
||||
}
|
||||
|
||||
export function batchPrefetch<T extends ReturnType<TRPCQueryOptions<any>>>(queryOptionsArray: T[]) {
|
||||
export function batchPrefetch<T extends AnyQueryOptions>(queryOptionsArray: T[]) {
|
||||
const queryClient = getQueryClient();
|
||||
|
||||
for (const queryOptions of queryOptionsArray) {
|
||||
|
||||
@@ -42,8 +42,8 @@ export function formatDate(date: string, dateFormat?: string | null, checkYear =
|
||||
return format(new Date(date), dateFormat ?? "P");
|
||||
}
|
||||
|
||||
export function formatNumber(value: number): string {
|
||||
return Intl.NumberFormat("en-US").format(value);
|
||||
export function formatNumber(value: number | undefined): string {
|
||||
return Intl.NumberFormat("en-US").format(value ?? 0);
|
||||
}
|
||||
|
||||
export function getInitials(value: string) {
|
||||
|
||||
@@ -1,11 +1,5 @@
|
||||
{
|
||||
"compilerOptions": {
|
||||
"baseUrl": ".",
|
||||
"paths": {
|
||||
"@basango/api/*": ["../api/src/*"],
|
||||
"@basango/ui/*": ["../../packages/ui/src/*"],
|
||||
"#dashboard/*": ["./src/*"]
|
||||
},
|
||||
"plugins": [
|
||||
{
|
||||
"name": "next"
|
||||
@@ -14,10 +8,5 @@
|
||||
},
|
||||
"exclude": ["node_modules"],
|
||||
"extends": "@basango/tsconfig/nextjs.json",
|
||||
"include": ["next-env.d.ts", "next.config.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
|
||||
"references": [
|
||||
{
|
||||
"path": "../api"
|
||||
}
|
||||
]
|
||||
"include": ["next-env.d.ts", "next.config.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"]
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user