feat: introduced bulk actions for runs

This commit is contained in:
2026-08-28 14:53:24 +02:00
parent 8788b9f5b7
commit d7c88ad88e
39 changed files with 362 additions and 223 deletions
+3 -2
View File
@@ -40,7 +40,7 @@ Architecture
- Keep route composition, navigation, environment access, and platform behavior in the owning application.
- Start new behavior in its owning application. Extract it only for at least two real consumers or a clear package-owned responsibility.
- Packages expose small, intentional interfaces and hide cohesive implementation details.
- No package may import from an application. No client application may import `@basango/db`, `@basango/logger`, or `@basango/encryption`.
- No package may import from an application. No client application may import `@basango/db` or `@basango/logger`.
- Dashboard code imports API router types only through the explicit `@basango/api/trpc/routers/_app` export.
- Mobile may share platform-neutral domain contracts but must not import the DOM-based `@basango/ui` package.
- Follow the dependency graph in `docs/web/README.md`. A new lateral package dependency requires a real ownership relationship and a documentation update.
@@ -128,7 +128,8 @@ Logging
- Production logs are structured JSON; non-production uses `pino-pretty` transport.
Testing
- Use `vitest` where present. Add tests locally to the package being changed.
- Keep tests under the root `tests/` directory, mirroring their owning workspace and source path.
- Use `bun:test` unless an existing test area already uses another runner.
- Keep tests fast and focused. Do not introduce global test state.
Quality Gates
-1
View File
@@ -21,7 +21,6 @@
|-------------------|-----------------------------------------------|
| Database | [README.md](./packages/db/README.md) |
| Domain | [README.md](./packages/domain/README.md) |
| Encryption | [README.md](./packages/encryption/README.md) |
| Logger | [README.md](./packages/logger/README.md) |
| User Interface | [README.md](./packages/ui/README.md) |
-1
View File
@@ -2,7 +2,6 @@
"dependencies": {
"@basango/db": "workspace:*",
"@basango/domain": "workspace:*",
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"@better-auth/drizzle-adapter": "^1.7.1",
"@hono/node-server": "^1.19.6",
@@ -1,10 +1,11 @@
import { createHash } from "node:crypto";
import { createArticle } from "@basango/db/queries";
import {
articleHashSchema,
createArticleResponseSchema,
createArticleSchema,
} from "@basango/domain/models";
import { md5 } from "@basango/encryption";
import { OpenAPIHono, createRoute } from "@hono/zod-openapi";
import { HTTPException } from "hono/http-exception";
import { z } from "zod";
@@ -70,3 +71,7 @@ app.openapi(
);
export const articleIngestionRouter = app;
function md5(value: string): string {
return createHash("md5").update(value).digest("hex");
}
+17 -5
View File
@@ -6,6 +6,7 @@ import type {
IngestionSignal,
IngestionSignalType,
} from "@basango/domain/models";
import * as uuid from "uuid";
import { invalidateIngestionThroughput } from "./throughput";
@@ -36,7 +37,7 @@ export async function acceptIngestionSignal(db: Database, signal: IngestionSigna
invalidateIngestionThroughput();
}
queueIngestionChange(signal);
queueSignalChange(signal);
}
return result;
@@ -52,13 +53,24 @@ export function getIngestionChangeTopics(type: IngestionSignalType) {
return topicsBySignalType[type];
}
function queueIngestionChange(signal: IngestionSignal) {
for (const topic of getIngestionChangeTopics(signal.type)) {
export function announceIngestionChange(topics: readonly IngestionChangeTopic[]) {
queueIngestionChange({ latestSignalId: uuid.v7(), topics: [...topics] });
}
function queueSignalChange(signal: IngestionSignal) {
queueIngestionChange({
latestSignalId: signal.signalId,
topics: getIngestionChangeTopics(signal.type),
});
}
function queueIngestionChange(change: IngestionChange) {
for (const topic of change.topics) {
pendingTopics.add(topic);
}
if (!pendingSignalId || signal.signalId > pendingSignalId) {
pendingSignalId = signal.signalId;
if (!pendingSignalId || change.latestSignalId > pendingSignalId) {
pendingSignalId = change.latestSignalId;
}
if (!flushTimer) {
+19 -2
View File
@@ -1,10 +1,27 @@
import { getIngestionAgents, getIngestionSummary, listIngestionRuns } from "@basango/db/queries";
import { ingestionRunsQuerySchema } from "@basango/domain/models";
import {
closeIngestionRuns,
getIngestionAgents,
getIngestionSummary,
listIngestionRuns,
} from "@basango/db/queries";
import { closeIngestionRunsSchema, ingestionRunsQuerySchema } from "@basango/domain/models";
import { announceIngestionChange } from "#api/services/ingestion/signals";
import { getIngestionThroughputSnapshot } from "#api/services/ingestion/throughput";
import { adminProcedure, createTRPCRouter } from "#api/trpc/init";
export const operationsRouter = createTRPCRouter({
closeIngestionRuns: adminProcedure
.input(closeIngestionRunsSchema)
.mutation(async ({ ctx, input }) => {
const result = await closeIngestionRuns(ctx.db, input);
if (result.updatedCount > 0 || result.releasedAgentCount > 0) {
announceIngestionChange(["agents", "runs", "summary"]);
}
return result;
}),
getIngestionAgents: adminProcedure.query(({ ctx }) => getIngestionAgents(ctx.db)),
getIngestionSummary: adminProcedure.query(({ ctx }) => getIngestionSummary(ctx.db)),
getIngestionThroughput: adminProcedure.query(({ ctx }) => getIngestionThroughputSnapshot(ctx.db)),
@@ -0,0 +1,150 @@
"use client";
import type { IngestionRunState } from "@basango/domain/models";
import {
AlertDialog,
AlertDialogAction,
AlertDialogCancel,
AlertDialogContent,
AlertDialogDescription,
AlertDialogFooter,
AlertDialogHeader,
AlertDialogTitle,
} from "@basango/ui/components/alert-dialog";
import { Button } from "@basango/ui/components/button";
import { useMutation, useQueryClient } from "@tanstack/react-query";
import type { Table } from "@tanstack/react-table";
import { CircleCheckIcon, CircleXIcon } from "lucide-react";
import { useState } from "react";
import { toast } from "sonner";
import { useTRPC } from "#dashboard/app/trpc/client";
import type { IngestionRun } from "../types";
type TerminalRunState = Extract<IngestionRunState, "completed" | "failed">;
type IngestionRunsBulkActionsProps = {
runIds: string[];
table: Table<IngestionRun>;
};
export function IngestionRunsBulkActions({ runIds, table }: IngestionRunsBulkActionsProps) {
const trpc = useTRPC();
const queryClient = useQueryClient();
const [targetState, setTargetState] = useState<TerminalRunState | undefined>(undefined);
const selectedCount = runIds.length;
const closeRuns = useMutation(
trpc.operations.closeIngestionRuns.mutationOptions({
onError(error) {
toast.error(error.message ?? "Unable to close the selected runs.");
},
onSuccess(result, input) {
if (result.updatedCount > 0) {
toast.success(formatSuccessMessage(result.updatedCount, input.state));
}
if (result.unchangedCount > 0) {
toast.info(
`${result.unchangedCount} ${pluralize("run", result.unchangedCount)} already had that status or were unavailable.`,
);
}
table.resetRowSelection();
setTargetState(undefined);
void Promise.all([
queryClient.invalidateQueries({
queryKey: trpc.operations.getIngestionAgents.queryKey(),
}),
queryClient.invalidateQueries({
queryKey: trpc.operations.getIngestionSummary.queryKey(),
}),
queryClient.invalidateQueries({
queryKey: trpc.operations.listIngestionRuns.queryKey(),
}),
]);
},
}),
);
if (selectedCount === 0) {
return null;
}
function closeSelectedRuns() {
if (!targetState) {
return;
}
closeRuns.mutate({
runIds,
state: targetState,
});
}
return (
<>
<span className="text-xs text-muted-foreground">
{selectedCount} {pluralize("run", selectedCount)} selected
</span>
<Button
disabled={closeRuns.isPending}
onClick={() => setTargetState("completed")}
size="sm"
type="button"
variant="outline"
>
<CircleCheckIcon />
Mark completed
</Button>
<Button
disabled={closeRuns.isPending}
onClick={() => setTargetState("failed")}
size="sm"
type="button"
variant="destructive"
>
<CircleXIcon />
Mark failed
</Button>
<AlertDialog
onOpenChange={(open) => {
if (!open && !closeRuns.isPending) {
setTargetState(undefined);
}
}}
open={targetState !== undefined}
>
<AlertDialogContent>
<AlertDialogHeader>
<AlertDialogTitle>
Mark {selectedCount} {pluralize("run", selectedCount)} as {targetState}?
</AlertDialogTitle>
<AlertDialogDescription>
The selected runs will be assigned this final status. This also releases any agent
that still points to one of these runs.
</AlertDialogDescription>
</AlertDialogHeader>
<AlertDialogFooter>
<AlertDialogCancel disabled={closeRuns.isPending}>Cancel</AlertDialogCancel>
<AlertDialogAction
disabled={closeRuns.isPending}
onClick={closeSelectedRuns}
variant={targetState === "failed" ? "destructive" : "default"}
>
{closeRuns.isPending ? "Updating…" : "Confirm"}
</AlertDialogAction>
</AlertDialogFooter>
</AlertDialogContent>
</AlertDialog>
</>
);
}
function formatSuccessMessage(count: number, state: TerminalRunState) {
return `${count} ${pluralize("run", count)} marked as ${state}.`;
}
function pluralize(noun: string, count: number) {
return count === 1 ? noun : `${noun}s`;
}
@@ -1,4 +1,5 @@
import { Badge } from "@basango/ui/components/badge";
import { Checkbox } from "@basango/ui/components/checkbox";
import type { ColumnDef } from "@tanstack/react-table";
import { formatDuration, relativeTime, stateVariant } from "../../shared/ingestion-formatters";
@@ -6,6 +7,27 @@ import type { IngestionRun } from "../types";
export function createIngestionRunColumns(): ColumnDef<IngestionRun>[] {
return [
{
cell: ({ row }) => (
<Checkbox
aria-label={`Select run for ${row.original.sourceId}`}
checked={row.getIsSelected()}
disabled={!row.getCanSelect()}
onCheckedChange={(checked) => row.toggleSelected(checked)}
/>
),
enableHiding: false,
enableSorting: false,
header: ({ table }) => (
<Checkbox
aria-label="Select all runs on this page"
checked={table.getIsAllPageRowsSelected()}
indeterminate={table.getIsSomePageRowsSelected()}
onCheckedChange={(checked) => table.toggleAllPageRowsSelected(checked)}
/>
),
id: "select",
},
{
accessorKey: "sourceId",
cell: ({ row }) => (
@@ -20,6 +20,7 @@ import { useTRPC } from "#dashboard/app/trpc/client";
import { buildIngestionRunsQuery, resolveIngestionRunStates } from "../ingestion-runs-query";
import type { IngestionRun } from "../types";
import { IngestionRunsBulkActions } from "./ingestion-runs-bulk-actions";
import { createIngestionRunColumns } from "./ingestion-runs-columns";
const DEFAULT_TABLE_ID = "operations.recent-runs";
@@ -96,7 +97,15 @@ export function IngestionRunsTable({
filters={(currentTable) => <RunStateFilter table={currentTable} />}
store={tableStore}
table={table}
/>
>
<IngestionRunsBulkActions
runIds={table
.getRowModel()
.rows.filter((row) => tableStore.rowSelection[row.id])
.map((row) => row.id)}
table={table}
/>
</DataTableToolbar>
)}
/>
</>
-22
View File
@@ -24,7 +24,6 @@
"dependencies": {
"@basango/db": "workspace:*",
"@basango/domain": "workspace:*",
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"@better-auth/drizzle-adapter": "^1.7.1",
"@hono/node-server": "^1.19.6",
@@ -91,7 +90,6 @@
"@ai-sdk/google": "^2.0.44",
"@ai-sdk/openai": "^2.0.75",
"@basango/domain": "workspace:*",
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"@date-fns/utc": "^2.1.1",
"ai": "^5.0.105",
@@ -119,16 +117,6 @@
"@basango/tsconfig": "workspace:*",
},
},
"packages/encryption": {
"name": "@basango/encryption",
"dependencies": {
"@basango/domain": "workspace:*",
"bcrypt": "^6.0.0",
},
"devDependencies": {
"@types/bcrypt": "^6.0.0",
},
},
"packages/logger": {
"name": "@basango/logger",
"dependencies": {
@@ -273,8 +261,6 @@
"@basango/domain": ["@basango/domain@workspace:packages/domain"],
"@basango/encryption": ["@basango/encryption@workspace:packages/encryption"],
"@basango/logger": ["@basango/logger@workspace:packages/logger"],
"@basango/tsconfig": ["@basango/tsconfig@workspace:packages/tsconfig"],
@@ -765,8 +751,6 @@
"@turbo/windows-arm64": ["@turbo/windows-arm64@2.10.11", "", { "os": "win32", "cpu": "arm64" }, "sha512-m8tJkIrTrbQ9O1uHxV0GUq623Zg1678xGna8eMsy4KbygUX/wVFgdZDYV/AxyoyaW/U7nyKoBvaDVwm22p9xqA=="],
"@types/bcrypt": ["@types/bcrypt@6.0.0", "", { "dependencies": { "@types/node": "*" } }, "sha512-/oJGukuH3D2+D+3H4JWLaAsJ/ji86dhRidzZ/Od7H/i8g+aCmvkeCc6Ni/f9uxGLSQVCRZkX2/lqEFG2BvWtlQ=="],
"@types/bun": ["@types/bun@1.4.0", "", { "dependencies": { "bun-types": "1.4.0" } }, "sha512-K+lZULY23vRgK/CfTjFIV+tyifaNdSMlPh9j+6mQ/cLfpOznLyAuzgV/JQysyECpkBQLVMSyvjlr2fBUSA9wFQ=="],
"@types/chai": ["@types/chai@5.2.3", "", { "dependencies": { "@types/deep-eql": "*", "assertion-error": "^2.0.1" } }, "sha512-Mw558oeA9fFbv65/y4mHtXDs9bPnFMZAL/jxdPFUpOHHIXX91mcgEHbS5Lahr+pwZFR8A7GQleRWeI6cGFC2UA=="],
@@ -869,8 +853,6 @@
"baseline-browser-mapping": ["baseline-browser-mapping@2.11.18", "", { "bin": { "baseline-browser-mapping": "dist/cli.cjs" } }, "sha512-1iEmLEYSiE1SeBoAfPo/Mnx3PzfzHUkDK61ASkCpuk3YXugYLH5DYK1SzqV55F8FMI6s0F+/tCP7Polz1QRjxw=="],
"bcrypt": ["bcrypt@6.0.0", "", { "dependencies": { "node-addon-api": "^8.3.0", "node-gyp-build": "^4.8.4" } }, "sha512-cU8v/EGSrnH+HnxV2z0J7/blxH8gq7Xh2JFT6Aroax7UohdmiJJlxApMxtKfuI7z68NvvVcmR78k2LbT6efhRg=="],
"better-auth": ["better-auth@1.7.1", "", { "dependencies": { "@better-auth/core": "1.7.1", "@better-auth/drizzle-adapter": "1.7.1", "@better-auth/kysely-adapter": "1.7.1", "@better-auth/memory-adapter": "1.7.1", "@better-auth/mongo-adapter": "1.7.1", "@better-auth/prisma-adapter": "1.7.1", "@better-auth/telemetry": "1.7.1", "@better-auth/utils": "0.4.2", "@better-fetch/fetch": "1.3.1", "@noble/ciphers": "^2.2.0", "@noble/hashes": "^2.2.0", "better-call": "1.4.0", "defu": "^6.1.4", "jose": "^6.2.3", "kysely": "^0.28.17 || ^0.29.0", "nanostores": "^1.3.0", "zod": "^4.3.6" }, "peerDependencies": { "@lynx-js/react": "*", "@prisma/client": "^5.0.0 || ^6.0.0 || ^7.0.0", "@sveltejs/kit": "^2.0.0", "@tanstack/react-start": "^1.0.0", "@tanstack/solid-start": "^1.0.0", "better-sqlite3": "^12.0.0", "drizzle-kit": ">=0.31.4 || >=1.0.0-beta.1", "drizzle-orm": "^0.45.2 || >=1.0.0-rc.1 <2.0.0", "mongodb": "^6.0.0 || ^7.0.0", "mysql2": "^3.0.0", "next": "^14.0.0 || ^15.0.0 || ^16.0.0", "pg": "^8.0.0", "prisma": "^5.0.0 || ^6.0.0 || ^7.0.0", "react": "^18.0.0 || ^19.0.0", "react-dom": "^18.0.0 || ^19.0.0", "solid-js": "^1.0.0", "svelte": "^4.0.0 || ^5.0.0", "vitest": "^2.0.0 || ^3.0.0 || ^4.0.0", "vue": "^3.0.0" }, "optionalPeers": ["@lynx-js/react", "@prisma/client", "@sveltejs/kit", "@tanstack/react-start", "@tanstack/solid-start", "better-sqlite3", "drizzle-kit", "drizzle-orm", "mongodb", "mysql2", "next", "pg", "prisma", "react", "react-dom", "solid-js", "svelte", "vitest", "vue"] }, "sha512-g8WlTQijxXWJjPVZfFu1+EJg9cwwHrKDmIkcYMzx8CzYA+tDxl6NI7qQbKkbgw5UtHILsT5VH+RMzFzwnVJqAg=="],
"better-call": ["better-call@1.4.0", "", { "dependencies": { "@better-auth/utils": "^0.5.0", "@better-fetch/fetch": "^1.3.1", "rou3": "^0.9.1", "set-cookie-parser": "^3.1.2" }, "peerDependencies": { "zod": "^4.0.0" }, "optionalPeers": ["zod"] }, "sha512-bBKOT4vv1kZLDgxVePdilk/Jwkn+dtRRsmi3DzHcDP+WnswyVl6dR59l2HEeP/0cB+bDoopASAesWDPIdd/zZA=="],
@@ -1427,10 +1409,6 @@
"nitro": ["nitro@3.0.260610-beta", "", { "dependencies": { "consola": "^3.4.2", "crossws": "^0.4.6", "db0": "^0.3.4", "env-runner": "^0.1.12", "h3": "2.0.1-rc.22", "hookable": "^6.1.1", "nf3": "^0.3.17", "ocache": "^0.1.5", "ofetch": "2.0.0-alpha.3", "ohash": "^2.0.11", "rolldown": "^1.1.0", "srvx": "^0.11.16", "unenv": "2.0.0-rc.24", "unstorage": "2.0.0-alpha.7" }, "peerDependencies": { "@vercel/queue": "^0.3.0", "dotenv": "*", "giget": "*", "jiti": "^2.7.0", "rollup": "^4.61.1", "vite": "^7 || ^8", "xml2js": "^0.6.2", "zephyr-agent": "^0.2.0" }, "optionalPeers": ["@vercel/queue", "dotenv", "giget", "jiti", "rollup", "vite", "xml2js", "zephyr-agent"], "bin": { "nitro": "dist/cli/index.mjs" } }, "sha512-KPb4L5yaF/Rx/xoGMpgHRJvZhbhGiqbRKOwwPLCH9jKTKTsEUHLjnJas85AeCzaswqa8Wi52eQBtRsODC4PS0Q=="],
"node-addon-api": ["node-addon-api@8.9.2", "", {}, "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg=="],
"node-gyp-build": ["node-gyp-build@4.8.4", "", { "bin": { "node-gyp-build": "bin.js", "node-gyp-build-optional": "optional.js", "node-gyp-build-test": "build-test.js" } }, "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ=="],
"node-releases": ["node-releases@2.0.53", "", {}, "sha512-D9UOmYG3UH1V+ENW56t5QXBwJw1YEY18ruVeus89Rw+SyIgjPkCO84bRzO3uNIYosJbNwiabWVn48o3uJLjxFQ=="],
"normalize-path": ["normalize-path@3.0.0", "", {}, "sha512-6eZs5Ls3WtCisHWp9S2GUy8dqkpGi4BVSz3GaqiE6ezub0512ESztXUwUB6C6IKbQkY2Pnb/mD4WYojCRwcwLA=="],
+1 -1
View File
@@ -40,7 +40,7 @@ their owning applications. Shared packages own stable, reusable contracts or inf
home for new application behavior.
The dashboard consumes the API through its app-owned tRPC client and imports router types only from the API's explicit
public export. Client applications never import persistence, logging, or encryption packages. The React Native
public export. Client applications never import persistence or logging packages. The React Native
application may share platform-neutral domain contracts, but it must not consume the DOM-based `@basango/ui` package.
See the [TypeScript application architecture](web/README.md) for the dependency graph and package recommendations, and
+1 -1
View File
@@ -69,7 +69,7 @@ BETTER_AUTH_URL=https://api.basango.ngandu.dev
BETTER_AUTH_COOKIE_DOMAIN=.basango.ngandu.dev
```
Put the API database, encryption key, crawler token, and Better Auth secret in root `.env.local`.
Put the API database, crawler token, and Better Auth secret in root `.env.local`.
That file is loaded after `.env.prod`, so it is the final file-based override and stays outside Git.
Variables injected directly by PM2 or the host still take precedence. See the
[environment configuration guide](environment.md) for the complete convention.
-1
View File
@@ -53,7 +53,6 @@ BETTER_AUTH_COOKIE_DOMAIN=.basango.ngandu.dev
# .env.local
BASANGO_API_CRAWLER_TOKEN=replace-with-a-long-random-secret
BASANGO_DATABASE_URL=postgresql://...
BASANGO_ENCRYPTION_KEY=replace-with-a-64-character-key
BETTER_AUTH_SECRET=replace-with-a-long-random-secret
```
+2 -4
View File
@@ -32,7 +32,6 @@ must not be added under `apps/` or `packages/`.
- `@basango/db` owns Drizzle schemas, PostgreSQL access, queries, and persistence services. Client applications must
not import it.
- `@basango/logger` owns structured server logging.
- `@basango/encryption` owns server-side encryption and hashing behavior.
- `@basango/tsconfig` owns shared TypeScript compiler defaults.
A shared package is justified when it has at least two real consumers or one clear package-owned responsibility. A
@@ -47,8 +46,7 @@ Each arrow points from the importer to the dependency:
dashboard -> api (exported tRPC types only), domain, ui
mobile -> domain (platform-neutral contracts only)
api -> db, domain, logger
db -> domain, encryption, logger
encryption -> domain
db -> domain, logger
logger -> domain
ui -> third-party UI libraries only
domain -> platform-neutral libraries only
@@ -57,7 +55,7 @@ domain -> platform-neutral libraries only
The following constraints preserve those boundaries:
- no package imports from an application;
- no client application imports `@basango/db`, `@basango/logger`, or `@basango/encryption`;
- no client application imports `@basango/db` or `@basango/logger`;
- dashboard imports API router types only through the explicit `@basango/api/trpc/routers/_app` export;
- mobile does not import `@basango/ui` or browser-only modules;
- packages do not re-export another package's domain symbols as their own;
+2 -2
View File
@@ -60,8 +60,8 @@
"prepare": "husky",
"start:api": "turbo start --filter=@basango/api",
"start:dashboard": "turbo start --filter=@basango/dashboard",
"test": "turbo run test --parallel",
"typecheck": "turbo run typecheck"
"test": "bun test tests",
"typecheck": "turbo run typecheck && tsc --noEmit --project tests/tsconfig.json"
},
"workspaces": [
"packages/*",
-1
View File
@@ -3,7 +3,6 @@
"@ai-sdk/google": "^2.0.44",
"@ai-sdk/openai": "^2.0.75",
"@basango/domain": "workspace:*",
"@basango/encryption": "workspace:*",
"@basango/logger": "workspace:*",
"@date-fns/utc": "^2.1.1",
"ai": "^5.0.105",
+6 -1
View File
@@ -1,3 +1,5 @@
import { createHash } from "node:crypto";
import { DEFAULT_TIMEZONE } from "@basango/domain/constants";
import {
Article,
@@ -8,7 +10,6 @@ import {
Publications,
Sentiment,
} from "@basango/domain/models";
import { md5 } from "@basango/encryption";
import type { SQL } from "drizzle-orm";
import { count, desc, eq, getTableColumns, sql } from "drizzle-orm";
import * as uuid from "uuid";
@@ -248,3 +249,7 @@ export async function getArticlesSourceDistribution(
total: data.rows.reduce((acc, item) => acc + item.count, 0),
};
}
function md5(value: string): string {
return createHash("md5").update(value).digest("hex");
}
+1 -1
View File
@@ -1,5 +1,5 @@
export { getIngestionAgents } from "./agents";
export { listIngestionRuns } from "./runs";
export { closeIngestionRuns, listIngestionRuns } from "./runs";
export { applyIngestionSignal } from "./signals";
export { getIngestionSummary } from "./summary";
export { getIngestionThroughput } from "./throughput";
+37 -3
View File
@@ -1,12 +1,13 @@
import type { IngestionRunsQuery } from "@basango/domain/models";
import type { CloseIngestionRuns, IngestionRunsQuery } from "@basango/domain/models";
import type { SQL } from "drizzle-orm";
import { and, asc, count, desc, eq, ilike, inArray, or } from "drizzle-orm";
import { and, asc, count, desc, eq, ilike, inArray, ne, or, sql } from "drizzle-orm";
import type { Database } from "#db/client";
import { ingestionRuns } from "#db/schema";
import { ingestionAgents, ingestionRuns } from "#db/schema";
const DEFAULT_PAGE = 1;
const DEFAULT_PAGE_SIZE = 10;
const MANUAL_FAILURE_MESSAGE = "Manually marked as failed from the dashboard.";
const sortColumns = {
agentId: ingestionRuns.agentId,
@@ -55,6 +56,39 @@ export async function listIngestionRuns(db: Database, params: IngestionRunsQuery
};
}
export async function closeIngestionRuns(db: Database, params: CloseIngestionRuns) {
const completedAt = new Date();
return db.transaction(async (tx) => {
const releasedAgents = await tx
.update(ingestionAgents)
.set({ activeRunId: null, state: "idle" })
.where(inArray(ingestionAgents.activeRunId, params.runIds))
.returning({ id: ingestionAgents.id });
const updatedRuns = await tx
.update(ingestionRuns)
.set({
completedAt: sql`COALESCE(${ingestionRuns.completedAt}, ${completedAt})`,
durationMs: sql`COALESCE(${ingestionRuns.durationMs}, GREATEST(0, FLOOR(EXTRACT(EPOCH FROM (${completedAt} - COALESCE(${ingestionRuns.startedAt}, ${ingestionRuns.createdAt}))) * 1000)::bigint))`,
...(params.state === "failed"
? { error: sql`COALESCE(${ingestionRuns.error}, ${MANUAL_FAILURE_MESSAGE})` }
: { error: null }),
lastSignalAt: completedAt,
state: params.state,
})
.where(and(inArray(ingestionRuns.id, params.runIds), ne(ingestionRuns.state, params.state)))
.returning({ id: ingestionRuns.id });
const updatedRunIds = updatedRuns.map((run) => run.id);
return {
releasedAgentCount: releasedAgents.length,
runIds: updatedRunIds,
unchangedCount: params.runIds.length - updatedRunIds.length,
updatedCount: updatedRunIds.length,
};
});
}
function buildRunsFilter(params: IngestionRunsQuery): SQL | undefined {
const conditions: SQL[] = [];
const query = params.filters?.query;
-9
View File
@@ -1,9 +0,0 @@
{
"encryption": {
"algorithm": "aes-256-gcm",
"authTagLength": 16,
"bcryptSaltRounds": 12,
"ivLength": 16,
"key": "%env(BASANGO_ENCRYPTION_KEY)%"
}
}
-1
View File
@@ -20,7 +20,6 @@
"private": true,
"scripts": {
"clean": "rm -rf .turbo node_modules",
"test": "bun test src",
"typecheck": "tsc --noEmit"
},
"type": "module"
-18
View File
@@ -1,18 +0,0 @@
import z from "zod";
import {
DEFAULT_AUTH_TAG_LENGTH,
DEFAULT_BCRYPT_SALT_ROUNDS,
DEFAULT_IV_LENGTH,
} from "../constants";
export const EncryptionConfigurationSchema = z.object({
algorithm: z.enum(["aes-128-gcm", "aes-192-gcm", "aes-256-gcm"]),
authTagLength: z.number().nonnegative().default(DEFAULT_AUTH_TAG_LENGTH),
bcryptSaltRounds: z.number().nonnegative().default(DEFAULT_BCRYPT_SALT_ROUNDS),
ivLength: z.number().nonnegative().default(DEFAULT_IV_LENGTH),
key: z.string(),
});
// types
export type EncryptionConfiguration = z.infer<typeof EncryptionConfigurationSchema>;
-6
View File
@@ -5,14 +5,12 @@ import z from "zod";
import { ApiConfigurationSchema } from "./api";
import { DatabaseConfigurationSchema } from "./database";
import { EncryptionConfigurationSchema } from "./encryption";
import { resolveEnvFiles } from "./environment";
import { LoggerConfigurationSchema } from "./logger";
import { SharedConfigurationSchema } from "./shared";
export * from "./api";
export * from "./database";
export * from "./encryption";
export * from "./logger";
export * from "./shared";
@@ -27,7 +25,6 @@ export const { env, config } = await defineConfig({
"BASANGO_API_CRAWLER_TOKEN",
"BASANGO_API_KEY",
"BASANGO_DATABASE_URL",
"BASANGO_ENCRYPTION_KEY",
"BASANGO_RESEND_API_KEY",
"BETTER_AUTH_SECRET",
],
@@ -57,7 +54,6 @@ export const { env, config } = await defineConfig({
BASANGO_CRAWLER_SQLITE_PATH: z.string().optional(),
BASANGO_CRAWLER_UPDATE_DIRECTION: z.string().optional(),
BASANGO_DATABASE_URL: z.string().min(1),
BASANGO_ENCRYPTION_KEY: z.string().min(1),
BASANGO_ENV_PATH: z.string().optional(),
BASANGO_LOGGER_LEVEL: z.string().default("info"),
BASANGO_LOGGER_PRETTY: z.string().optional(),
@@ -72,14 +68,12 @@ export const { env, config } = await defineConfig({
schema: z.object({
api: ApiConfigurationSchema,
database: DatabaseConfigurationSchema,
encryption: EncryptionConfigurationSchema,
logger: LoggerConfigurationSchema,
shared: SharedConfigurationSchema,
}),
sources: [
jsonFile("config/api.json", { name: "api" }),
jsonFile("config/database.json", { name: "database" }),
jsonFile("config/encryption.json", { name: "encryption" }),
jsonFile("config/logger.json", { name: "logger" }),
jsonFile("config/shared.json", { name: "shared" }),
],
-3
View File
@@ -11,7 +11,4 @@ export const DEFAULT_PUBLICATION_GRAPH_DAYS = 30;
export const DEFAULT_CATEGORY_SHARES_LIMIT = 10;
export const DEFAULT_TIMEZONE = "Africa/Lubumbashi";
export const DEFAULT_IV_LENGTH = 16;
export const DEFAULT_AUTH_TAG_LENGTH = 16;
export const DEFAULT_BCRYPT_SALT_ROUNDS = 12;
export const DEFAULT_CATEGORY = "divers-autres";
+12
View File
@@ -2,6 +2,8 @@ import z from "zod";
export const INGESTION_RUN_STATES = ["preparing", "running", "completed", "failed"] as const;
export const INGESTION_RUN_TERMINAL_STATES = ["completed", "failed"] as const;
export const INGESTION_CHANGE_TOPICS = ["agents", "runs", "summary", "throughput"] as const;
export const INGESTION_RUN_SORT_FIELDS = [
@@ -20,6 +22,15 @@ export const INGESTION_RUN_SORT_FIELDS = [
export const ingestionRunStateSchema = z.enum(INGESTION_RUN_STATES);
export const closeIngestionRunsSchema = z.object({
runIds: z
.array(z.uuid())
.min(1)
.max(100)
.refine((runIds) => new Set(runIds).size === runIds.length, "Run IDs must be unique."),
state: z.enum(INGESTION_RUN_TERMINAL_STATES),
});
export const ingestionRunsQuerySchema = z.object({
filters: z
.object({
@@ -105,6 +116,7 @@ export const ingestionChangeSchema = z.object({
export type IngestionChange = z.infer<typeof ingestionChangeSchema>;
export type IngestionChangeTopic = (typeof INGESTION_CHANGE_TOPICS)[number];
export type CloseIngestionRuns = z.infer<typeof closeIngestionRunsSchema>;
export type IngestionRunMetrics = z.infer<typeof ingestionRunMetricsSchema>;
export type IngestionRunsQuery = z.infer<typeof ingestionRunsQuerySchema>;
export type IngestionRunState = z.infer<typeof ingestionRunStateSchema>;
-16
View File
@@ -1,16 +0,0 @@
{
"dependencies": {
"@basango/domain": "workspace:*",
"bcrypt": "^6.0.0"
},
"devDependencies": {
"@types/bcrypt": "^6.0.0"
},
"main": "src/index.ts",
"name": "@basango/encryption",
"private": true,
"scripts": {
"clean": "rm -rf .turbo node_modules",
"typecheck": "tsc --noEmit"
}
}
-95
View File
@@ -1,95 +0,0 @@
import crypto from "node:crypto";
import { config } from "@basango/domain/config";
import type * as Bcrypt from "bcrypt";
const loadBcrypt = async (): Promise<typeof Bcrypt> => {
const moduleName = "bcrypt";
return await import(moduleName);
};
function getKey(): Buffer {
const key = config.encryption.key;
if (Buffer.from(key, "hex").length !== 32) {
throw new Error("BASANGO_ENCRYPTION_KEY must be a 64-character hex string (32 bytes).");
}
return Buffer.from(key, "hex");
}
const getEncryptionSettings = () => ({
algorithm: config.encryption.algorithm as crypto.CipherGCMTypes,
authTagLength: config.encryption.authTagLength,
ivLength: config.encryption.ivLength,
});
/**
* Encrypts a plaintext string using AES-256-GCM.
* @param text The plaintext string to encrypt.
* @returns A string containing the IV, auth tag, and encrypted text, concatenated and base64 encoded.
*/
export function encrypt(text: string): string {
const key = getKey();
const { algorithm, ivLength } = getEncryptionSettings();
const iv = crypto.randomBytes(ivLength);
const cipher = crypto.createCipheriv(algorithm, key, iv);
let encrypted = cipher.update(text, "utf8", "hex");
encrypted += cipher.final("hex");
const authTag = cipher.getAuthTag();
// Concatenate IV, auth tag, and encrypted data
const encryptedPayload = Buffer.concat([iv, authTag, Buffer.from(encrypted, "hex")]).toString(
"base64",
);
return encryptedPayload;
}
/**
* Decrypts an AES-256-GCM encrypted string.
* @param encryptedPayload The base64 encoded string containing the IV, auth tag, and encrypted text.
* @returns The original plaintext string.
*/
export function decrypt(encryptedPayload: string): string {
const key = getKey();
const { algorithm, authTagLength, ivLength } = getEncryptionSettings();
const dataBuffer = Buffer.from(encryptedPayload, "base64");
// Extract IV, auth tag, and encrypted data
const iv = dataBuffer.subarray(0, ivLength);
const authTag = dataBuffer.subarray(ivLength, ivLength + authTagLength);
const encryptedText = dataBuffer.subarray(ivLength + authTagLength);
const decipher = crypto.createDecipheriv(algorithm, key, iv);
decipher.setAuthTag(authTag);
let decrypted = decipher.update(encryptedText.toString("hex"), "hex", "utf8");
decrypted += decipher.final("utf8");
return decrypted;
}
export function hash(str: string): string {
return crypto.createHash("sha256").update(str).digest("hex");
}
export function md5(str: string): string {
return crypto.createHash("md5").update(str).digest("hex");
}
export function generateRandomBytes(size: number): string {
return crypto.randomBytes(size).toString("hex");
}
export async function hashPassword(password: string): Promise<string> {
const rounds = config.encryption.bcryptSaltRounds;
const bcrypt = await loadBcrypt();
return bcrypt.hash(password, rounds);
}
export async function verifyPassword(password: string, hashed: string): Promise<boolean> {
const bcrypt = await loadBcrypt();
return bcrypt.compare(password, hashed);
}
-11
View File
@@ -1,11 +0,0 @@
{
"compilerOptions": {
"paths": {
"#domain/*": ["../../domain/src/*"],
"#encryption/*": ["./src/*"]
}
},
"exclude": ["node_modules"],
"extends": "@basango/tsconfig/base.json",
"include": ["src/**/*"]
}
@@ -12,7 +12,11 @@ export type DataTableToolbarProps<TData> = {
filters?: (table: Table<TData>) => ReactNode;
store: Pick<
TableStore,
"globalFilter" | "globalFilterInput" | "setGlobalFilter" | "setGlobalFilterInput"
| "globalFilter"
| "globalFilterInput"
| "rowSelection"
| "setGlobalFilter"
| "setGlobalFilterInput"
>;
table: Table<TData>;
};
@@ -1,6 +1,6 @@
import { describe, expect, test } from "bun:test";
import { getIngestionChangeTopics } from "./signals";
import { getIngestionChangeTopics } from "../../../../apps/api/src/services/ingestion/signals";
describe("ingestion change topics", () => {
test("limits heartbeats to agent state", () => {
@@ -1,11 +1,11 @@
import { describe, expect, test } from "bun:test";
import type { IngestionRun } from "../runs/types";
import {
createTimelineWindow,
getRunDurationMs,
getTimelineBarBounds,
} from "./crawl-history-model";
} from "../../../../../apps/dashboard/src/features/ingestion/history/crawl-history-model";
import type { IngestionRun } from "../../../../../apps/dashboard/src/features/ingestion/runs/types";
describe("crawl history model", () => {
test("uses signal timestamps when a run has no final duration", () => {
@@ -1,11 +1,11 @@
import { describe, expect, test } from "bun:test";
import type { CrawlTimelineWindow } from "./crawl-history-model";
import type { CrawlTimelineWindow } from "../../../../../apps/dashboard/src/features/ingestion/history/crawl-history-model";
import {
createCrawlTimelineTicks,
expandCrawlTimelineWindow,
getCrawlTimelineWidth,
} from "./crawl-timeline-scale";
} from "../../../../../apps/dashboard/src/features/ingestion/history/crawl-timeline-scale";
const HOUR_MS = 60 * 60 * 1_000;
const referenceMs = new Date("2026-08-26T12:00:00.000Z").getTime();
@@ -3,7 +3,11 @@ import fs from "node:fs";
import os from "node:os";
import path from "node:path";
import { normalizeNodeEnvironment, readEnvFiles, resolveEnvFiles } from "./environment";
import {
normalizeNodeEnvironment,
readEnvFiles,
resolveEnvFiles,
} from "../../../packages/domain/src/config/environment";
const temporaryDirectories: string[] = [];
@@ -1,6 +1,10 @@
import { describe, expect, test } from "bun:test";
import { articleHashSchema, createArticleResponseSchema, getArticlesSchema } from "./articles";
import {
articleHashSchema,
createArticleResponseSchema,
getArticlesSchema,
} from "../../../packages/domain/src/models/articles";
describe("article ingestion contracts", () => {
test("accepts the crawler's lowercase MD5 article identity", () => {
@@ -1,6 +1,10 @@
import { describe, expect, test } from "bun:test";
import { loginSchema, passwordSchema, requestPasswordResetSchema } from "./auth";
import {
loginSchema,
passwordSchema,
requestPasswordResetSchema,
} from "../../../packages/domain/src/models/auth";
describe("authentication schemas", () => {
test("normalizes login and reset email addresses", () => {
@@ -1,10 +1,11 @@
import { describe, expect, test } from "bun:test";
import {
closeIngestionRunsSchema,
ingestionChangeSchema,
ingestionRunsQuerySchema,
ingestionSignalSchema,
} from "./ingestion";
} from "../../../packages/domain/src/models/ingestion";
const envelope = {
agentId: "crawler-lubumbashi-01",
@@ -112,6 +113,34 @@ describe("ingestion runs query", () => {
});
});
describe("manual ingestion run closure", () => {
test("accepts multiple runs with a terminal state", () => {
const closure = closeIngestionRunsSchema.parse({
runIds: ["0198d7e4-df8c-7000-8000-000000000002", "0198d7e4-df8c-7000-8000-000000000003"],
state: "completed",
});
expect(closure.runIds).toHaveLength(2);
expect(closure.state).toBe("completed");
});
test("rejects empty batches and active target states", () => {
expect(closeIngestionRunsSchema.safeParse({ runIds: [], state: "failed" }).success).toBe(false);
expect(
closeIngestionRunsSchema.safeParse({
runIds: ["0198d7e4-df8c-7000-8000-000000000002"],
state: "running",
}).success,
).toBe(false);
expect(
closeIngestionRunsSchema.safeParse({
runIds: ["0198d7e4-df8c-7000-8000-000000000002", "0198d7e4-df8c-7000-8000-000000000002"],
state: "completed",
}).success,
).toBe(false);
});
});
describe("ingestion realtime changes", () => {
test("parses a selective change notification", () => {
const change = ingestionChangeSchema.parse({
@@ -1,6 +1,9 @@
import { describe, expect, test } from "bun:test";
import { getSourcePublicationBoundsResponseSchema, getSourcesSchema } from "./sources";
import {
getSourcePublicationBoundsResponseSchema,
getSourcesSchema,
} from "../../../packages/domain/src/models/sources";
describe("source list contracts", () => {
test("accepts offset pagination", () => {
+14
View File
@@ -0,0 +1,14 @@
{
"compilerOptions": {
"noEmit": true,
"paths": {
"#api/*": ["../apps/api/src/*"],
"#dashboard/*": ["../apps/dashboard/src/*"],
"#db/*": ["../packages/db/src/*"],
"#domain/*": ["../packages/domain/src/*"]
},
"types": ["bun"]
},
"extends": "../packages/tsconfig/base.json",
"include": ["**/*.ts"]
}
-3
View File
@@ -22,9 +22,6 @@
"start": {
"cache": false
},
"test": {
"cache": false
},
"topo": {
"dependsOn": ["^topo"]
},