refactor: centralize configuration

2025-11-23 19:54:32 +02:00
parent 57a8501c88
commit 72dfa53f80
78 changed files with 2252 additions and 1385 deletions
@@ -0,0 +1,29 @@
+import z from "zod";
+
+export const ApiConfigurationSchema = z.object({
+  cors: z.object({
+    allowedHeaders: z.array(z.string()).default([]),
+    allowMethods: z.array(z.string()).default([]),
+    exposeHeaders: z.array(z.string()).default([]),
+    maxAge: z.number().int().min(0).optional(),
+    origin: z
+      .array(z.string())
+      .optional()
+      .default(["http://localhost:3000", "http://127.0.0.1:3000", "https://dashboard.basango.io"]),
+  }),
+  security: z.object({
+    accessTokenTtl: z.string(),
+    audience: z.string(),
+    crawlerToken: z.string(),
+    issuer: z.string(),
+    jwtSecret: z.string(),
+    refreshTokenTtl: z.string(),
+  }),
+  server: z.object({
+    host: z.string().default("localhost"),
+    port: z.number().int().min(1).max(65535).default(3080),
+    version: z.string().default("1.0.0"),
+  }),
+});
+
+export type ApiConfiguration = z.infer<typeof ApiConfigurationSchema>;
@@ -0,0 +1,107 @@
+import { z } from "zod";
+
+import { SOURCE_KINDS } from "../constants";
+import { PageRangeSchema, TimestampRangeSchema, UpdateDirectionSchema } from "../models";
+
+export const SourceKindSchema = z.enum(SOURCE_KINDS);
+
+export const SourceDateSchema = z.object({
+  format: z.string().default("yyyy-LL-dd HH:mm"),
+});
+
+const SourceOptionsSchema = z.object({
+  categories: z.array(z.string()).default([]),
+  requiresDetails: z.boolean().default(false),
+  requiresRateLimit: z.boolean().default(false),
+  sourceDate: SourceDateSchema,
+  sourceId: z.string(),
+  sourceKind: SourceKindSchema,
+  sourceUrl: z.url(),
+  supportsCategories: z.boolean().default(false),
+});
+
+export const HtmlSourceOptionsSchema = SourceOptionsSchema.extend({
+  paginationTemplate: z.string(),
+  sourceKind: z.literal("html"),
+  sourceSelectors: z.object({
+    articleBody: z.string(),
+    articleCategories: z.string().optional(),
+    articleDate: z.string(),
+    articleLink: z.string(),
+    articles: z.string(),
+    articleTitle: z.string(),
+    pagination: z.string().default("ul.pagination > li a"),
+  }),
+});
+
+export const WordPressSourceOptionsSchema = SourceOptionsSchema.extend({
+  sourceDate: SourceDateSchema.default(SourceDateSchema.parse({ format: "yyyy-LL-dd'T'HH:mm:ss" })),
+  sourceKind: z.literal("wordpress"),
+});
+
+export const CrawlerConfigurationSchema = z.object({
+  backend: z.object({
+    endpoint: z.url(),
+    token: z.string(),
+  }),
+  fetch: z.object({
+    async: z.object({
+      prefix: z.string().default("basango:crawler:queue"),
+      queues: z.object({
+        details: z.string().default("details"),
+        listing: z.string().default("listing"),
+        processing: z.string().default("processing"),
+      }),
+      redisUrl: z.string().default("redis://localhost:6379/0"),
+      ttl: z.object({
+        default: z.number().int().positive().default(600),
+        failure: z.number().int().nonnegative().default(3600),
+        result: z.number().int().nonnegative().default(3600),
+      }),
+    }),
+    client: z.object({
+      backoffInitial: z.number().nonnegative().default(1),
+      backoffMax: z.number().nonnegative().default(30),
+      backoffMultiplier: z.number().positive().default(2),
+      followRedirects: z.boolean().default(true),
+      maxRetries: z.number().int().nonnegative().default(3),
+      respectRetryAfter: z.boolean().default(true),
+      rotate: z.boolean().default(true),
+      timeout: z.number().positive().default(20),
+      userAgent: z.string().default("Basango/0.1 (+https://github.com/bernard-ng/basango)"),
+      verifySsl: z.boolean().default(true),
+    }),
+    crawler: z.object({
+      category: z.string().optional(),
+      dateRange: TimestampRangeSchema.optional(),
+      direction: UpdateDirectionSchema.default("forward"),
+      isUpdate: z.boolean().default(false),
+      maxWorkers: z.number().int().positive().default(5),
+      notify: z.boolean().default(false),
+      pageRange: PageRangeSchema.optional(),
+      source: z.union([HtmlSourceOptionsSchema, WordPressSourceOptionsSchema]).optional(),
+      useMultiThreading: z.boolean().default(false),
+    }),
+  }),
+  paths: z.object({
+    data: z.string(),
+    root: z.string(),
+  }),
+  sources: z.object({
+    html: z.array(HtmlSourceOptionsSchema).default([]),
+    wordpress: z.array(WordPressSourceOptionsSchema).default([]),
+  }),
+});
+
+// types
+export type SourceKind = z.infer<typeof SourceKindSchema>;
+export type SourceDate = z.infer<typeof SourceDateSchema>;
+export type HtmlSourceOptions = z.infer<typeof HtmlSourceOptionsSchema>;
+export type WordPressSourceOptions = z.infer<typeof WordPressSourceOptionsSchema>;
+export type AnySourceOptions = HtmlSourceOptions | WordPressSourceOptions;
+
+export type CrawlerConfiguration = z.infer<typeof CrawlerConfigurationSchema>;
+export type CrawlerHttpOptions = CrawlerConfiguration["fetch"]["client"];
+export type CrawlerFetchingOptions = CrawlerConfiguration["fetch"]["crawler"];
+export type CrawlerAsyncOptions = CrawlerConfiguration["fetch"]["async"];
+export type CrawlerBackendOptions = CrawlerConfiguration["backend"];
@@ -0,0 +1,15 @@
+import z from "zod";
+
+export const DatabaseConfigurationSchema = z.object({
+  legacy: z.object({
+    host: z.string().min(1),
+    name: z.string().min(1),
+    password: z.string().min(1),
+    port: z.number().optional(),
+    user: z.string().min(1),
+  }),
+  url: z.string().min(1),
+});
+
+// types
+export type DatabaseConfiguration = z.infer<typeof DatabaseConfigurationSchema>;
@@ -0,0 +1,18 @@
+import z from "zod";
+
+import {
+  DEFAULT_AUTH_TAG_LENGTH,
+  DEFAULT_BCRYPT_SALT_ROUNDS,
+  DEFAULT_IV_LENGTH,
+} from "../constants";
+
+export const EncryptionConfigurationSchema = z.object({
+  algorithm: z.enum(["aes-128-gcm", "aes-192-gcm", "aes-256-gcm"]),
+  authTagLength: z.number().nonnegative().default(DEFAULT_AUTH_TAG_LENGTH),
+  bcryptSaltRounds: z.number().nonnegative().default(DEFAULT_BCRYPT_SALT_ROUNDS),
+  ivLength: z.number().nonnegative().default(DEFAULT_IV_LENGTH),
+  key: z.string(),
+});
+
+// types
+export type EncryptionConfiguration = z.infer<typeof EncryptionConfigurationSchema>;
@@ -0,0 +1,72 @@
+import path from "node:path";
+
+import { defineConfig } from "@devscast/config";
+import z from "zod";
+
+import { ApiConfigurationSchema } from "./api";
+import { CrawlerConfigurationSchema } from "./crawler";
+import { DatabaseConfigurationSchema } from "./database";
+import { EncryptionConfigurationSchema } from "./encryption";
+import { LoggerConfigurationSchema } from "./logger";
+import { SharedConfigurationSchema } from "./shared";
+
+export * from "./api";
+export * from "./crawler";
+export * from "./database";
+export * from "./encryption";
+export * from "./logger";
+export * from "./shared";
+
+const root = path.resolve(__dirname, "../../../../");
+const domain = path.join(root, "packages", "domain", "config");
+
+export const { env, config } = defineConfig({
+  env: {
+    knownKeys: [
+      "NODE_ENV",
+      "BASANGO_API_HOST",
+      "BASANGO_API_PORT",
+      "BASANGO_API_ALLOWED_ORIGINS",
+      "BASANGO_API_KEY",
+      "BASANGO_API_CRAWLER_TOKEN",
+      "BASANGO_API_JWT_SECRET",
+      "BASANGO_DATABASE_URL",
+      "BASANGO_DATABASE_LEGACY_HOST",
+      "BASANGO_DATABASE_LEGACY_PASSWORD",
+      "BASANGO_DATABASE_LEGACY_NAME",
+      "BASANGO_DATABASE_LEGACY_USER",
+      "BASANGO_CRAWLER_ROOT_PATH",
+      "BASANGO_CRAWLER_DATA_PATH",
+      "BASANGO_CRAWLER_LOGS_PATH",
+      "BASANGO_CRAWLER_CONFIG_PATH",
+      "BASANGO_CRAWLER_UPDATE_DIRECTION",
+      "BASANGO_CRAWLER_FETCH_USER_AGENT",
+      "BASANGO_CRAWLER_FETCH_MAX_RETRIES",
+      "BASANGO_CRAWLER_FETCH_RESPECT_RETRY_AFTER",
+      "BASANGO_CRAWLER_ASYNC_REDIS_URL",
+      "BASANGO_CRAWLER_ASYNC_TTL_RESULT",
+      "BASANGO_CRAWLER_ASYNC_TTL_FAILURE",
+      "BASANGO_CRAWLER_ASYNC_QUEUE_LISTING",
+      "BASANGO_CRAWLER_ASYNC_QUEUE_DETAILS",
+      "BASANGO_CRAWLER_ASYNC_QUEUE_PROCESSING",
+      "BASANGO_ENCRYPTION_KEY",
+    ] as const,
+    path: path.join(root, ".env"),
+  },
+  schema: z.object({
+    api: ApiConfigurationSchema,
+    crawler: CrawlerConfigurationSchema,
+    database: DatabaseConfigurationSchema,
+    encryption: EncryptionConfigurationSchema,
+    logger: LoggerConfigurationSchema,
+    shared: SharedConfigurationSchema,
+  }),
+  sources: [
+    path.join(domain, "api.json"),
+    path.join(domain, "crawler.json"),
+    path.join(domain, "database.json"),
+    path.join(domain, "encryption.json"),
+    path.join(domain, "logger.json"),
+    path.join(domain, "shared.json"),
+  ],
+});
@@ -0,0 +1,8 @@
+import z from "zod";
+
+export const LoggerConfigurationSchema = z.object({
+  level: z.string().default("info"),
+});
+
+// types
+export type LoggerConfiguration = z.infer<typeof LoggerConfigurationSchema>;
@@ -0,0 +1,17 @@
+import z from "zod";
+
+export const SharedConfigurationSchema = z.object({
+  categorySharesLimit: z.number().int().min(1).default(10),
+  dateFormat: z.string(),
+  dateTimeFormat: z.string(),
+  name: z.string().default("Basango"),
+  pagination: z.object({
+    defaultLimit: z.number().int().min(1).max(100),
+    maxLimit: z.number().int().min(1).max(100),
+    page: z.number().int().min(1),
+  }),
+  publicationGraphDays: z.number().int().min(1),
+  timezone: z.string(),
+});
+
+export type SharedConfiguration = z.infer<typeof SharedConfigurationSchema>;