refactoring: add initial pipeline configuration and model classes

2025-08-04 16:12:25 +02:00
parent 19c66fd0ee
commit f4689faf80
82 changed files with 7176 additions and 1218 deletions
@@ -0,0 +1,48 @@
+# Production Environment Configuration
+# Optimized settings for production deployment
+
+name: "drc_names_pipeline"
+version: "1.0.0"
+environment: "production"
+debug: false
+
+# Production processing settings (optimized for performance)
+processing:
+  batch_size: 10_000
+  max_workers: 8
+  checkpoint_interval: 10
+  use_multiprocessing: true  # Enable multiprocessing for CPU-bound tasks
+
+# Pipeline stages
+stages:
+  - "data_cleaning"
+  - "feature_extraction"
+  - "llm_annotation"
+  - "data_splitting"
+
+
+# Production LLM settings
+llm:
+  model_name: "mistral:7b"
+  requests_per_minute: 360
+  requests_per_second: 3
+  retry_attempts: 3
+  timeout_seconds: 45
+  max_concurrent_requests: 4
+  enable_rate_limiting: true
+
+# Production data settings
+data:
+  split_evaluation: true
+  split_by_gender: true
+  evaluation_fraction: 0.2
+  random_seed: 42
+
+# Production logging (less verbose)
+logging:
+  level: "INFO"
+  console_logging: false  # Disable console in production
+  file_logging: true
+  log_file: "pipeline.production.log"
+  max_log_size: 52428800  # 50MB
+  backup_count: 10