feat: enhance logging and memory management across modules

2025-08-13 23:09:05 +02:00
parent 47e52d130c
commit 9601c5e44d
48 changed files with 1004 additions and 773 deletions
@@ -19,7 +19,13 @@ class PipelineMonitor:

        self.paths = paths
        self.checkpoint_dir = paths.checkpoints_dir
-        self.steps = ["data_cleaning", "feature_extraction", "ner_annotation", "llm_annotation", "data_splitting"]
+        self.steps = [
+            "data_cleaning",
+            "feature_extraction",
+            "ner_annotation",
+            "llm_annotation",
+            "data_splitting",
+        ]

    def get_step_status(self, step_name: str) -> Dict:
        """Get status of a specific pipeline step"""