feat: implement NER dataset feature engineering with multiple transformation formats

2025-08-12 00:11:46 +02:00
parent d5a4aaaf4a
commit 3977d5c313
9 changed files with 439 additions and 0 deletions
@@ -0,0 +1,28 @@
+from typing import Dict
+
+import pandas as pd
+
+from processing.ner.formats import BaseNameFormatter
+
+
+class NativeOnlyFormatter(BaseNameFormatter):
+    def transform(self, row: pd.Series) -> Dict:
+        native_parts = self.parse_native_components(row['probable_native'])
+
+        # Only native components
+        full_name = row['probable_native']
+
+        return {
+            'name': full_name,
+            'probable_native': row['probable_native'],
+            'identify_name': row['probable_native'],
+            'probable_surname': '',
+            'identify_surname': '',
+            'ner_entities': str(self.create_ner_tags(full_name, native_parts, '')),
+            'transformation_type': self.transformation_type,
+            **self.compute_derived_attributes(full_name)
+        }
+
+    @property
+    def transformation_type(self) -> str:
+        return 'native_only'