fix: escape csv field to avoid error on empty fields

processing/ner/name_builder.py

@@ -11,6 +11,10 @@ from .name_tagger import NameTagger
 
 class NameBuilder:
     def __init__(self, config: PipelineConfig):
+        config = config.model_copy(deep=True)
+        config.data.max_dataset_size = 1_000_000
+        config.data.balance_by_sex = True
+
         self.config = config
         self.data_loader = DataLoader(config)
         self.tagger = NameTagger()
@@ -33,8 +37,8 @@ class NameBuilder:
         nlp = spacy.blank("fr")
 
         # Use NERNameTagger for parsing and validation
-        parsed_entities = NameTagger.parse_entities(ner_df["ner_entities"])
-        validated_entities = NameTagger.validate_entities(ner_df["name"], parsed_entities)
+        parsed_entities = self.tagger.parse_entities(ner_df["ner_entities"])
+        validated_entities = self.tagger.validate_entities(ner_df["name"], parsed_entities)
 
         # Drop rows with no valid entities
         mask = validated_entities.map(bool)
@@ -51,7 +55,7 @@ class NameBuilder:
         )
 
         # Use NERNameTagger to create spaCy DocBin
-        docs = NameTagger.create_docs(nlp, ner_df["name"].tolist(), validated_entities.tolist())
+        docs = self.tagger.create_docs(nlp, ner_df["name"].tolist(), validated_entities.tolist())
         doc_bin = DocBin(docs=docs)
 
         # Save

processing/ner/name_model.py

@@ -8,6 +8,7 @@ from typing import Dict, Any, List, Tuple
 import spacy
 from spacy.training import Example
 from spacy.util import minibatch
+from tqdm import tqdm
 
 from core.config.pipeline_config import PipelineConfig
 
@@ -198,13 +199,10 @@ class NameModel:
 
             # Create training examples
             examples = []
-            for text, annotations in data:
+            for text, annotations in tqdm(data, description="Create training examples"):
                 doc = self.nlp.make_doc(text)
                 example = Example.from_dict(doc, annotations)
                 examples.append(example)
-                logging.info(
-                    f"Training example: {text[:30]} with entities {annotations.get('entities', [])}"
-                )
 
             # Train in batches
             batches = minibatch(examples, size=batch_size)