bernard-ng/drc-ners-nlp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: enhance logging and memory management across modules

Browse Source
This commit is contained in:
bernard-ng
2025-08-13 23:09:05 +02:00
parent 47e52d130c
commit 9601c5e44d
48 changed files with 1004 additions and 773 deletions
Download Patch File Download Diff File Expand all files Collapse all files
research/models/lightgbm_model.py
+10 -6
View File
@@ -50,14 +50,18 @@ class LightGBMModel(TraditionalModel):
self.vectorizers[feature_key] = CountVectorizer(
analyzer="char", ngram_range=(2, 3), max_features=50
)
char_features = self.vectorizers[feature_key].fit_transform(
column.fillna("").astype(str)
).toarray()
char_features = (
self.vectorizers[feature_key]
.fit_transform(column.fillna("").astype(str))
.toarray()
)
else:
# Subsequent times - use existing vectorizer
char_features = self.vectorizers[feature_key].transform(
column.fillna("").astype(str)
).toarray()
char_features = (
self.vectorizers[feature_key]
.transform(column.fillna("").astype(str))
.toarray()
)
features.append(char_features)
else:
research/models/logistic_regression_model.py
+1 -3
View File
@@ -20,9 +20,7 @@ class LogisticRegressionModel(TraditionalModel):
)
classifier = LogisticRegression(
max_iter=params.get("max_iter", 1000),
random_state=self.config.random_seed,
verbose=2
max_iter=params.get("max_iter", 1000), random_state=self.config.random_seed, verbose=2
)
return Pipeline([("vectorizer", vectorizer), ("classifier", classifier)])
research/models/random_forest_model.py
+1 -1
View File
@@ -18,7 +18,7 @@ class RandomForestModel(TraditionalModel):
n_estimators=params.get("n_estimators", 100),
max_depth=params.get("max_depth", None),
random_state=self.config.random_seed,
verbose=2
verbose=2,
)
def prepare_features(self, X: pd.DataFrame) -> np.ndarray:
research/models/svm_model.py
+1 -1
View File
@@ -25,7 +25,7 @@ class SVMModel(TraditionalModel):
gamma=params.get("gamma", "scale"),
probability=True, # Enable probability prediction
random_state=self.config.random_seed,
verbose=2
verbose=2,
)
return Pipeline([("vectorizer", vectorizer), ("classifier", classifier)])
research/models/xgboost_model.py
+11 -7
View File
@@ -28,7 +28,7 @@ class XGBoostModel(TraditionalModel):
colsample_bytree=params.get("colsample_bytree", 0.8),
random_state=self.config.random_seed,
eval_metric="logloss",
verbosity=2
verbosity=2,
)
def prepare_features(self, X: pd.DataFrame) -> np.ndarray:
@@ -50,14 +50,18 @@ class XGBoostModel(TraditionalModel):
self.vectorizers[feature_key] = CountVectorizer(
analyzer="char", ngram_range=(2, 3), max_features=100
)
char_features = self.vectorizers[feature_key].fit_transform(
column.fillna("").astype(str)
).toarray()
char_features = (
self.vectorizers[feature_key]
.fit_transform(column.fillna("").astype(str))
.toarray()
)
else:
# Subsequent times - use existing vectorizer
char_features = self.vectorizers[feature_key].transform(
column.fillna("").astype(str)
).toarray()
char_features = (
self.vectorizers[feature_key]
.transform(column.fillna("").astype(str))
.toarray()
)
features.append(char_features)
else: