基于Python 3.9版本演示
一、写在前面
最近看了一篇在Lancet子刊《eClinicalMedicine》上发表的机器学习分类的文章：《Development of a novel dementia risk prediction model in the general population: A large, longitudinal, population-based machine-learning study》。
学到一种叫做“概率校准”的骚操作，顺手利用GPT系统学习学习。
文章中用的技术是：保序回归（Isotonic regression）。
为了体现举一反三，顺便问了GPT还有哪些方法也可以实现概率校准。它给我列举了很多，那么就一个一个学习吧。
这一期，介绍一个叫做 Histogram Binning 的方法。
二、Histogram Binning
Histogram Binning（直方图分箱）是一种用于数据预处理和特征工程的技术，它通过将连续变量分成离散的区间来简化数据处理和分析。基本思想是将连续变量划分成若干个区间（bin），每个区间包含一定范围内的数据点。数据点被分配到相应的区间中，通常每个区间用一个代表值（如区间的中位数或均值）来表示。
（1）作用和优势
1）处理连续变量：许多机器学习算法（如决策树、逻辑回归等）在处理连续变量时效果不佳，分箱可以将连续变量转换为离散变量，从而提高算法性能。
2）减少噪音和异常值影响：连续变量中的噪音和异常值可能会对模型训练产生不利影响。通过分箱，可以平滑数据，减少噪音和异常值的影响。
3）平衡类分布：在处理不平衡数据时，可以通过分箱技术对少数类和多数类的数据进行平衡，增强分类器对少数类的识别能力。
三、Histogram Binning代码实现
下面，我编一个1比3的不太平衡的数据进行测试，对照组使用不进行校准的SVM模型，实验组就是加入校准的SVM模型，看看性能能够提高多少？
（1）不进行校准的SVM模型（默认参数）
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve# 加载数据
dataset = pd.read_csv('8PSMjianmo.csv')
X = dataset.iloc[:, 1:20].values
Y = dataset.iloc[:, 0].values# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.30, random_state=666)# 标准化数据
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)# 使用SVM分类器
classifier = SVC(kernel='linear', probability=True)
classifier.fit(X_train, y_train)# 预测结果
y_pred = classifier.predict(X_test)
y_testprba = classifier.decision_function(X_test)y_trainpred = classifier.predict(X_train)
y_trainprba = classifier.decision_function(X_train)# 混淆矩阵
cm_test = confusion_matrix(y_test, y_pred)
cm_train = confusion_matrix(y_train, y_trainpred)
print(cm_train)
print(cm_test)# 绘制测试集混淆矩阵
classes = list(set(y_test))
classes.sort()
plt.imshow(cm_test, cmap=plt.cm.Blues)
indices = range(len(cm_test))
plt.xticks(indices, classes)
plt.yticks(indices, classes)
plt.colorbar()
plt.xlabel('Predicted')
plt.ylabel('Actual')
for first_index in range(len(cm_test)):for second_index in range(len(cm_test[first_index])):plt.text(first_index, second_index, cm_test[first_index][second_index])plt.show()# 绘制训练集混淆矩阵
classes = list(set(y_train))
classes.sort()
plt.imshow(cm_train, cmap=plt.cm.Blues)
indices = range(len(cm_train))
plt.xticks(indices, classes)
plt.yticks(indices, classes)
plt.colorbar()
plt.xlabel('Predicted')
plt.ylabel('Actual')
for first_index in range(len(cm_train)):for second_index in range(len(cm_train[first_index])):plt.text(first_index, second_index, cm_train[first_index][second_index])plt.show()# 计算并打印性能参数
def calculate_metrics(cm, y_true, y_pred_prob):a = cm[0, 0]b = cm[0, 1]c = cm[1, 0]d = cm[1, 1]acc = (a + d) / (a + b + c + d)error_rate = 1 - accsen = d / (d + c)sep = a / (a + b)precision = d / (b + d)F1 = (2 * precision * sen) / (precision + sen)MCC = (d * a - b * c) / (np.sqrt((d + b) * (d + c) * (a + b) * (a + c)))auc_score = roc_auc_score(y_true, y_pred_prob)metrics = {"Accuracy": acc,"Error Rate": error_rate,"Sensitivity": sen,"Specificity": sep,"Precision": precision,"F1 Score": F1,"MCC": MCC,"AUC": auc_score}return metricsmetrics_test = calculate_metrics(cm_test, y_test, y_testprba)
metrics_train = calculate_metrics(cm_train, y_train, y_trainprba)print("Performance Metrics (Test):")
for key, value in metrics_test.items():print(f"{key}: {value:.4f}")print("\nPerformance Metrics (Train):")
for key, value in metrics_train.items():
print(f"{key}: {value:.4f}")结果输出：
记住这些个数字。
这个参数的SVM还没有LR好。
（2）进行校准的SVM模型（默认参数）
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, brier_score_loss
from sklearn.calibration import calibration_curve# 加载数据
dataset = pd.read_csv('8PSMjianmo.csv')
X = dataset.iloc[:, 1:20].values
Y = dataset.iloc[:, 0].values# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.30, random_state=666)# 标准化数据
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)# 使用SVM分类器
classifier = SVC(kernel='rbf', C=0.1, probability=True)
classifier.fit(X_train, y_train)# 获取未校准的概率预测
y_train_probs = classifier.predict_proba(X_train)[:, 1]
y_test_probs = classifier.predict_proba(X_test)[:, 1]# Histogram Binning 校准函数
def histogram_binning(probabilities, y_true, n_bins=10):bin_edges = np.linspace(0, 1, n_bins + 1)bin_indices = np.digitize(probabilities, bin_edges) - 1bin_sums = np.zeros(n_bins)bin_true_sums = np.zeros(n_bins)bin_counts = np.zeros(n_bins)for i in range(n_bins):bin_counts[i] = np.sum(bin_indices == i)if bin_counts[i] > 0:bin_sums[i] = np.sum(probabilities[bin_indices == i])bin_true_sums[i] = np.sum(y_true[bin_indices == i])bin_probs = np.zeros_like(probabilities)for i in range(n_bins):if bin_counts[i] > 0:bin_probs[bin_indices == i] = bin_true_sums[i] / bin_counts[i]return bin_probs# 进行Histogram Binning校准
calibrated_train_probs = histogram_binning(y_train_probs, y_train)
calibrated_test_probs = histogram_binning(y_test_probs, y_test)# 预测结果
y_train_pred = (calibrated_train_probs >= 0.5).astype(int)
y_test_pred = (calibrated_test_probs >= 0.5).astype(int)# 混淆矩阵
cm_test = confusion_matrix(y_test, y_test_pred)
cm_train = confusion_matrix(y_train, y_train_pred)
print(cm_train)
print(cm_test)# 绘制混淆矩阵函数
def plot_confusion_matrix(cm, classes, title='Confusion Matrix'):plt.imshow(cm, cmap=plt.cm.Blues)indices = range(len(cm))plt.xticks(indices, classes)plt.yticks(indices, classes)plt.colorbar()plt.xlabel('Predicted')plt.ylabel('Actual')for first_index in range(len(cm)):for second_index in range(len(cm[first_index])):plt.text(first_index, second_index, cm[first_index][second_index])plt.title(title)plt.show()# 绘制测试集混淆矩阵
plot_confusion_matrix(cm_test, list(set(y_test)), 'Confusion Matrix (Test)')# 绘制训练集混淆矩阵
plot_confusion_matrix(cm_train, list(set(y_train)), 'Confusion Matrix (Train)')# 计算并打印性能参数
def calculate_metrics(cm, y_true, y_pred_prob):a = cm[0, 0]b = cm[0, 1]c = cm[1, 0]d = cm[1, 1]acc = (a + d) / (a + b + c + d)error_rate = 1 - accsen = d / (d + c)sep = a / (a + b)precision = d / (b + d)F1 = (2 * precision * sen) / (precision + sen)MCC = (d * a - b * c) / (np.sqrt((d + b) * (d + c) * (a + b) * (a + c)))auc_score = roc_auc_score(y_true, y_pred_prob)brier_score = brier_score_loss(y_true, y_pred_prob)metrics = {"Accuracy": acc,"Error Rate": error_rate,"Sensitivity": sen,"Specificity": sep,"Precision": precision,"F1 Score": F1,"MCC": MCC,"AUC": auc_score,"Brier Score": brier_score}return metricsmetrics_test = calculate_metrics(cm_test, y_test, calibrated_test_probs)
metrics_train = calculate_metrics(cm_train, y_train, calibrated_train_probs)print("Performance Metrics (Test):")
for key, value in metrics_test.items():print(f"{key}: {value:.4f}")print("\nPerformance Metrics (Train):")
for key, value in metrics_train.items():print(f"{key}: {value:.4f}")看看结果：
大同小异吧。
四、换个策略
参考那篇文章的策略：采用五折交叉验证来建立和评估模型，其中四折用于训练，一折用于评估，在训练集中，其中三折用于建立SVM模型，另一折采用Histogram Binning概率校正，在训练集内部采用交叉验证对超参数进行调参。
代码：
import numpy as np
import matplotlib.pyplot as plt
import pandas as pd
from sklearn.model_selection import train_test_split, GridSearchCV, KFold
from sklearn.preprocessing import StandardScaler
from sklearn.svm import SVC
from sklearn.metrics import confusion_matrix, roc_auc_score, roc_curve, brier_score_loss
from sklearn.calibration import calibration_curve# 加载数据
dataset = pd.read_csv('8PSMjianmo.csv')
X = dataset.iloc[:, 1:20].values
Y = dataset.iloc[:, 0].values# 分割数据集
X_train, X_test, y_train, y_test = train_test_split(X, Y, test_size=0.30, random_state=666)# 标准化数据
sc = StandardScaler()
X_train = sc.fit_transform(X_train)
X_test = sc.transform(X_test)# 定义五折交叉验证
kf = KFold(n_splits=5, shuffle=True, random_state=666)
calibrated_probs = []
true_labels = []for train_index, test_index in kf.split(X_train):X_train_fold, X_val_fold = X_train[train_index], X_train[test_index]y_train_fold, y_val_fold = y_train[train_index], y_train[test_index]# 内部三折交叉验证用于超参数调优inner_kf = KFold(n_splits=3, shuffle=True, random_state=666)param_grid = {'C': [0.01, 0.1, 1, 10, 100], 'kernel': ['rbf']}svm = SVC(probability=True)clf = GridSearchCV(svm, param_grid, cv=inner_kf, scoring='roc_auc')clf.fit(X_train_fold, y_train_fold)best_params = clf.best_params_# 使用最佳参数训练SVMclassifier = SVC(kernel=best_params['kernel'], C=best_params['C'], probability=True)classifier.fit(X_train_fold, y_train_fold)# 获取未校准的概率预测y_val_fold_probs = classifier.predict_proba(X_val_fold)[:, 1]# Histogram Binning 校准函数def histogram_binning(probabilities, y_true, n_bins=10):bin_edges = np.linspace(0, 1, n_bins + 1)bin_indices = np.digitize(probabilities, bin_edges) - 1bin_sums = np.zeros(n_bins)bin_true_sums = np.zeros(n_bins)bin_counts = np.zeros(n_bins)for i in range(n_bins):bin_counts[i] = np.sum(bin_indices == i)if bin_counts[i] > 0:bin_sums[i] = np.sum(probabilities[bin_indices == i])bin_true_sums[i] = np.sum(y_true[bin_indices == i])bin_probs = np.zeros_like(probabilities)for i in range(n_bins):if bin_counts[i] > 0:bin_probs[bin_indices == i] = bin_true_sums[i] / bin_counts[i]return bin_probs# 进行Histogram Binning校准calibrated_val_fold_probs = histogram_binning(y_val_fold_probs, y_val_fold)calibrated_probs.extend(calibrated_val_fold_probs)true_labels.extend(y_val_fold)# 用于测试集的SVM模型训练和校准
classifier_final = SVC(kernel=best_params['kernel'], C=best_params['C'], probability=True)
classifier_final.fit(X_train, y_train)
y_test_probs = classifier_final.predict_proba(X_test)[:, 1]
calibrated_test_probs = histogram_binning(y_test_probs, y_test)# 预测结果
y_train_pred = (np.array(calibrated_probs) >= 0.5).astype(int)
y_test_pred = (calibrated_test_probs >= 0.5).astype(int)# 混淆矩阵
cm_test = confusion_matrix(y_test, y_test_pred)
cm_train = confusion_matrix(y_train, y_train_pred)
print("Training Confusion Matrix:\n", cm_train)
print("Testing Confusion Matrix:\n", cm_test)# 绘制混淆矩阵函数
def plot_confusion_matrix(cm, classes, title='Confusion Matrix'):plt.imshow(cm, cmap=plt.cm.Blues)indices = range(len(cm))plt.xticks(indices, classes)plt.yticks(indices, classes)plt.colorbar()plt.xlabel('Predicted')plt.ylabel('Actual')for first_index in range(len(cm)):for second_index in range(len(cm[first_index])):plt.text(first_index, second_index, cm[first_index][second_index])plt.title(title)plt.show()# 绘制测试集混淆矩阵
plot_confusion_matrix(cm_test, list(set(y_test)), 'Confusion Matrix (Test)')# 绘制训练集混淆矩阵
plot_confusion_matrix(cm_train, list(set(y_train)), 'Confusion Matrix (Train)')# 计算并打印性能参数
def calculate_metrics(cm, y_true, y_pred_prob):a = cm[0, 0]b = cm[0, 1]c = cm[1, 0]d = cm[1, 1]acc = (a + d) / (a + b + c + d)error_rate = 1 - accsen = d / (d + c)sep = a / (a + b)precision = d / (b + d)F1 = (2 * precision * sen) / (precision + sen)MCC = (d * a - b * c) / (np.sqrt((d + b) * (d + c) * (a + b) * (a + c)))auc_score = roc_auc_score(y_true, y_pred_prob)brier_score = brier_score_loss(y_true, y_pred_prob)metrics = {"Accuracy": acc,"Error Rate": error_rate,"Sensitivity": sen,"Specificity": sep,"Precision": precision,"F1 Score": F1,"MCC": MCC,"AUC": auc_score,"Brier Score": brier_score}return metricsmetrics_test = calculate_metrics(cm_test, y_test, calibrated_test_probs)
metrics_train = calculate_metrics(cm_train, y_train, np.array(calibrated_probs))print("Performance Metrics (Test):")
for key, value in metrics_test.items():print(f"{key}: {value:.4f}")print("\nPerformance Metrics (Train):")
for key, value in metrics_train.items():print(f"{key}: {value:.4f}")# 绘制校准曲线
def plot_calibration_curve(y_true, probs, title='Calibration Curve'):fraction_of_positives, mean_predicted_value = calibration_curve(y_true, probs, n_bins=10)plt.plot(mean_predicted_value, fraction_of_positives, "s-", label="Histogram Binning")plt.plot([0, 1], [0, 1], "k--")plt.xlabel('Mean predicted value')plt.ylabel('Fraction of positives')plt.title(title)plt.legend()plt.show()# 绘制校准曲线
plot_calibration_curve(y_test, calibrated_test_probs, title='Calibration Curve (Test)')
plot_calibration_curve(y_train, np.array(calibrated_probs), title='Calibration Curve (Train)')输出：
哈哈哈，撤了撤了。
五、最后
各位可以去试一试在其他数据或者在其他机器学习分类模型中使用的效果。
数据不分享啦。
