I'm writing a python script for Orange Data Mining to plot the gaussian processes in order to find the best hyperparameters for the 5-FoldCrossValidation Accuracy metric. The three models are SVC, Neural Network and Classification Tree. The code I wrote so far correctly shows and run the bayesian optimization but I can't write the function to plot the Guassian Processes. I used the pure python bayesian-optimization library.
from sklearn.svm import SVC from sklearn.model_selection import StratifiedKFold, cross_val_score from sklearn.neural_network import MLPClassifier from sklearn.tree import DecisionTreeClassifier from bayes_opt import BayesianOptimization import numpy as np # Get the dataset from Orange X = in_data.X # The features y = in_data.Y # The target # Objective function for SVC optimization def svc_cv(C, gamma): """Calculate the mean accuracy using StratifiedKFold.""" model = SVC(C=C, gamma=gamma, kernel='rbf', random_state=42) cv = StratifiedKFold(n_splits=5, shuffle=False, random_state=None) scores = cross_val_score(model, X, y, cv=cv, scoring='accuracy') return scores.mean() # Objective function for neural network optimization def nn_cv(alpha, hidden_layer_1, hidden_layer_2, hidden_layer_3): """Calculate the mean accuracy using StratifiedKFold.""" hidden_layers = (int(hidden_layer_1), int(hidden_layer_2), int(hidden_layer_3)) model = MLPClassifier( hidden_layer_sizes=hidden_layers, activation='relu', max_iter=500, alpha=alpha, random_state=42 ) cv = StratifiedKFold(n_splits=5, shuffle=False, random_state=None) scores = cross_val_score(model, X, y, cv=cv, scoring='accuracy') return scores.mean() # Objective function for decision tree optimization def dt_cv(min_samples_leaf, max_depth, min_samples_split): """Calculate the mean accuracy using StratifiedKFold.""" model = DecisionTreeClassifier( min_samples_leaf=int(min_samples_leaf), max_depth=int(max_depth), min_samples_split=int(min_samples_split), random_state=42 ) cv = StratifiedKFold(n_splits=5, shuffle=False, random_state=None) scores = cross_val_score(model, X, y, cv=cv, scoring='accuracy') return scores.mean() # Bayesian Optimization for SVC svc_bounds = {'C': (0.1, 500), 'gamma': (0.01, 10)} svc_optimizer = BayesianOptimization( f=svc_cv, pbounds=svc_bounds, random_state=42, verbose=2 ) svc_optimizer.maximize(init_points=10, n_iter=25) # Bayesian Optimization for neural network nn_bounds = { 'alpha': (1e-5, 1e-1), 'hidden_layer_1': (1, 10), 'hidden_layer_2': (1, 10), 'hidden_layer_3': (1, 10) } nn_optimizer = BayesianOptimization( f=nn_cv, pbounds=nn_bounds, random_state=42, verbose=2 ) nn_optimizer.maximize(init_points=10, n_iter=25) # Bayesian Optimization for decision tree dt_bounds = { 'min_samples_leaf': (1, 20), 'max_depth': (1, 20), 'min_samples_split': (2, 20) } dt_optimizer = BayesianOptimization( f=dt_cv, pbounds=dt_bounds, random_state=42, verbose=2 ) dt_optimizer.maximize(init_points=10, n_iter=25) # Retrieve the optimal parameters svc_best_params = svc_optimizer.max['params'] nn_best_params = nn_optimizer.max['params'] dt_best_params = dt_optimizer.max['params'] # Calculate the final accuracy for each model svc_model = SVC(C=svc_best_params['C'], gamma=svc_best_params['gamma'], kernel='rbf', random_state=42) svc_cv_final = StratifiedKFold(n_splits=5, shuffle=False, random_state=None) svc_final_accuracy = cross_val_score(svc_model, X, y, cv=svc_cv_final, scoring='accuracy').mean() nn_model = MLPClassifier( hidden_layer_sizes=( int(nn_best_params['hidden_layer_1']), int(nn_best_params['hidden_layer_2']), int(nn_best_params['hidden_layer_3']) ), activation='relu', max_iter=500, alpha=nn_best_params['alpha'], random_state=42 ) nn_cv_final = StratifiedKFold(n_splits=5, shuffle=False, random_state=None) nn_final_accuracy = cross_val_score(nn_model, X, y, cv=nn_cv_final, scoring='accuracy').mean() dt_model = DecisionTreeClassifier( min_samples_leaf=int(dt_best_params['min_samples_leaf']), max_depth=int(dt_best_params['max_depth']), min_samples_split=int(dt_best_params['min_samples_split']), random_state=42 ) dt_cv_final = StratifiedKFold(n_splits=5, shuffle=False, random_state=None) dt_final_accuracy = cross_val_score(dt_model, X, y, cv=dt_cv_final, scoring='accuracy').mean() # Print results print("SVC Results:") print(f"Best parameters found: C={svc_best_params['C']:.4f}, gamma={svc_best_params['gamma']:.4f}") print(f"Mean accuracy with optimal parameters: {svc_final_accuracy:.4f}") print("\nNeural Network Results:") print(f"Best parameters found: alpha={nn_best_params['alpha']:.6f}, \ hidden_layer_sizes=({int(nn_best_params['hidden_layer_1'])}, {int(nn_best_params['hidden_layer_2'])}, {int(nn_best_params['hidden_layer_3'])})") print(f"Mean accuracy with optimal parameters: {nn_final_accuracy:.4f}") print("\nDecision Tree Results:") print(f"Best parameters found: min_samples_leaf={int(dt_best_params['min_samples_leaf'])}, \ max_depth={int(dt_best_params['max_depth'])}, min_samples_split={int(dt_best_params['min_samples_split'])}") print(f"Mean accuracy with optimal parameters: {dt_final_accuracy:.4f}") # Output for Orange out_object = {'svc': svc_model, 'nn': nn_model, 'dt': dt_model} 
