How to choose a model

import warnings
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.dummy import DummyClassifier
from sklearn.linear_model import LogisticRegression
from sklearn.neighbors import KNeighborsClassifier
from sklearn.naive_bayes import GaussianNB
from sklearn.svm import SVC
from sklearn.tree import DecisionTreeClassifier
from sklearn.ensemble import RandomForestClassifier, GradientBoostingClassifier
from sklearn.neural_network import MLPClassifier
from sklearn.model_selection import cross_val_score, StratifiedKFold

warnings.filterwarnings("ignore")
rng = np.random.default_rng(21)
n = 900
X = rng.normal(size=(n, 8))
# the truth mixes a straight effect, an interaction and a threshold, plus noise
score = 1.0 * X[:, 0] + 1.2 * X[:, 1] * X[:, 2] + 1.5 * (X[:, 3] > 0.5) + rng.normal(0, 0.8, n)
y = (score > np.median(score)).astype(int)

models = {
    "majority baseline": DummyClassifier(),
    "logistic regression": make_pipeline(StandardScaler(), LogisticRegression()),
    "k-nearest neighbours": make_pipeline(StandardScaler(), KNeighborsClassifier(15)),
    "naive Bayes": GaussianNB(),
    "RBF support vector machine": make_pipeline(StandardScaler(), SVC()),
    "decision tree (depth 4)": DecisionTreeClassifier(max_depth=4, random_state=0),
    "random forest": RandomForestClassifier(60, min_samples_leaf=3, random_state=0),
    "gradient boosting": GradientBoostingClassifier(n_estimators=80, max_depth=2, random_state=0),
    "neural network (16 units)": make_pipeline(StandardScaler(), MLPClassifier((16,), solver="lbfgs", max_iter=400, random_state=0)),
}
cv = StratifiedKFold(5, shuffle=True, random_state=0)
print(f"{'model':28s} | accuracy | spread across folds")
for name, m in models.items():
    s = cross_val_score(m, X, y, cv=cv)
    print(f"{name:28s} | {s.mean():8.3f} | {s.min():.3f} to {s.max():.3f}")

model                        | accuracy | spread across folds
majority baseline            |    0.500 | 0.500 to 0.500
logistic regression          |    0.726 | 0.700 to 0.761
k-nearest neighbours         |    0.766 | 0.739 to 0.778
naive Bayes                  |    0.727 | 0.706 to 0.744
RBF support vector machine   |    0.798 | 0.778 to 0.828
decision tree (depth 4)      |    0.722 | 0.678 to 0.744
random forest                |    0.754 | 0.739 to 0.778
gradient boosting            |    0.737 | 0.694 to 0.767
neural network (16 units)    |    0.758 | 0.733 to 0.789

import warnings
import numpy as np
from sklearn.pipeline import make_pipeline
from sklearn.preprocessing import StandardScaler
from sklearn.linear_model import LogisticRegression
from sklearn.ensemble import GradientBoostingClassifier
from sklearn.svm import SVC
from sklearn.model_selection import cross_val_score, StratifiedKFold

warnings.filterwarnings("ignore")
cv = StratifiedKFold(5, shuffle=True, random_state=0)
def make(kind, n, seed):
    r = np.random.default_rng(seed)
    X = r.normal(size=(n, 6))
    if kind == "straight-line signal":
        s = X[:, 0] + 0.8 * X[:, 1] - 0.6 * X[:, 2] + r.normal(0, 1.0, n)
    else:
        s = X[:, 0] * X[:, 1] + 1.5 * (X[:, 2] > 0.3) + r.normal(0, 0.6, n)
    return X, (s > np.median(s)).astype(int)

models = {
    "logistic": make_pipeline(StandardScaler(), LogisticRegression()),
    "RBF SVM": make_pipeline(StandardScaler(), SVC()),
    "boosting": GradientBoostingClassifier(n_estimators=80, max_depth=2, random_state=0),
}
print("data | rows | " + " | ".join(models))
for kind in ["straight-line signal", "interactions and thresholds"]:
    for n in [120, 1200]:
        X, y = make(kind, n, 3)
        row = [cross_val_score(m, X, y, cv=cv).mean() for m in models.values()]
        print(f"{kind} | {n} | " + " | ".join(f"{v:.3f}" for v in row))

data | rows | logistic | RBF SVM | boosting
straight-line signal | 120 | 0.783 | 0.817 | 0.800
straight-line signal | 1200 | 0.798 | 0.786 | 0.793
interactions and thresholds | 120 | 0.733 | 0.808 | 0.733
interactions and thresholds | 1200 | 0.733 | 0.823 | 0.786