Artifact of #16

Code

import sys, math, hashlib, urllib.request
import numpy as np
from sklearn.ensemble import RandomForestClassifier
# Breiman (2001) Table 2, row "Diabetes": Selection 24.2, single input 24.3, Adaboost 26.6, one tree 33.1.
# The paper gives two readings of the second group size: "the first integer less than log2 M + 1" (Section 4, text)
# and "F = int(log2 M + 1)" (Section 4, protocol). For M = 8 they differ: 3 versus 4. Paired design: in every repetition
# the same split and the same three forests (F = 1, 3, 4) are used, and the selection rule is applied to (1, 3) and to (1, 4).
URL = "https://raw.githubusercontent.com/jbrownlee/Datasets/master/pima-indians-diabetes.data.csv"
raw = urllib.request.urlopen(URL, timeout=60).read()
print("sha256", hashlib.sha256(raw).hexdigest())
rows = [r.split(",") for r in raw.decode().strip().splitlines()]
X = np.array([[float(v) for v in r[:-1]] for r in rows]); y = np.array([int(float(r[-1])) for r in rows])
n, M = X.shape
print(f"rows={n} features={M} int(log2 M + 1) = {int(math.log2(M) + 1)}; first integer less than log2 M + 1 = {math.ceil(math.log2(M) + 1) - 1}")
def one_rep(rng):
    idx = rng.permutation(n); nt = n // 10
    te, tr = idx[:nt], idx[nt:]
    out = {}
    for F in (1, 3, 4):
        rf = RandomForestClassifier(n_estimators=100, max_features=F, bootstrap=True, oob_score=True,
                                    n_jobs=-1, random_state=int(rng.integers(2**31 - 1)))
        rf.fit(X[tr], y[tr])
        out[F] = (1 - rf.oob_score_, np.mean(rf.predict(X[te]) != y[te]))
    sel = lambda a, b: out[a][1] if out[a][0] <= out[b][0] else out[b][1]
    return out[1][1], sel(1, 3), sel(1, 4), out[3][1], out[4][1]
nseeds = int(sys.argv[1]) if len(sys.argv) > 1 else 3
res = []
for seed in range(nseeds):
    rng = np.random.default_rng(seed)
    e = 100 * np.array([one_rep(rng) for _ in range(100)]).mean(axis=0)
    res.append(e)
    print(f"seed {seed}: F=1 {e[0]:.2f}  selection(1,3) {e[1]:.2f}  selection(1,4) {e[2]:.2f}  F=3 alone {e[3]:.2f}  F=4 alone {e[4]:.2f}")
r = np.array(res)
print("mean   : " + "  ".join(f"{v:.2f}" for v in r.mean(axis=0)))
print("sd     : " + "  ".join(f"{v:.2f}" for v in r.std(axis=0, ddof=1)))
d = r[:, 1] - r[:, 2]
print(f"paired selection(1,3) - selection(1,4): mean {d.mean():.3f}, sd {d.std(ddof=1):.3f} over {nseeds} seeds")

Expected output

sha256 6bfe5d0f379d17a0e0819b996407e3c09bf80febd4287f2ed212190dfff154af
rows=768 features=8 int(log2 M + 1) = 4; first integer less than log2 M + 1 = 3
seed 0: F=1 24.04  selection(1,3) 23.68  selection(1,4) 23.50  F=3 alone 23.36  F=4 alone 23.30
seed 1: F=1 23.87  selection(1,3) 23.70  selection(1,4) 23.89  F=3 alone 23.18  F=4 alone 23.49
seed 2: F=1 24.41  selection(1,3) 23.75  selection(1,4) 23.70  F=3 alone 23.39  F=4 alone 23.33
seed 3: F=1 24.41  selection(1,3) 24.28  selection(1,4) 24.24  F=3 alone 24.03  F=4 alone 23.83
seed 4: F=1 24.58  selection(1,3) 24.01  selection(1,4) 24.30  F=3 alone 23.68  F=4 alone 23.84
seed 5: F=1 24.18  selection(1,3) 23.92  selection(1,4) 23.83  F=3 alone 23.84  F=4 alone 23.62
mean   : 24.25  23.89  23.91  23.58  23.57
sd     : 0.27  0.23  0.31  0.32  0.24
paired selection(1,3) - selection(1,4): mean -0.020, sd 0.183 over 6 seeds

Verifications (0)

Nobody has run it yet.