Test multiple seeds (#174)

* Test multiple rngs

* Add comment

* Fix rng not being passed correctly

* Revert some changes

* Update comment

* Extend tests

* Simplify test

* Lower accuracy bound

* Fix a bug in the usage of Mersenne Twister

* Fix tests

* Use some more StableRNG

* Use some more StableRNG

* Use some more StableRNG

* Use some more StableRNG

* Use some more StableRNG

* Use some more StableRNG

* Fix Julia 1.6

* Use `StableRNG` in test/classification/adult

* Use `StableRNG` for data generation too

* Put old numbers back

* Add one more rng

Author: rikhuijzer

.github/workflows/CI.yml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        version:          
+        version:
           - '1.0'
           - '1.6'
           - '1' # automatically expands to the latest stable 1.x release of Julia

Project.toml

@@ -7,14 +7,19 @@ version = "0.10.12"
 [deps]
 AbstractTrees = "1520ce14-60c1-5f80-bbc7-55ef81b5835c"
 DelimitedFiles = "8bb1440f-4735-579b-a4ab-409b98df4dab"
-Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
 Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
 ScikitLearnBase = "6e75b9c4-186b-50bd-896f-2d2496a4843e"
 Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
-Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
 AbstractTrees = "0.3"
 ScikitLearnBase = "0.5"
 julia = "1"
+
+[extras]
+StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["StableRNGs", "Test"]

src/classification/main.jl

@@ -37,7 +37,8 @@ end
 function _convert(
         node   :: treeclassifier.NodeMeta{S},
         list   :: AbstractVector{T},
-        labels :: AbstractVector{T}) where {S, T}
+        labels :: AbstractVector{T}
+    ) where {S, T}
 
     if node.is_leaf
         return Leaf{T}(list[node.label], labels[node.region])
@@ -138,7 +139,7 @@ function prune_tree(tree::LeafOrNode{S, T}, purity_thresh=1.0) where {S, T}
 end
 
 
-apply_tree(leaf::Leaf{T}, feature::AbstractVector{S}) where {S, T} = leaf.majority
+apply_tree(leaf::Leaf, feature::AbstractVector) = leaf.majority
 
 function apply_tree(tree::Node{S, T}, features::AbstractVector{S}) where {S, T}
     if tree.featid == 0
@@ -197,7 +198,7 @@ function build_forest(
         min_samples_leaf    = 1,
         min_samples_split   = 2,
         min_purity_increase = 0.0;
-        rng                 = Random.GLOBAL_RNG) where {S, T}
+        rng::Union{Integer,AbstractRNG} = Random.GLOBAL_RNG) where {S, T}
 
     if n_trees < 1
         throw("the number of trees must be >= 1")
@@ -221,7 +222,12 @@ function build_forest(
 
     if rng isa Random.AbstractRNG
         Threads.@threads for i in 1:n_trees
-            inds = rand(rng, 1:t_samples, n_samples)
+            # The Mersenne Twister (Julia's default) is not thread-safe.
+            _rng = copy(rng)
+            # Take some elements from the ring to have different states for each tree.
+            # This is the only way given that only a `copy` can be expected to exist for RNGs.
+            rand(_rng, i)
+            inds = rand(_rng, 1:t_samples, n_samples)
             forest[i] = build_tree(
                 labels[inds],
                 features[inds,:],
@@ -231,9 +237,9 @@ function build_forest(
                 min_samples_split,
                 min_purity_increase,
                 loss = loss,
-                rng = rng)
+                rng = _rng)
         end
-    elseif rng isa Integer # each thread gets its own seeded rng
+    else # each thread gets its own seeded rng
         Threads.@threads for i in 1:n_trees
             Random.seed!(rng + i)
             inds = rand(1:t_samples, n_samples)
@@ -247,8 +253,6 @@ function build_forest(
                 min_purity_increase,
                 loss = loss)
         end
-    else
-        throw("rng must of be type Integer or Random.AbstractRNG")
     end
 
     return Ensemble{S, T}(forest)
@@ -298,7 +302,7 @@ function build_adaboost_stumps(
         labels       :: AbstractVector{T},
         features     :: AbstractMatrix{S},
         n_iterations :: Integer;
-        rng           = Random.GLOBAL_RNG) where {S, T}
+        rng = Random.GLOBAL_RNG) where {S, T}
     N = length(labels)
     weights = ones(N) / N
     stumps = Node{S, T}[]

src/measures.jl

@@ -135,7 +135,7 @@ function _nfoldCV(classifier::Symbol, labels::AbstractVector{T}, features::Abstr
             predictions = apply_forest(model, test_features)
         elseif classifier == :stumps
             model, coeffs = build_adaboost_stumps(
-                train_labels, train_features, n_iterations)
+                train_labels, train_features, n_iterations; rng=rng)
             predictions = apply_adaboost_stumps(model, coeffs, test_features)
         end
         cm = confusion_matrix(test_labels, predictions)
@@ -186,6 +186,7 @@ function nfoldCV_stumps(
         n_iterations ::Integer = 10;
         verbose             :: Bool = true,
         rng          = Random.GLOBAL_RNG) where {S, T}
+    rng = mk_rng(rng)::Random.AbstractRNG
     _nfoldCV(:stumps, labels, features, n_folds, n_iterations; verbose=verbose, rng=rng)
 end
 

src/regression/main.jl

@@ -56,7 +56,7 @@ function build_forest(
         min_samples_leaf    = 5,
         min_samples_split   = 2,
         min_purity_increase = 0.0;
-        rng                 = Random.GLOBAL_RNG) where {S, T <: Float64}
+        rng::Union{Integer,AbstractRNG} = Random.GLOBAL_RNG) where {S, T <: Float64}
 
     if n_trees < 1
         throw("the number of trees must be >= 1")
@@ -77,7 +77,12 @@ function build_forest(
 
     if rng isa Random.AbstractRNG
         Threads.@threads for i in 1:n_trees
-            inds = rand(rng, 1:t_samples, n_samples)
+            # The Mersenne Twister (Julia's default) is not thread-safe.
+            _rng = copy(rng)
+            # Take some elements from the ring to have different states for each tree.
+            # This is the only way given that only a `copy` can be expected to exist for RNGs.
+            rand(_rng, i)
+            inds = rand(_rng, 1:t_samples, n_samples)
             forest[i] = build_tree(
                 labels[inds],
                 features[inds,:],
@@ -86,9 +91,9 @@ function build_forest(
                 min_samples_leaf,
                 min_samples_split,
                 min_purity_increase,
-                rng = rng)
+                rng = _rng)
         end
-    elseif rng isa Integer # each thread gets its own seeded rng
+    else # each thread gets its own seeded rng
         Threads.@threads for i in 1:n_trees
             Random.seed!(rng + i)
             inds = rand(1:t_samples, n_samples)
@@ -101,8 +106,6 @@ function build_forest(
                 min_samples_split,
                 min_purity_increase)
         end
-    else
-        throw("rng must of be type Integer or Random.AbstractRNG")
     end
 
     return Ensemble{S, T}(forest)

test/classification/adult.jl

@@ -5,7 +5,7 @@
 
 features, labels = load_data("adult")
 
-model = build_tree(labels, features)
+model = build_tree(labels, features; rng=StableRNG(1))
 preds = apply_tree(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.99
@@ -15,35 +15,35 @@ labels   = string.(labels)
 
 n_subfeatures = 3
 n_trees = 5
-model = build_forest(labels, features, n_subfeatures, n_trees)
+model = build_forest(labels, features, n_subfeatures, n_trees; rng=StableRNG(1))
 preds = apply_forest(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 
 n_iterations = 15
-model, coeffs = build_adaboost_stumps(labels, features, n_iterations);
+model, coeffs = build_adaboost_stumps(labels, features, n_iterations; rng=StableRNG(1));
 preds = apply_adaboost_stumps(model, coeffs, features);
 cm = confusion_matrix(labels, preds);
 @test cm.accuracy > 0.8
 
 println("\n##### 3 foldCV Classification Tree #####")
 pruning_purity = 0.9
 nfolds = 3
-accuracy = nfoldCV_tree(labels, features, nfolds, pruning_purity; verbose=false);
+accuracy = nfoldCV_tree(labels, features, nfolds, pruning_purity; rng=StableRNG(1), verbose=false);
 @test mean(accuracy) > 0.8
 
 println("\n##### 3 foldCV Classification Forest #####")
 n_subfeatures = 2
 n_trees = 10
 n_folds = 3
 partial_sampling = 0.5
-accuracy = nfoldCV_forest(labels, features, n_folds, n_subfeatures, n_trees, partial_sampling; verbose=false)
+accuracy = nfoldCV_forest(labels, features, n_folds, n_subfeatures, n_trees, partial_sampling; rng=StableRNG(1), verbose=false)
 @test mean(accuracy) > 0.8
 
 println("\n##### nfoldCV Classification Adaboosted Stumps #####")
 n_iterations = 15
 n_folds = 3
-accuracy = nfoldCV_stumps(labels, features, n_folds, n_iterations; verbose=false);
+accuracy = nfoldCV_stumps(labels, features, n_folds, n_iterations; rng=StableRNG(1), verbose=false);
 @test mean(accuracy) > 0.8
 
 end # @testset

test/classification/digits.jl

@@ -69,7 +69,8 @@ model = DecisionTree.build_forest(
         max_depth,
         min_samples_leaf,
         min_samples_split,
-        min_purity_increase)
+        min_purity_increase;
+        rng=StableRNG(1))
 preds = apply_forest(model, X)
 cm = confusion_matrix(Y, preds)
 @test cm.accuracy > 0.95

test/classification/heterogeneous.jl

@@ -5,29 +5,29 @@
 m, n = 10^2, 5
 
 tf = [trues(Int(m/2)) falses(Int(m/2))]
-inds = Random.randperm(m)
+inds = Random.randperm(StableRNG(1), m)
 labels = string.(tf[inds])
 
 features = Array{Any}(undef, m, n)
-features[:,:] = randn(m, n)
-features[:,2] = string.(tf[Random.randperm(m)])
+features[:,:] = randn(StableRNG(1), m, n)
+features[:,2] = string.(tf[Random.randperm(StableRNG(1), m)])
 features[:,3] = map(t -> round.(Int, t), features[:,3])
 features[:,4] = tf[inds]
 
-model = build_tree(labels, features)
+model = build_tree(labels, features; rng=StableRNG(1))
 preds = apply_tree(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 
 n_subfeatures = 2
 n_trees = 3
-model = build_forest(labels, features, n_subfeatures, n_trees)
+model = build_forest(labels, features, n_subfeatures, n_trees; rng=StableRNG(1))
 preds = apply_forest(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
 
 n_subfeatures = 7
-model, coeffs = build_adaboost_stumps(labels, features, n_subfeatures)
+model, coeffs = build_adaboost_stumps(labels, features, n_subfeatures; rng=StableRNG(1))
 preds = apply_adaboost_stumps(model, coeffs, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9

test/classification/iris.jl

@@ -59,14 +59,14 @@ cm = confusion_matrix(labels, preds)
 # run n-fold cross validation for pruned tree
 println("\n##### nfoldCV Classification Tree #####")
 nfolds = 3
-accuracy = nfoldCV_tree(labels, features, nfolds)
+accuracy = nfoldCV_tree(labels, features, nfolds; rng=StableRNG(1))
 @test mean(accuracy) > 0.8
 
 # train random forest classifier
 n_trees = 10
 n_subfeatures = 2
 partial_sampling = 0.5
-model = build_forest(labels, features, n_subfeatures, n_trees, partial_sampling)
+model = build_forest(labels, features, n_subfeatures, n_trees, partial_sampling; rng=StableRNG(2))
 preds = apply_forest(model, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.95
@@ -80,12 +80,12 @@ n_subfeatures = 2
 n_trees = 10
 n_folds = 3
 partial_sampling = 0.5
-accuracy = nfoldCV_forest(labels, features, nfolds, n_subfeatures, n_trees, partial_sampling)
+accuracy = nfoldCV_forest(labels, features, nfolds, n_subfeatures, n_trees, partial_sampling; rng=StableRNG(1))
 @test mean(accuracy) > 0.9
 
 # train adaptive-boosted decision stumps
 n_iterations = 15
-model, coeffs = build_adaboost_stumps(labels, features, n_iterations)
+model, coeffs = build_adaboost_stumps(labels, features, n_iterations; rng=StableRNG(1))
 preds = apply_adaboost_stumps(model, coeffs, features)
 cm = confusion_matrix(labels, preds)
 @test cm.accuracy > 0.9
@@ -97,7 +97,7 @@ probs = apply_adaboost_stumps_proba(model, coeffs, features, classes)
 println("\n##### nfoldCV Classification Adaboosted Stumps #####")
 n_iterations = 15
 nfolds = 3
-accuracy = nfoldCV_stumps(labels, features, nfolds, n_iterations)
+accuracy = nfoldCV_stumps(labels, features, nfolds, n_iterations; rng=StableRNG(1))
 @test mean(accuracy) > 0.85
 
 end # @testset

test/classification/low_precision.jl

@@ -5,9 +5,9 @@ Random.seed!(16)
 
 n,m = 10^3, 5;
 features = Array{Any}(undef, n, m);
-features[:,:] = rand(n, m);
+features[:,:] = rand(StableRNG(1), n, m);
 features[:,1] = round.(Int32, features[:,1]); # convert a column of 32bit integers
-weights = rand(-1:1,m);
+weights = rand(StableRNG(1), -1:1, m);
 labels = round.(Int32, features * weights);
 
 model = build_stump(labels, features)
@@ -25,7 +25,8 @@ model = build_tree(
         n_subfeatures, max_depth,
         min_samples_leaf,
         min_samples_split,
-        min_purity_increase)
+        min_purity_increase;
+        rng=StableRNG(1))
 preds = apply_tree(model, features)
 cm = confusion_matrix(labels, preds)
 @test typeof(preds) == Vector{Int32}
@@ -40,14 +41,15 @@ model = build_forest(
         n_subfeatures,
         n_trees,
         partial_sampling,
-        max_depth)
+        max_depth;
+        rng=StableRNG(1))
 preds = apply_forest(model, features)
 cm = confusion_matrix(labels, preds)
 @test typeof(preds) == Vector{Int32}
 @test cm.accuracy > 0.9
 
 n_iterations        = Int32(25)
-model, coeffs = build_adaboost_stumps(labels, features, n_iterations);
+model, coeffs = build_adaboost_stumps(labels, features, n_iterations; rng=StableRNG(1));
 preds = apply_adaboost_stumps(model, coeffs, features);
 cm = confusion_matrix(labels, preds)
 @test typeof(preds) == Vector{Int32}
@@ -67,7 +69,8 @@ accuracy = nfoldCV_tree(
                 max_depth,
                 min_samples_leaf,
                 min_samples_split,
-                min_purity_increase)
+                min_purity_increase;
+                rng=StableRNG(1))
 @test mean(accuracy) > 0.7
 
 println("\n##### nfoldCV Classification Forest #####")
@@ -87,12 +90,13 @@ accuracy = nfoldCV_forest(
         max_depth,
         min_samples_leaf,
         min_samples_split,
-        min_purity_increase)
+        min_purity_increase;
+        rng=StableRNG(1))
 @test mean(accuracy) > 0.7
 
 println("\n##### nfoldCV Adaboosted Stumps #####")
 n_iterations        = Int32(25)
-accuracy = nfoldCV_stumps(labels, features, n_folds, n_iterations)
+accuracy = nfoldCV_stumps(labels, features, n_folds, n_iterations; rng=StableRNG(1))
 @test mean(accuracy) > 0.6