Adding weights normalization. Improvements in simple example

Author: emmanuellujan

docs/Project.toml

@@ -2,5 +2,6 @@
 Documenter = "e30172f5-a6a5-5a46-863b-614d45cd2de4"
 DocumenterCitations = "daee34ce-89f3-4625-b898-19384cb65244"
 Literate = "98b081ad-f1c9-55d3-8b20-4c87d4299306"
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 StreamingSampling = "e1325ea1-13b9-452d-9115-4dbfefa12b3b"

examples/simple-example/Project.toml

@@ -1,3 +1,4 @@
 [deps]
+Plots = "91a5bcdd-55d7-5caf-9e0b-520d859cae80"
 StatsBase = "2913bbd2-ae8a-5f71-8c99-4fb6c76f3a91"
 StreamingSampling = "e1325ea1-13b9-452d-9115-4dbfefa12b3b"

examples/simple-example/plot_weights.jl

@@ -0,0 +1,51 @@
+# helper
+padlims(v; frac=0.10) = begin
+    vmin, vmax = extrema(v)
+    span = max(vmax - vmin, eps())
+    (vmin - frac*span, vmax + frac*span)
+end
+
+function plot_weights(ws, inds;
+        size=(640, 480), dpi=300,
+        guidefs=20, tickfs=14, legendfs=16,
+        grid=:y,
+        margins=(left=30, right=40, bottom=30, top=20))
+
+    perm = sortperm(ws)
+    s    = ws[perm]
+    x    = collect(eachindex(s))
+    m    = in(Set(inds)).(perm)
+
+    # ensure margins are AbsoluteLength in px (avoids Int + AbsoluteLength conflicts)
+    lp = margins.left  * Plots.px
+    rp = margins.right * Plots.px
+    bp = margins.bottom* Plots.px
+    tp = margins.top   * Plots.px
+
+    plt = plot(size=size, dpi=dpi, grid=grid, framestyle=:box,
+               left_margin=lp, right_margin=rp,
+               bottom_margin=bp, top_margin=tp,
+               xguidefont=font(guidefs), yguidefont=font(guidefs),
+               xtickfont=font(tickfs),  ytickfont=font(tickfs),
+               legendfont=font(legendfs))
+
+    scatter!(plt, x, s;
+        color=:gray, alpha=0.25,
+        marker=:circle, markersize=8, markerstrokewidth=0,
+        label="All",
+        xlabel="Sorted element indices",
+        ylabel="Weights",
+        ylims=padlims(s))
+
+    if any(m)
+        scatter!(plt, x[m], s[m];
+            color=:red, marker=:utriangle,
+            markersize=10, markerstrokewidth=0,
+            label="Selected")
+    else
+        plot!(plt, NaN, NaN; label="Selected")
+    end
+
+    return plt
+end
+

examples/simple-example/simple-example.jl

@@ -1,5 +1,6 @@
 using StreamingSampling
 using StatsBase: sample, Weights
+using Plots
 
 # Define file paths
 base = haskey(ENV, "BASE_PATH") ? ENV["BASE_PATH"] : "../../"
@@ -9,14 +10,17 @@ file_paths = ["$path/data1.txt",
               "$path/data3.txt",
               "$path/data4.txt"];
 
+include("$base/examples/simple-example/plot_weights.jl"); #hide
+
 # Define sample size
-n = 100;
+n = 30;
 
 # Streaming weighted sampling
 ws = compute_weights(file_paths; chunksize=500, subchunksize=100)
 inds_w = sample(1:length(ws), Weights(ws), n; replace=false)
+plot_weights(ws, inds_w) #hide
 
 # Streaming maximum entropy sampling
 s = UPmaxentropy(inclusion_prob(ws, n))
 inds_me = findall(s .== 1)
-
+plot_weights(ws, inds_me) #hide

src/Weights.jl

@@ -7,7 +7,8 @@ function compute_weights(file_paths::Vector{String};
                          subchunksize=200,
                          buffersize=32,
                          max=Inf,
-                         randomized=true)
+                         randomized=true,
+                         normalize=true)
     ch, N = chunk_iterator(file_paths;
                            read_element=read_element,
                            chunksize=subchunksize, 
@@ -16,11 +17,16 @@ function compute_weights(file_paths::Vector{String};
     if max == Inf
         max = N
     end
-    return compute_weights(ch;
-                           create_feature=create_feature,
-                           chunksize=chunksize,
-                           subchunksize=subchunksize,
-                           max=max)
+    ws = compute_weights(ch;
+                         create_feature=create_feature,
+                         chunksize=chunksize,
+                         subchunksize=subchunksize,
+                         max=max)
+    if normalize
+        wmin, wmax = minimum(ws), maximum(ws)
+        ws = (ws .- wmin) ./ (wmax - wmin)
+    end
+    return ws
 end
 
 function compute_weights(A::Vector;
@@ -30,7 +36,8 @@ function compute_weights(A::Vector;
                          subchunksize=100,
                          buffersize=32,
                          max=Inf,
-                         randomized=true)
+                         randomized=true,
+                         normalize=true)
     ch, N = chunk_iterator(A;
                            read_element=read_element,
                            chunksize=subchunksize,
@@ -39,11 +46,16 @@ function compute_weights(A::Vector;
     if max == Inf
         max = N
     end
-    return compute_weights(ch;
-                           create_feature=create_feature,
-                           chunksize=chunksize,
-                           subchunksize=subchunksize,
-                           max=max)
+    ws = compute_weights(ch;
+                         create_feature=create_feature,
+                         chunksize=chunksize,
+                         subchunksize=subchunksize,
+                         max=max)
+    if normalize
+        wmin, wmax = minimum(ws), maximum(ws)
+        ws = (ws .- wmin) ./ (wmax - wmin)
+    end
+    return ws
 end
 
 function compute_weights(ch::Channel;