Update of examples and streaming weighted sampling

Author: emmanuellujan

examples/simple-example/simple-example.jl

@@ -1,5 +1,5 @@
 using StreamingSampling
-using StatsBase
+using StatsBase: sample, Weights
 
 # Define file paths
 base = haskey(ENV, "BASE_PATH") ? ENV["BASE_PATH"] : "../../"
@@ -9,22 +9,14 @@ file_paths = ["$path/data1.txt",
               "$path/data3.txt",
               "$path/data4.txt"];
 
-# Compute streaming weights
-ws = compute_weights(file_paths; chunksize=500, subchunksize=100)
-
 # Define sample size
 n = 100;
 
-# Compute first-order inclusion probabilities with sample size n
-ps = inclusion_prob(ws, n)
-
-# Option 1: Sample by weighted sampling
-inds_w = StatsBase.sample(1:length(ws), Weights(ws), n; replace=false)
-
-# Option 2: Sample by weighted sampling and first-order inclusion probabilities
-inds_p = StatsBase.sample(1:length(ps), Weights(ps), n; replace=false)
+# Streaming weighted sampling
+ws = compute_weights(file_paths; chunksize=500, subchunksize=100)
+inds_w = sample(1:length(ws), Weights(ws), n; replace=false)
 
-# Option 3: Sample by UPmaxentropy
-s = UPmaxentropy(ps)
+# Streaming maximum entropy sampling
+s = UPmaxentropy(inclusion_prob(ws, n))
 inds_me = findall(s .== 1)
 

src/StreamWeights.jl

@@ -0,0 +1,54 @@
+# Streaming Weights
+
+mutable struct StreamWeights
+    weights::Vector{Float64}
+    chunksize::Int
+    subchunksize::Int
+
+    function StreamWeights(file_paths::Vector{String}; 
+                              read_element=read_element,
+                              create_feature=create_feature,
+                              chunksize=2000,
+                              subchunksize=200,
+                              buffersize=32,
+                              max=Inf,
+                              randomized=true)
+        ws = new(Vector{Float64}(),
+                 chunksize,
+                 subchunksize)
+        sampler.weights = compute_weights(sampler, file_paths;
+                                          read_element=read_element,
+                                          create_feature=create_feature,
+                                          chunksize=chunksize,
+                                          subchunksize=subchunksize,
+                                          buffersize=buffersize,
+                                          max=max,
+                                          randomized=randomized)
+        return ws
+    end
+    
+    function StreamWeights(A::Vector;
+                              read_element=read_element,
+                              create_feature=create_feature,
+                              chunksize=2000,
+                              subchunksize=200,
+                              buffersize=32,
+                              max=Inf,
+                              randomized=true)
+        ws = new(Vector{Float64}(),
+                 chunksize,
+                 subchunksize)
+        ws.weights = compute_weights(sampler, A;
+                                     read_element=read_element,
+                                     create_feature=create_feature,
+                                     chunksize=chunksize,
+                                     subchunksize=subchunksize,
+                                     buffersize=buffersize,
+                                     max=max,
+                                     randomized=randomized)
+        return ws
+    end
+end
+
+
+

src/StreamingSampling.jl

@@ -31,7 +31,7 @@ include("RelFreqs.jl")
 
 # Sampling
 include("UPmaxentropy.jl")
-#include("StreamWeights.jl")
+include("StreamWeights.jl")
 include("StreamMaxEnt.jl")
 include("Sampling.jl")