Merge pull request #1913 from presidentbeef/lazy_file_lists

Use lazy file lists in AppTree

Author: presidentbeef

lib/brakeman/app_tree.rb

@@ -145,6 +145,17 @@ def gemspec
       end
     end
 
+
+    # Call this to be able to marshall the AppTree
+    def marshallable
+      @initializer_paths = @initializer_paths.to_a
+      @controller_paths = @controller_paths.to_a
+      @template_paths = @template_paths.to_a
+      @lib_files = @file_paths.to_a
+
+      self
+    end
+
   private
 
     def find_helper_paths
@@ -160,7 +171,7 @@ def find_additional_lib_paths
     end
 
     def find_paths(directory, extensions = ".rb")
-      select_files(glob_files(directory, "*", extensions).sort)
+      select_files(glob_files(directory, "*", extensions))
     end
 
     def glob_files(directory, name, extensions = ".rb")
@@ -179,10 +190,10 @@ def glob_files(directory, name, extensions = ".rb")
         end
 
         files = patterns.flat_map { |pattern| Dir.glob(pattern) }
-        files.uniq
+        files.uniq.lazy
       else
         pattern = "#{root_search_pattern}#{directory}/**/#{name}#{extensions}"
-        Dir.glob(pattern)
+        Dir.glob(pattern).lazy
       end
     end
 
@@ -191,7 +202,8 @@ def select_files(paths)
       paths = reject_skipped_files(paths)
       paths = convert_to_file_paths(paths)
       paths = reject_global_excludes(paths)
-      reject_directories(paths)
+      paths = reject_directories(paths)
+      paths
     end
 
     def reject_directories(paths)

lib/brakeman/tracker.rb

@@ -441,4 +441,10 @@ def reset_initializer path
 
     @call_index.remove_indexes_by_file path
   end
+
+  # Call this to be able to marshal the Tracker
+  def marshallable
+    @app_tree.marshallable
+    self
+  end
 end

test/test.rb

@@ -186,7 +186,7 @@ def before_rescan_of changed, app = "rails3.2", options = {}
       yield dir if block_given?
 
       # Not really sure why we do this..?
-      t = Marshal.load(Marshal.dump(@original))
+      t = Marshal.load(Marshal.dump(@original.marshallable))
 
       @rescanner = Brakeman::Rescanner.new(t.options, t.processor, changed)
       @rescan = @rescanner.recheck

test/tests/app_tree.rb

@@ -51,77 +51,77 @@ def temp_dir_relative_symlink_and_file_from_path(relative_path)
   def test_directory_absolute_symlink_support
     temp_dir_absolute_symlink_and_file_from_path("test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, follow_symlinks: true)
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_directory_relative_symlink_support
     temp_dir_relative_symlink_and_file_from_path("test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, follow_symlinks: true)
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_directory_relative_disabled_symlink_support
     temp_dir_relative_symlink_and_file_from_path("test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, follow_symlinks: false)
-      assert_empty at.ruby_file_paths.collect(&:absolute)
+      assert_empty at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths
     temp_dir_and_file_from_path("test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir)
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_skip_vendor_false
     temp_dir_and_file_from_path("vendor/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :skip_vendor => false)
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_skip_vendor_true
     temp_dir_and_file_from_path("vendor/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :skip_vendor => true)
-      assert_equal [], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_skip_vendor_true_add_libs_path
     temp_dir_and_file_from_path("vendor/bundle/gems/gem123/lib/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :skip_vendor => true, :additional_libs_path => ["vendor/bundle/gems/gem123/lib"])
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_skip_vendor_true_add_engine_path
     temp_dir_and_file_from_path("vendor/bundle/gems/gem123/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :skip_vendor => true, :engine_paths => ["vendor/bundle/gems"])
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_skip_vendor_true_tests_in_engine_path_still_excluded
     temp_dir_and_file_from_path("vendor/bundle/gems/gem123/test/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :skip_vendor => true, :engine_paths => ["vendor/bundler/gems"])
-      assert_equal [], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_add_engine_path
     temp_dir_and_file_from_path("lib/gems/gem123/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :engine_paths => ["lib/gems"])
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
   def test_ruby_file_paths_add_libs_path
     temp_dir_and_file_from_path("gem123/lib/test.rb") do |dir, file|
       at = Brakeman::AppTree.new(dir, :additional_libs_path => ["gems/gem123/lib"])
-      assert_equal [file], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [file], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 
@@ -130,7 +130,7 @@ def test_ruby_file_paths_directory_with_rb_extension
       FileUtils.mkdir_p(File.join(dir, "test.rb"))
 
       at = Brakeman::AppTree.new(dir)
-      assert_equal [], at.ruby_file_paths.collect(&:absolute)
+      assert_equal [], at.ruby_file_paths.collect(&:absolute).to_a
     end
   end
 end