prevents explosion of global queue

dchuk · Jan 17, 2015 · 3bf7d66 · 3bf7d66
1 parent d3e56c8
commit 3bf7d66
Showing 1 changed file with 3 additions and 4 deletions.
diff --git a/lib/arachnid.rb b/lib/arachnid.rb
@@ -34,9 +34,8 @@ def crawl(options = {})
 		@global_queue << @start_url
 
 		while(@global_queue.size > 0 && (max_urls.nil? || @global_visited.size.to_i < max_urls))
-			temp_queue = @global_queue
-
-			temp_queue.each do |q|
+			(0..@global_queue.length).each do |i|
+        q = @global_queue[i]
 
 				begin
 					ip,port,user,pass = grab_proxy
@@ -59,7 +58,7 @@ def crawl(options = {})
 
 									absolute_link = make_absolute(sanitized_link, response.effective_url)
 									if(absolute_link)
-										@global_queue << absolute_link
+										(@global_queue << absolute_link) unless @global_queue.include?(absolute_link)
 									end
 								end
 							end