Add a test for the recursive queue example in README using cheerio

bda-research · Nov 8, 2014 · 23476c0 · 23476c0
1 parent bb50cf4
commit 23476c0
Show file tree

Hide file tree

Showing 4 changed files with 26 additions and 13 deletions.
diff --git a/lib/crawler.js b/lib/crawler.js
@@ -12,8 +12,6 @@ var zlib = require('zlib');
 var fs = require('fs');
 var Pool = require('generic-pool').Pool;
 
-var debug = require('./debug');
-
 // Fallback on iconv-lite if we didn't succeed compiling iconv
 // https://github.com/sylvinus/node-crawler/pull/29
 var iconv, iconvLite;
@@ -46,7 +44,6 @@ function readJqueryUrl (url, callback) {
             callback(err, jq);
         });
     } else {
-        debug('Unable to load jquery from file', url);
         callback(null, url);
     }
 }

diff --git a/tests/examples.test.js b/tests/examples.test.js
@@ -2,21 +2,19 @@
 
 var Crawler = require('../lib/crawler');
 var expect = require('chai').expect;
-var c;
+var httpbinHost = 'localhost:8000';
+var sinon = require('sinon');
+var url = require('url');
+var c, spy;
 
 describe('Simple test', function() {
-    beforeEach(function() {
-        c = new Crawler({
-            forceUTF8: true
-        });
-    });
     afterEach(function() {
         c = {};
+        spy = {};
     });
-    it('should run the readme examples', function(done) {
+    it('should run the first readme examples', function(done) {
         c = new Crawler({
             maxConnections: 10,
-            jquery: true,
             onDrain: function() {
                 done();
             },
@@ -26,6 +24,24 @@ describe('Simple test', function() {
         });
         c.queue('http://google.com');
     });
+    it('should run the readme examples', function(done) {
+        c = new Crawler({
+            maxConnections: 10,
+            onDrain: function() {
+                expect(spy.calledTwice).to.be.true;
+                done();
+            },
+            callback: function(error, result, $) {
+                var baseUrl = result.uri;
+                $('a').each(function(index, a) {
+                    var toQueueUrl = url.resolve(baseUrl, $(a).attr('href'));
+                    c.queue(toQueueUrl);
+                });
+            }
+        });
+        spy = sinon.spy(c, 'queue');
+        c.queue('http://'+httpbinHost+'/links/1/1');
+    });
     it('should run the with an array queue', function(done) {
         c = new Crawler();
         c.queue([{

diff --git a/tests/linksResolving.test.js b/tests/linksResolving.test.js
@@ -31,7 +31,7 @@ describe('Links', function() {
             }
         }]);
     });
-    it('should resolved links to absolute urls after redirect', function(done) {
+    it('should resolved links to absolute urls after redirect with jsdom', function(done) {
         c.queue([{
             uri : 'http://'+httpbinHost+'/redirect-to?url=http://example.com/',
             callback: function(error, result) {

diff --git a/tests/uriOption.test.js b/tests/uriOption.test.js
@@ -5,7 +5,7 @@ var expect = require('chai').expect;
 var httpbinHost = 'localhost:8000';
 var c;
 
-describe('Simple test', function() {
+describe('Uri Options', function() {
     afterEach(function() {
         c = {};
     });