Fix missing reference url slash (#26)

fffunction · Nov 17, 2017 · e260967 · e260967
1 parent e6ecdd8
commit e260967
Show file tree

Hide file tree

Showing 4 changed files with 43 additions and 9 deletions.
diff --git a/index.js b/index.js
@@ -50,8 +50,6 @@ if (cli.flags.limitSimilar) {
 }
 
 if (cli.flags.referenceUrl) {
-    // Remove a trailing slash so we don't end up with multiple later
-    cli.flags.referenceUrl = cli.flags.referenceUrl.replace(/\/$/, '')
 
     if (!validurl(cli.flags.referenceUrl)) {
         console.error(

diff --git a/lib/crawl.js b/lib/crawl.js
@@ -9,6 +9,7 @@ const chalk = require('chalk');
 const mkpath = require('mkpath');
 const jsonfile = require('jsonfile');
 const cheerio = require('cheerio');
+const normalurl = require('normalize-url');
 const limitSimilar = require('./limit-similar');
 
 const dirname = path.dirname;
@@ -99,11 +100,22 @@ function crawl(url, flags) {
             // Clone the default scenario and adjust urls as needed
             const currentScenario = Object.assign({}, defaultScenario);
             currentScenario.label = queueItem.path;
+            currentScenario.url = queueItem.url;
+
             if (flags.referenceUrl) {
-                currentScenario.referenceUrl = queueItem.url.replace(
-                    url,
-                    flags.referenceUrl
+                // Normalize the URL to remove duplicate slashes. See #25
+                currentScenario.referenceUrl = normalurl( 
+                    queueItem.url.replace(
+                        url,
+                        flags.referenceUrl
+                    ) 
                 );
+
+                // Add a trailing slash to referenceUrl if url has a trailing slash and referenceUrl does not
+                if ( currentScenario.url.substr(-1) === '/' && currentScenario.referenceUrl.substr(-1) !== '/' ) {
+                    currentScenario.referenceUrl += '/';
+                }
+
             } else if (
                 Object.prototype.hasOwnProperty.call(
                     currentScenario,
@@ -112,7 +124,7 @@ function crawl(url, flags) {
             ) {
                 delete currentScenario.referenceUrl;
             }
-            currentScenario.url = queueItem.url;
+
             urls.push(currentScenario);
         }
     });

diff --git a/package-lock.json b/package-lock.json
diff --git a/package.json b/package.json
@@ -53,6 +53,7 @@
     "jsonfile": "^3.0.0",
     "meow": "3.7.0",
     "mkpath": "1.0.0",
+    "normalize-url": "^1.9.1",
     "object.entries": "1.0.4",
     "ora": "^1.3.0",
     "simplecrawler": "^1.1.3",