v0.4.0

CHANGELOG.md

@@ -0,0 +1,3 @@
+# Synt Changelog
+
+Please see the GithHub [releases](https://github.com/brentlintner/synt/releases) section.

README.md

@@ -1,5 +1,7 @@
 # synt [![Circle CI](https://circleci.com/gh/brentlintner/synt.svg?style=shield)](https://circleci.com/gh/brentlintner/synt) [![score-badge](https://vile.io/api/v0/projects/synt/badges/score?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![security-badge](https://vile.io/api/v0/projects/synt/badges/security?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![coverage-badge](https://vile.io/api/v0/projects/synt/badges/coverage?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![dependency-badge](https://vile.io/api/v0/projects/synt/badges/dependency?token=USryyHar5xQs7cBjNUdZ)](https://vile.io/~brentlintner/synt) [![npm version](https://badge.fury.io/js/synt.svg)](https://badge.fury.io/js/synt)
 
+![demo image](https://user-images.githubusercontent.com/93340/26853130-c50f2724-4ade-11e7-905e-6923af2a759d.png)
+
 Find similar functions and classes in your JavaScript/TypeScript code.
 
 ## Supported Languages
@@ -34,7 +36,9 @@ synt -h
 *example*
 
 ```sh
-synt analyze lib
+git clone https://github.com/brentlintner/synt.git
+cd synt
+synt analyze src
 ```
 
 ### Library

lib/cli.js

@@ -0,0 +1,43 @@
+"use strict";
+var program = require("commander");
+var similar = require("./similar");
+var fs_collector = require("./cli/file_collector");
+var pkg = require("./../package.json");
+var compare = function (targets, opts) {
+    var files = fs_collector.files(targets);
+    var nocolors = !!opts.disablecolors;
+    fs_collector.print(files, nocolors);
+    var _a = similar.compare(files, opts), js = _a.js, ts = _a.ts;
+    similar.print(js, nocolors);
+    similar.print(ts, nocolors);
+};
+var configure = function () {
+    program
+        .version(pkg.version)
+        .command("analyze [paths...]")
+        .alias("a")
+        .option("-s, --similarity [number]", "Lowest % similarity to look for " +
+        ("[default=" + similar.DEFAULT_THRESHOLD + "]."))
+        .option("-m, --minlength [number]", "Default token length a function needs to be to compare it " +
+        ("[default=" + similar.DEFAULT_TOKEN_LENGTH + "]."))
+        .option("-n, --ngram [number]", "Specify ngram length for comparing token sequences. " +
+        ("[default=" + similar.DEFAULT_NGRAM_LENGTH + ",2,3...]"))
+        .option("-d, --disablecolors", "Disable color output")
+        .action(compare);
+    program.on("--help", function () {
+        console.log("  Command specific help:");
+        console.log("");
+        console.log("    {cmd} -h, --help");
+        console.log("");
+        console.log("  Examples:");
+        console.log("");
+        console.log("    $ synt analyze lib");
+        console.log("    $ synt analyze -s 90 foo.js bar.js baz.js");
+        console.log("");
+    });
+};
+var interpret = function (argv) {
+    configure();
+    program.parse(argv);
+};
+module.exports = { interpret: interpret };

lib/cli/file_collector.js

@@ -0,0 +1,41 @@
+"use strict";
+var fs = require("fs");
+var path = require("path");
+var _ = require("lodash");
+var chalk = require("chalk");
+var walk_sync = require("walk-sync");
+var all_files = function (target) {
+    if (fs.statSync(target).isDirectory()) {
+        var dirs = walk_sync(target, { directories: false });
+        return _.map(dirs, function (dir) { return path.join(target, dir); });
+    }
+    else {
+        return [target];
+    }
+};
+var normalize_cli_targets = function (targets) {
+    targets = _.concat([], targets);
+    var files = _.uniq(_.reduce(targets, function (paths, target) {
+        return _.concat(paths, all_files(target));
+    }, []));
+    files = _.map(files, function (file) {
+        return path.relative(process.cwd(), file);
+    });
+    return _.filter(files, function (file) {
+        return /\.(js|ts)$/.test(file);
+    });
+};
+var print_found = function (files, nocolors) {
+    return _.each(files, function (file) {
+        if (nocolors) {
+            console.log("found:", file);
+        }
+        else {
+            console.log(chalk.gray("found:"), chalk.green(file));
+        }
+    });
+};
+module.exports = {
+    files: normalize_cli_targets,
+    print: print_found
+};

lib/index.js

@@ -0,0 +1,3 @@
+"use strict";
+var similar = require("./similar");
+module.exports = similar;

lib/similar.js

@@ -0,0 +1,138 @@
+"use strict";
+var _ = require("lodash");
+var ngram = require("./similar/ngram");
+var parse_js = require("./similar/javascript");
+var parse_ts = require("./similar/typescript");
+var print = require("./similar/print");
+var DEFAULT_NGRAM_LENGTH = 1;
+var DEFAULT_THRESHOLD = 70;
+var DEFAULT_TOKEN_LENGTH = 10;
+var similarity = function (src, cmp) {
+    var a = _.uniq(src);
+    var b = _.uniq(cmp);
+    var i = _.intersection(a, b);
+    var u = _.union(a, b);
+    return _.toNumber(_.toNumber((i.length / u.length) * 100)
+        .toFixed(0));
+};
+var parse_token_length = function (str) {
+    return _.isEmpty(str) ?
+        DEFAULT_TOKEN_LENGTH :
+        _.toNumber(str);
+};
+var parse_ngram_length = function (str) {
+    return _.isEmpty(str) ?
+        DEFAULT_NGRAM_LENGTH :
+        _.toNumber(str);
+};
+var parse_threshold = function (str) {
+    var threshold = _.toNumber(str);
+    return threshold || DEFAULT_THRESHOLD;
+};
+var is_ts_ancestor = function (src, cmp) {
+    var match = false;
+    var last = src.ast;
+    while (true) {
+        var parent_1 = last.parent;
+        if (parent_1 === cmp.ast)
+            match = true;
+        if (!parent_1 || last === parent_1 || match)
+            break;
+        last = parent_1;
+    }
+    return match;
+};
+var false_positive = function (src, cmp, t_len) {
+    var same_node = function () { return cmp.ast === src.ast; };
+    var size_is_too_different = function () {
+        var l1 = src.tokens.length;
+        var l2 = cmp.tokens.length;
+        return l1 * 2 < l2 || l2 * 2 < l1;
+    };
+    var one_is_too_short = function () {
+        return src.tokens.length < t_len ||
+            cmp.tokens.length < t_len;
+    };
+    var subset_of_other = function () {
+        var is_eithers_ancestor = function () {
+            return is_ts_ancestor(src, cmp) ||
+                is_ts_ancestor(cmp, src);
+        };
+        var is_eithers_middle = function () {
+            var src_j = src.tokens.join("");
+            var cmp_j = cmp.tokens.join("");
+            return src_j !== cmp_j &&
+                (_.includes(src_j, cmp_j) ||
+                    _.includes(cmp_j, src_j));
+        };
+        return is_eithers_ancestor() || is_eithers_middle();
+    };
+    var both_are_not_classes = function () {
+        return (src.is_class && !cmp.is_class) ||
+            (!src.is_class && cmp.is_class);
+    };
+    return same_node() ||
+        both_are_not_classes() ||
+        subset_of_other() ||
+        one_is_too_short() ||
+        size_is_too_different();
+};
+var each_pair = function (items, callback) {
+    _.each(items, function (src) {
+        _.each(items, function (cmp) {
+            callback(src, cmp);
+        });
+    });
+};
+var filter_redundencies = function (group) {
+    _.each(group, function (results, sim) {
+        group[sim] = _.reduce(results, function (new_arr, result) {
+            var already_added = _.some(new_arr, function (result_two) {
+                return _.xor(result, result_two).length === 0;
+            });
+            if (!already_added) {
+                new_arr.push(result);
+            }
+            return new_arr;
+        }, []);
+    });
+    return group;
+};
+var _compare = function (files, ftype, n_len, t_len, sim_min) {
+    var is_ts = ftype === "ts";
+    var is_file = is_ts ? /\.ts$/ : /\.js$/;
+    files = _.filter(files, function (file) { return is_file.test(file); });
+    var parse = is_ts ? parse_ts : parse_js;
+    var items = parse.find(files);
+    var group = {};
+    each_pair(items, function (src, cmp) {
+        if (false_positive(src, cmp, t_len))
+            return;
+        var src_grams = ngram.generate(src.tokens, n_len);
+        var cmp_grams = ngram.generate(cmp.tokens, n_len);
+        var val = similarity(src_grams, cmp_grams);
+        if (val < sim_min)
+            return;
+        if (_.isEmpty(group[val]))
+            group[val] = [];
+        group[val].push([src, cmp]);
+    });
+    return filter_redundencies(group);
+};
+var compare = function (files, opts) {
+    if (opts === void 0) { opts = {}; }
+    files = _.concat([], files);
+    var t_len = parse_token_length(_.toString(opts.minlength));
+    var n_len = parse_ngram_length(_.toString(opts.ngram));
+    var threshold = parse_threshold(_.toString(opts.similarity));
+    var js_group = _compare(files, "js", n_len, t_len, threshold);
+    var ts_group = _compare(files, "ts", n_len, t_len, threshold);
+    return { js: js_group, ts: ts_group };
+};
+module.exports = {
+    DEFAULT_NGRAM_LENGTH: DEFAULT_NGRAM_LENGTH,
+    DEFAULT_THRESHOLD: DEFAULT_THRESHOLD,
+    DEFAULT_TOKEN_LENGTH: DEFAULT_TOKEN_LENGTH,
+    compare: compare,
+    print: print.print
+};

lib/similar/javascript.js

@@ -0,0 +1,61 @@
+"use strict";
+var fs = require("fs");
+var _ = require("lodash");
+var codegen = require("escodegen");
+var esprima = require("esprima");
+var estraverse = require("estraverse");
+var FUNCTION_OR_CLASS_NODE = [
+    "ArrowFunctionExpression",
+    "ClassDeclaration",
+    "FunctionDeclaration",
+    "FunctionExpression"
+];
+var normalize = function (token_list) {
+    return _.map(token_list, function (t) { return t.value; });
+};
+var tokenize = function (code) {
+    return normalize(esprima.tokenize(code));
+};
+var astify = function (code) {
+    return esprima.parse(code, { loc: true });
+};
+var ast_to_code = function (node) {
+    var opts = { format: { indent: { style: "  " } } };
+    return codegen.generate(node, opts);
+};
+var is_a_method_or_class = function (node) {
+    return _.some(FUNCTION_OR_CLASS_NODE, function (type) { return type === node.type; });
+};
+var line_info = function (node) { return node.loc; };
+var parse_methods_and_classes = function (root_node, filepath) {
+    var entries = [];
+    estraverse.traverse(root_node, {
+        enter: function (node, parent) {
+            if (!is_a_method_or_class(node))
+                return;
+            var method = ast_to_code(node);
+            var tokens = tokenize(method);
+            var result = {
+                ast: node,
+                code: method,
+                is_class: node.type === "ClassDeclaration",
+                path: filepath,
+                pos: line_info(node),
+                tokens: tokens,
+                type: node.type
+            };
+            entries.push(result);
+        }
+    });
+    return entries;
+};
+var find_similar_methods_and_classes = function (filepaths) {
+    return _.flatMap(filepaths, function (filepath) {
+        var code = fs.readFileSync(filepath).toString();
+        var node = astify(code);
+        return parse_methods_and_classes(node, filepath);
+    });
+};
+module.exports = {
+    find: find_similar_methods_and_classes
+};

lib/similar/ngram.js

@@ -0,0 +1,17 @@
+"use strict";
+var generate = function (arr, len) {
+    if (len === void 0) { len = 1; }
+    if (len > arr.length)
+        len = 1;
+    if (len == 1)
+        return arr;
+    var sets = [];
+    arr.forEach(function (token, index) {
+        var s_len = index + len;
+        if (s_len <= arr.length) {
+            sets.push(arr.slice(index, s_len).join(""));
+        }
+    });
+    return sets;
+};
+module.exports = { generate: generate };

lib/similar/print.js

@@ -0,0 +1,56 @@
+"use strict";
+var _ = require("lodash");
+var chalk = require("chalk");
+var cardinal = require("cardinal");
+var print = function (group, nocolors) {
+    _.each(group, function (results, sim) {
+        _.each(results, function (result) {
+            var src = result[0], cmp = result[1];
+            console.log("");
+            var match_sim = sim + "% match";
+            if (nocolors) {
+                console.log(match_sim);
+            }
+            else {
+                console.log(chalk.white.bgRed.bold(match_sim));
+            }
+            console.log("");
+            if (nocolors) {
+                console.log("in: " + src.path);
+            }
+            else {
+                console.log(chalk.gray("in: ") + chalk.green(src.path));
+            }
+            console.log("");
+            if (nocolors) {
+                console.log(src.code);
+            }
+            else {
+                console.log(cardinal.highlight(src.code, {
+                    firstline: src.pos.start.line,
+                    linenos: true
+                }));
+            }
+            console.log("");
+            if (src.path !== cmp.path) {
+                if (nocolors) {
+                    console.log("in: " + cmp.path);
+                }
+                else {
+                    console.log(chalk.gray("in: ") + chalk.green(cmp.path));
+                }
+                console.log("");
+            }
+            if (nocolors) {
+                console.log(cmp.code);
+            }
+            else {
+                console.log(cardinal.highlight(cmp.code, {
+                    firstline: cmp.pos.start.line,
+                    linenos: true
+                }));
+            }
+        });
+    });
+};
+module.exports = { print: print };