transgfwlist2acl.js

/*
 * 2017 all rights reserved qinyuhangxiaoxiang@gmail.com
 * this file is help transfor gfwlist.txt to useful acl file
 * about acl see https://github.com/shadowsocks/shadowsocks-libev
 * NOTICE I have see there is lots of wired domain and domain in
 * GFWLIST, I think there is a problem with GFWLIST
 * We should not buy it at all
 * You should try filter useless domian(or poission domain)
 *
 * The default filter will drop all IP in the gfwlist.txt
 * TODO during the transfer the project should check it by send
 * HTTP request to the domain(not the full URL) to check if it is true
 * Also how should the program distinct a site 404 with gfw 404
 *
 */
const fs = require('fs')
const path = require('path')
const http = require('http')
const https = require('https')
function filterNotRule(v){
    return v !== "" &&  v[0] !== "[" && v[0] !== "!" && v[0] !== "@" && !isIP(v)
}
function isIP(v){
    return /(\d{1,3}\.){3}\d{1,3}(:\d+)?/.test(v)
}
function usage(){
    console.error("Usage: ", path.basename(process.argv[0]), path.basename(process.argv[1]), "-i path/to/gfwlist.txt -o path/to/outputACLfile.acl");
}
function reduceList(listArray){
    let ret = Object.create(null)
    listArray.forEach( v => {
        if (!ret[v]){
            ret[v] = true
        }
    })
    return Array.prototype.slice.call(Object.keys(ret))
}
function main() {
    if (process.argv.length < 5) {
        usage();
        return;
    }
    let appArgv = process.argv.slice()
    const argvObj = {
        "-i": "inputFile",
        "-o": "outputFile"
    }
    let argvs = Object.create(null)
    appArgv.forEach( (v, i) => {
        if (argvObj.hasOwnProperty(v)) {
            argvs[argvObj[v]] = appArgv[i+1]
        }
    })
    let str = ""
    fs.readFile( path.resolve(__dirname, argvs["inputFile"]), (err, body) => {
        if (err) { console.error("cannot read file", err);return }
        str = Buffer.from(body.toString(), "base64").toString()
        let arr = str.split("\n")
            .filter( filterNotRule )
            .map( v => {
                while(v[0] === '|' || v[0] === '.'){
                    v = v.slice(1,v.length)
                }
                if (/^http:\/\//.test(v)) {
                    v = v.slice(7,v.lenth);
                }
                if (/^https:\/\//.test(v)) {
                    v = v.slice(8, v.length);
                }
                return "^(.*\.)?" + v.replace(/\./g, "\\.") + "$"
            })
        //console.log(arr)
        let w = fs.createWriteStream(path.resolve(__dirname, argvs["outputFile"]));
        w.write(reduceList(arr).join("\n"))
        // arr.forEach( v => {
        //     console.log(v)
        // })
    })
}
main()