Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

LinkFinder update. #145

Open
wants to merge 4 commits into
base: master
Choose a base branch
from
Open

LinkFinder update. #145

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
568cd62
Input as JS Files and save it in one HTML file
Jan 24, 2024
7b3f50a
ssl_PROTOCOL_TLSv1_2 is deprecated fix
Jan 27, 2024
8faef8c
Update README.md
mrrobot1o1 Jan 27, 2024
68798f8
Update README.md
mrrobot1o1 Jan 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion README.md
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
<img src="https://user-images.githubusercontent.com/18099289/62728809-f98b0900-ba1c-11e9-8dd8-67111263a21f.png" width=650px>
<img src="https://github.com/mrrobot1o1/LinkFinder/assets/66872759/bc6660ed-a84e-4e04-82ed-f9ffe6b7d17e" width=650px>

## About LinkFinder

Expand Down Expand Up @@ -51,6 +51,10 @@ Short Form | Long Form | Description

`python linkfinder.py -i https://example.com/1.js -o results.html`

OR

`python linkfinder.py -i js_urls.txt -o results.html`

* CLI/STDOUT output (doesn't use jsbeautifier, which makes it very fast):

`python linkfinder.py -i https://example.com/1.js -o cli`
Expand Down
82 changes: 58 additions & 24 deletions linkfinder.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,17 @@
os.environ["BROWSER"] = "open"

# Import libraries
import re, sys, glob, html, argparse, jsbeautifier, webbrowser, subprocess, base64, ssl, xml.etree.ElementTree
import re
import sys
import glob
import html
import argparse
import jsbeautifier
import webbrowser
import subprocess
import base64
import ssl
import xml.etree.ElementTree

from gzip import GzipFile
from string import Template
Expand All @@ -25,6 +35,12 @@
except ImportError:
from urllib2 import Request, urlopen

def combine_results(results):
combined_output = ''
for result in results:
combined_output += result
return combined_output

# Regex used
regex_str = r"""

Expand Down Expand Up @@ -77,7 +93,6 @@ def parser_error(errmsg):
print("Error: %s" % errmsg)
sys.exit()


def parser_input(input):
'''
Parse Input
Expand All @@ -98,7 +113,8 @@ def parser_input(input):
items = xml.etree.ElementTree.fromstring(open(args.input, "r").read())

for item in items:
jsfiles.append({"js":base64.b64decode(item.find('response').text).decode('utf-8',"replace"), "url":item.find('url').text})
jsfiles.append({"js": base64.b64decode(item.find('response').text).decode('utf-8', "replace"),
"url": item.find('url').text})
return jsfiles

# Method 4 - Folder with a wildcard
Expand All @@ -114,7 +130,6 @@ def parser_input(input):
return [path if os.path.exists(input) else parser_error("file could not \
be found (maybe you forgot to add http/https).")]


def send_request(url):
'''
Send requests with Requests
Expand All @@ -130,10 +145,16 @@ def send_request(url):
q.add_header('Cookie', args.cookies)

try:
sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1_2)
response = urlopen(q, timeout=args.timeout, context=sslcontext)
if args.insecure:
sslcontext = ssl.create_default_context()
sslcontext.check_hostname = False
sslcontext.verify_mode = ssl.CERT_NONE
response = urlopen(q, timeout=args.timeout, context=sslcontext)
else:
sslcontext = ssl.create_default_context()
response = urlopen(q, timeout=args.timeout, context=sslcontext)
except:
sslcontext = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
sslcontext = ssl.create_default_context()
response = urlopen(q, timeout=args.timeout, context=sslcontext)

if response.info().get('Content-Encoding') == 'gzip':
Expand Down Expand Up @@ -198,7 +219,7 @@ def parser_file(content, regex_str, mode=1, more_regex=None, no_dup=1):
if mode == 1:
# Beautify
if len(content) > 1000000:
content = content.replace(";",";\r\n").replace(",",",\r\n")
content = content.replace(";", ";\r\n").replace(",", ",\r\n")
else:
content = jsbeautifier.beautify(content)

Expand Down Expand Up @@ -288,12 +309,12 @@ def check_url(url):
# Parse command line
parser = argparse.ArgumentParser()
parser.add_argument("-d", "--domain",
help="Input a domain to recursively parse all javascript located in a page",
help="Input a domain to recursively parse all JavaScript located on a page",
action="store_true")
parser.add_argument("-i", "--input",
help="Input a: URL, file or folder. \
For folders a wildcard can be used (e.g. '/*.js').",
required="True", action="store")
required=True, action="store")
parser.add_argument("-o", "--output",
help="Where to save the file, \
including file name. Default: output.html",
Expand All @@ -312,6 +333,9 @@ def check_url(url):
parser.add_argument("-t", "--timeout",
help="How many seconds to wait for the server to send data before giving up (default: " + str(default_timeout) + " seconds)",
default=default_timeout, type=int, metavar="<seconds>")
parser.add_argument("--insecure",
help="Allow insecure SSL connections (disable certificate verification)",
action="store_true")
args = parser.parse_args()

if args.input[-1:] == "/":
Expand All @@ -321,22 +345,28 @@ def check_url(url):
if args.output == "cli":
mode = 0

# Convert input to URLs or JS files
urls = parser_input(args.input)
# Check if input is a URL or a file
if os.path.isfile(args.input):
# Read URLs from the file
with open(args.input, 'r') as file:
urls = file.read().splitlines()
Comment on lines +349 to +352
Copy link
Owner

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This seems to break analyzing JS files locally as the input is not treated as JS but instead split as individual URLs. Prior we would not make any fetch and analyze the content as is. It seems to me analyzing local files is useful so can we revert back to that?

else:
# Single URL provided
urls = [args.input]

# Convert URLs to JS
output = ''
results = []
for url in urls:
if not args.burp:
try:
file = send_request(url)
file_content = send_request(url)
except Exception as e:
parser_error("invalid input defined or SSL error: %s" % e)
else:
file = url['js']
file_content = url['js']
url = url['url']

endpoints = parser_file(file, regex_str, mode, args.regex)
endpoints = parser_file(file_content, regex_str, mode, args.regex)
if args.domain:
for endpoint in endpoints:
endpoint = html.escape(endpoint["link"]).encode('ascii', 'ignore').decode('utf8')
Expand All @@ -346,14 +376,14 @@ def check_url(url):
print("Running against: " + endpoint)
print("")
try:
file = send_request(endpoint)
new_endpoints = parser_file(file, regex_str, mode, args.regex)
file_content = send_request(endpoint)
new_endpoints = parser_file(file_content, regex_str, mode, args.regex)
if args.output == 'cli':
cli_output(new_endpoints)
else:
output += '''
<h1>File: <a href="%s" target="_blank" rel="nofollow noopener noreferrer">%s</a></h1>
''' % (html.escape(endpoint), html.escape(endpoint))
output = '''
<h1>File: <a href="%s" target="_blank" rel="nofollow noopener noreferrer">%s</a></h1>
''' % (html.escape(endpoint), html.escape(endpoint))

for endpoint2 in new_endpoints:
url = html.escape(endpoint2["link"])
Expand All @@ -370,15 +400,16 @@ def check_url(url):
html.escape(endpoint2["link"])
)
output += header + body
results.append(output)
except Exception as e:
print("Invalid input defined or SSL error for: " + endpoint)
continue

if args.output == 'cli':
cli_output(endpoints)
else:
output += '''
<h1>File: <a href="%s" target="_blank" rel="nofollow noopener noreferrer">%s</a></h1>
output = '''
<h1>URL: <a href="%s" target="_blank" rel="nofollow noopener noreferrer">%s</a></h1>
''' % (html.escape(url), html.escape(url))

for endpoint in endpoints:
Expand All @@ -397,6 +428,9 @@ def check_url(url):
)

output += header + body
results.append(output)

if args.output != 'cli':
html_save(output)
html_save(combine_results(results))