From 307fdf8064220f43ec5c91c4b828be4ec9c6fcc7 Mon Sep 17 00:00:00 2001 From: c_sto <7466346+C-Sto@users.noreply.github.com> Date: Sat, 27 Oct 2018 11:56:59 +0800 Subject: [PATCH] fix nasty soft 404 bug --- librecursebuster/logic.go | 5 +- librecursebuster/main_test.go | 114 ++++++++++++++++++---------- librecursebuster/net.go | 27 +++---- librecursebuster/testserver/main.go | 17 ++++- librecursebuster/util.go | 18 +++-- main.go | 2 +- 6 files changed, 118 insertions(+), 65 deletions(-) diff --git a/librecursebuster/logic.go b/librecursebuster/logic.go index 82d37e2..ed37657 100644 --- a/librecursebuster/logic.go +++ b/librecursebuster/logic.go @@ -1,6 +1,7 @@ package librecursebuster import ( + "bytes" "fmt" "io/ioutil" "net/http" @@ -171,7 +172,8 @@ func (gState *State) dirBust(page SpiderPage) { h, _, res := gState.evaluateURL(gState.Methods[0], page.URL+RandString(), gState.Client) //fmt.Println(page.URL, h, res) if res { //true response indicates a good response for a guid path, unlikely good - if detectSoft404(h, gState.Hosts.Get404(u.Host), gState.Cfg.Ratio404) { + is404, _ := detectSoft404(h, gState.Hosts.Get404(u.Host), gState.Cfg.Ratio404) + if is404 { //it's a soft404 probably, guess we can continue (this logic seems wrong??) } else { gState.PrintOutput( @@ -277,6 +279,7 @@ func (gState *State) StartBusting(randURL string, u url.URL) { Debug, 2, ) content, _ := ioutil.ReadAll(resp.Body) + resp.Body = ioutil.NopCloser(bytes.NewBuffer(content)) gState.Hosts.AddSoft404Content(u.Host, content, resp) // Soft404ResponseBody = xx } else { <-gState.Chans.workersChan diff --git a/librecursebuster/main_test.go b/librecursebuster/main_test.go index 5ade99c..6e9f280 100644 --- a/librecursebuster/main_test.go +++ b/librecursebuster/main_test.go @@ -2,6 +2,7 @@ package librecursebuster import ( "fmt" + "net/http" "net/url" "os" "strings" @@ -59,8 +60,8 @@ func TestBasicFunctionality(t *testing.T) { } for _, i := range ok200 { tested = append(tested, i) - if x, ok := found[i]; !ok || !x { - panic("Did not find " + i) + if x, ok := found[i]; !ok || x == nil { + t.Error("Did not find " + i) } } ok300 := []string{ @@ -68,8 +69,8 @@ func TestBasicFunctionality(t *testing.T) { } for _, i := range ok300 { tested = append(tested, i) - if x, ok := found[i]; !ok || !x { - panic("Did not find " + i) + if x, ok := found[i]; !ok || x == nil { + t.Error("Did not find " + i) } } ok400 := []string{ @@ -77,8 +78,8 @@ func TestBasicFunctionality(t *testing.T) { } for _, i := range ok400 { tested = append(tested, i) - if x, ok := found[i]; !ok || !x { - panic("Did not find " + i) + if x, ok := found[i]; !ok || x == nil { + t.Error("Did not find " + i) } } ok500 := []string{ @@ -86,17 +87,26 @@ func TestBasicFunctionality(t *testing.T) { } for _, i := range ok500 { tested = append(tested, i) - if x, ok := found[i]; !ok || !x { - panic("Did not find " + i) + if x, ok := found[i]; !ok || x == nil { + t.Error("Did not find " + i) } } //check for values that should not have been found for k := range found { if strings.Contains(k, "z") { - panic("Found (but should not have) " + k) + t.Error("Found (but should not have) " + k) } } + + if x, ok := found["/a/x"]; ok && x != nil { + t.Error("Found (but should not have) /a/x") + } + + if x, ok := found["/a/y"]; ok && x != nil { + t.Error("Found (but should not have) /a/x") + } + close(finished) } @@ -111,8 +121,8 @@ func TestAppendSlash(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/appendslash/"]; !ok || !x { - panic("didn't find it?") + if x, ok := found["/appendslash/"]; !ok || x == nil { + t.Error("didn't find it?") } close(finished) } @@ -128,8 +138,8 @@ func TestBasicAuth(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/a/b/c/basicauth"]; !ok || !x { - panic("Failed basic auth test!") + if x, ok := found["/a/b/c/basicauth"]; !ok || x == nil { + t.Error("Failed basic auth test!") } } @@ -146,7 +156,7 @@ func TestBadCodes(t *testing.T) { for x := range found { if strings.Contains(x, "badcode") { - panic("Failed bad header code test") + t.Error("Failed bad header code test") } } @@ -165,7 +175,7 @@ func TestBadHeaders(t *testing.T) { for x := range found { if strings.Contains(x, "badheader") { - panic("Failed bad header code test") + t.Error("Failed bad header code test") } } @@ -184,16 +194,16 @@ func TestAjax(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/ajaxonly"]; !ok || !x { - panic("Failed ajax header check") + if x, ok := found["/ajaxonly"]; !ok || x == nil { + t.Error("Failed ajax header check 1") } - if x, ok := found["/ajaxpost"]; !ok || !x { - panic("Failed ajax header check") + if x, ok := found["/ajaxpost"]; !ok || x == nil { + t.Error("Failed ajax header check 2") } - if x, ok := found["/onlynoajax"]; ok || x { - panic("Failed ajax header check") + if x, ok := found["/onlynoajax"]; ok && x != nil { + t.Error("Failed ajax header check 3") } } @@ -211,8 +221,8 @@ func TestBodyContent(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/postbody"]; !ok || !x { - panic("Failed body based request") + if x, ok := found["/postbody"]; !ok || x == nil { + t.Error("Failed body based request") } } @@ -227,12 +237,12 @@ func TestBlacklist(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/a/b"]; ok || x { - panic("Failed blacklist testing1") + if x, ok := found["/a/b"]; ok && x != nil { + t.Error("Failed blacklist testing1") } - if x, ok := found["/a/b/c"]; ok || x { - panic("Failed blacklist testing2") + if x, ok := found["/a/b/c"]; ok && x != nil { + t.Error("Failed blacklist testing2") } } @@ -246,8 +256,8 @@ func TestCookies(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/cookiesonly"]; !ok || !x { - panic("Failed Cookie test") + if x, ok := found["/cookiesonly"]; !ok || x == nil { + t.Error("Failed Cookie test") } } func TestExt(t *testing.T) { @@ -259,20 +269,48 @@ func TestExt(t *testing.T) { found := postSetupTest(urlSlice, gState) gState.Wait() - if x, ok := found["/a.exe"]; !ok || !x { - panic("Failed Ext test1") + if x, ok := found["/a.exe"]; !ok || x == nil { + t.Error("Failed Ext test1") + } + + if x, ok := found["/a.aspx"]; !ok || x == nil { + t.Error("Failed Ext test2") + } + + if x, ok := found["/a.csv"]; !ok || x == nil { + t.Error("Failed Ext test3") } +} + +func TestOutAll(t *testing.T) { + // sets all responses as 'found' for the purposes of output. Should not pass any logic tests for adding additional URLs etc + t.Parallel() + finished := make(chan struct{}) + cfg := getDefaultConfig() + cfg.ShowAll = true + gState, urlsSlice := preSetupTest(cfg, "2011", finished, t) + found := postSetupTest(urlsSlice, gState) + gState.Wait() - if x, ok := found["/a.aspx"]; !ok || !x { - panic("Failed Ext test2") + //Check the 404's were received and set as found + if x, ok := found["/a/x/c"]; ok && x != nil { + fmt.Println("/a/x/c:", x) + fmt.Println("/a/x", found["/a/x"]) + t.Error("Failed OutAll Test 1, performed check on non-existent prefix") } - if x, ok := found["/a.csv"]; !ok || !x { - panic("Failed Ext test3") + if x, ok := found["/x"]; ok && x != nil { + //have 404 response in found set + if x.StatusCode != 404 { + t.Error("Failed OutAll Test 3, got unexpected response recorded for 404 check") + } + } else { + //didn't have '/a/x' (soft 404) in found set + t.Error("Failed OutAll Test 2, did not have 404 response in found set") } } -func postSetupTest(urlSlice []string, gState *State) (found map[string]bool) { +func postSetupTest(urlSlice []string, gState *State) (found map[string]*http.Response) { //start up the management goroutines go gState.ManageRequests() go gState.ManageNewURLs() @@ -306,7 +344,7 @@ func postSetupTest(urlSlice []string, gState *State) (found map[string]bool) { }() //use the found map to determine later on if we have found the expected URL's - found = make(map[string]bool) + found = make(map[string]*http.Response) go func() { t := time.NewTicker(1 * time.Second).C for { @@ -318,7 +356,7 @@ func postSetupTest(urlSlice []string, gState *State) (found map[string]bool) { gState.wg.Done() panic(e) } - found[u.Path] = true + found[u.Path] = x.Result gState.wg.Done() //fmt.Println("CONFIRMED!", x) case <-t: diff --git a/librecursebuster/net.go b/librecursebuster/net.go index 2b9c08c..cb8e127 100644 --- a/librecursebuster/net.go +++ b/librecursebuster/net.go @@ -101,6 +101,7 @@ func (gState *State) HTTPReq(method, path string, client *http.Client) (resp *ht } defer resp.Body.Close() + //Set body to be readable again body, err := ioutil.ReadAll(resp.Body) if err != nil { return resp, err @@ -111,9 +112,7 @@ func (gState *State) HTTPReq(method, path string, client *http.Client) (resp *ht } func (gState *State) evaluateURL(method string, urlString string, client *http.Client) (headResp *http.Response, content []byte, success bool) { - success = true - //wg.Add(1) - //PrintOutput("EVALUATING:"+method+":"+urlString, Debug, 4, wg, printChan) + //optimize GET requests by sending a head first (it's cheaper) if method == "GET" && !gState.Cfg.NoHead { headResp, err := gState.HTTPReq("HEAD", urlString, client) //send a HEAD. Ignore body response @@ -143,6 +142,7 @@ func (gState *State) evaluateURL(method string, urlString string, client *http.C headResp, err := gState.HTTPReq(method, urlString, client) content, _ = ioutil.ReadAll(headResp.Body) + headResp.Body = ioutil.NopCloser(bytes.NewBuffer(content)) <-gState.Chans.workersChan //done with the net thread if err != nil { success = false @@ -151,12 +151,13 @@ func (gState *State) evaluateURL(method string, urlString string, client *http.C return headResp, content, success } - //Check if we care about it (header only) section + //Check if we care about it (response code only) section if gState.BadResponses[headResp.StatusCode] { success = false return headResp, content, success } + //check for bad headers in the response if len(gState.Cfg.BadHeader) > 0 { for _, x := range gState.Cfg.BadHeader { spl := strings.Split(x, ":") @@ -171,24 +172,18 @@ func (gState *State) evaluateURL(method string, urlString string, client *http.C } } } - //if gState.BadHeaders[headResp.Header.] - - //get content from validated path/file thing - if gState.Cfg.BurpMode { - gState.HTTPReq(method, urlString, gState.BurpClient) - } //check we care about it (body only) section //double check that it's not 404/error using smart blockchain AI tech - gState.PrintOutput( - fmt.Sprintf("Checking body for 404:\nContent: %v,\nSoft404:%v,\nResponse:%v", - string(content), string(gState.Hosts.Get404Body(headResp.Request.Host)), - detectSoft404(headResp, gState.Hosts.Get404(headResp.Request.Host), gState.Cfg.Ratio404)), - Debug, 4) - if detectSoft404(headResp, gState.Hosts.Get404(headResp.Request.Host), gState.Cfg.Ratio404) { + is404, _ := detectSoft404(headResp, gState.Hosts.Get404(headResp.Request.Host), gState.Cfg.Ratio404) + if is404 { //seems to be a soft 404 lol return headResp, content, false } + + if gState.Cfg.BurpMode { + gState.HTTPReq(method, urlString, gState.BurpClient) + } return headResp, content, true } diff --git a/librecursebuster/testserver/main.go b/librecursebuster/testserver/main.go index e7cc2c9..8e43970 100644 --- a/librecursebuster/testserver/main.go +++ b/librecursebuster/testserver/main.go @@ -53,6 +53,7 @@ y const bod404 = `404 not found 20/20/19` const bod404mod = `404 not found 20/20/20` const bod200 = `200ish response! This should be different enough that it is not detected as being a soft 404, ideally anyway.` +const bodNeither = `Totally different response indicating soemthing interesting, but probably not a 404` func (ts *TestServer) handler(w http.ResponseWriter, r *http.Request) { @@ -154,6 +155,7 @@ func (ts *TestServer) handler(w http.ResponseWriter, r *http.Request) { case "/postbody": if r.Method == "POST" && r.Body != nil { bod, err := ioutil.ReadAll(r.Body) + r.Body = ioutil.NopCloser(bytes.NewBuffer(bod)) if err != nil { panic(err) } @@ -169,21 +171,29 @@ func (ts *TestServer) handler(w http.ResponseWriter, r *http.Request) { default: respCode = 404 } - bod := bod404 + + bod := bodNeither if respCode == 200 { bod = bod200 - } else if strings.ToLower(string(r.URL.Path[len(r.URL.Path)-1])) == "x" { + } + if strings.HasSuffix(r.URL.Path, "/x") { // strings.ToLower(string(r.URL.Path[len(r.URL.Path)-1])) == "x" { //404 body bod = bod404 - } else if strings.ToLower(string(r.URL.Path[len(r.URL.Path)-1])) == "y" { + } + if strings.HasSuffix(r.URL.Path, "/y") { //modified 404 bod = bod404mod } + if respCode == 404 { + bod = bod404 + } + if respCode == 401 { if u, p, ok := r.BasicAuth(); strings.ToLower(string(r.URL.Path[len(r.URL.Path)-1])) == "basicauth" && ok && u == "test" && p == "test" { respCode = 200 + bod = bod200 } } @@ -205,6 +215,7 @@ func (ts *TestServer) handler(w http.ResponseWriter, r *http.Request) { if strings.ToLower(r.URL.Path) == "/badheader/" { w.Header().Add("X-Bad-Header", "test123") } + w.WriteHeader(respCode) fmt.Fprintln(w, bod) //fmt.Println(r.Method, r.URL, respCode, bod) diff --git a/librecursebuster/util.go b/librecursebuster/util.go index c8996d3..6a98ad4 100644 --- a/librecursebuster/util.go +++ b/librecursebuster/util.go @@ -45,16 +45,18 @@ func getUrls(page []byte) ([]string, error) { return ret, nil } -func detectSoft404(r1 *http.Response, r2 *http.Response, ratio float64) bool { +func detectSoft404(r1 *http.Response, r2 *http.Response, ratio float64) (bool, float64) { //a, b := []byte{} if r1 != nil && r2 != nil { - if r1.ContentLength > 0 && r2.ContentLength > 0 && - r1.StatusCode == r2.StatusCode { + if r1.ContentLength > 0 && r2.ContentLength > 0 { //&& + //r1.StatusCode == r2.StatusCode { a, e := ioutil.ReadAll(r1.Body) + r1.Body = ioutil.NopCloser(bytes.NewBuffer(a)) if e != nil { panic(e) } b, e := ioutil.ReadAll(r2.Body) + r2.Body = ioutil.NopCloser(bytes.NewBuffer(b)) if e != nil { panic(e) } @@ -63,11 +65,11 @@ func detectSoft404(r1 *http.Response, r2 *http.Response, ratio float64) bool { perc := (longer - dist) / longer if perc > ratio { //if diff.QuickRatio() > ratio { - return true + return true, perc } } } - return false + return false, 0 } //todo: test this is correct @@ -86,7 +88,11 @@ func levenshteinDistance(s []byte, t []byte) int { if len(t) == 0 { return len(s) } - + if len(s) < len(t) { + temp := s + s = t + t = temp + } // create two work vectors of integer distances v0 := make([]int, len(t)+1) v1 := make([]int, len(t)+1) diff --git a/main.go b/main.go index 5c86ee6..b546229 100644 --- a/main.go +++ b/main.go @@ -14,7 +14,7 @@ import ( "github.com/fatih/color" ) -const version = "1.5.15" +const version = "1.5.16" func main() { if runtime.GOOS == "windows" { //lol goos