[Bug]: Crash after servals hour on concurrency #71

iamat8080 · 2023-08-15T05:30:41Z

TLS client version

v1.3.12

System information

windows 10
amd ryzen 7 3700x

Issue description

github.com/bogdanfinn/fhttp.setRequestCancel.func4()
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/client.go:398 +0x8b
created by github.com/bogdanfinn/fhttp.setRequestCancel
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/client.go:397 +0x44a
goroutine 439378 [IO wait, 2 minutes]:
internal/poll.runtime_pollWait(0x181b7c2e2e8, 0x72)
        C:/Program Files/Go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xc3afeae5b260f336?, 0xc005b4b6c0?, 0x0)
        C:/Program Files/Go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.execIO(0xc004a09418, 0xe04580)
        C:/Program Files/Go/src/internal/poll/fd_windows.go:175 +0xf7
internal/poll.(*FD).Read(0xc004a09400, {0xc001098000, 0x1500, 0x1500})
        C:/Program Files/Go/src/internal/poll/fd_windows.go:436 +0x2b8
net.(*netFD).Read(0xc004a09400, {0xc001098000?, 0xc001098005?, 0x36?})
        C:/Program Files/Go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc00022f2d8, {0xc001098000?, 0x0?, 0x0?})
        C:/Program Files/Go/src/net/net.go:183 +0x45
github.com/bogdanfinn/utls.(*atLeastReader).Read(0xc0044d6678, {0xc001098000?, 0xc0044d6678?, 0x0?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:791 +0x3d
bytes.(*Buffer).ReadFrom(0xc001127430, {0xe74660, 0xc0044d6678})
        C:/Program Files/Go/src/bytes/buffer.go:202 +0x98
github.com/bogdanfinn/utls.(*Conn).readFromUntil(0xc001127180, {0x181b72aa0f8?, 0xc00022f2d8}, 0xc000bf4000?)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:813 +0xe5
github.com/bogdanfinn/utls.(*Conn).readRecordOrCCS(0xc001127180, 0x0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:620 +0x116
github.com/bogdanfinn/utls.(*Conn).readRecord(...)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:586
github.com/bogdanfinn/utls.(*Conn).Read(0xc001127180, {0xc004104000, 0x1000, 0xc003ee5e00?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:1300 +0x16f
bufio.(*Reader).Read(0xc002ec1b60, {0xc0036b6658, 0x9, 0x30?})
        C:/Program Files/Go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0xe740e0, 0xc002ec1b60}, {0xc0036b6658, 0x9, 0x9}, 0x9)
        C:/Program Files/Go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
github.com/bogdanfinn/fhttp/http2.readFrameHeader({0xc0036b6658?, 0x9?, 0xcd1ca2?}, {0xe740e0?, 0xc002ec1b60?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:238 +0x6e
github.com/bogdanfinn/fhttp/http2.(*Framer).ReadFrame(0xc0036b6620)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:494 +0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc005b4bfa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc002c6c1a0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

Steps to reproduce / Code Sample

type request struct {
	client       tls_client.HttpClient
	Thread       int
	userdata     string
        data string
}

func init_(thread int) request {
	tl, _ := tls_client.NewHttpClient(nil, option()...)
	return request{
		client:       tl,
		Thread:       thread,
		userdata:     userdataitter(),
	}
}

func (ms *request) Today() bool {
	url := "https://api.mydomain.com/data"
	payload := fmt.Sprintf("data=today", ms.userdata)
	req, err := http.NewRequest(http.MethodPost, url, strings.NewReader(payload))
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	req.Header = bheader
	resp, err := ms.client.Do(req)
	if err != nil {
		ms.clientChanger()
		PrintLogE(err.Error(), ms.Thread)
		return false
	}
        defer resp.Body.Close()

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	var data map[string]interface{}
	err = json.Unmarshal(readBytes, &data)
	if err != nil {
		PrintLogF("Error unmarshalling JSON:"+err.Error(), ms.Thread)
		return false, false
	}
        ms.data = data

	return true

}

func (ms *request) postdata() bool {

	url := "https://api.seconddomain.com/postdata"

	req, err := http.NewRequest(http.MethodPost, url, bytes.NewBuffer(ms.data))
	if err != nil {
		PrintLogF(err.Error(), ms.Thread)
		return false
	}

	req.Header = ms.headerstream
	resp, err := ms.client.Do(req)
	if err != nil {
		PrintLogE(err.Error(), ms.Thread)
		time.Sleep(5 * time.Second)
		return false
	}
        defer resp.Body.Close()
	if resp.StatusCode == 201 {
		return true
	} else {
		readBytes, err := io.ReadAll(resp.Body)
		if err != nil {
			PrintLogF(err.Error(), ms.Thread)
			return false
		}
		PrintLogM(fmt.Sprintf("status : %d dat : %s", resp.StatusCode, readBytes), ms.Thread)
	}
	return false
}

func run(Threads int) bool {
	self := init_(Threads)
	for i := 0; i < 3; i++{
		anid := self.Today()
		if !anid{
			if i == 2{
				PrintLogF(fmt.Sprintf("FAIL %s", self.userdata), self.Thread)
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	for i := 0; i < 3; i++{
		anid := self.postdata()
		if !anid{
			if i == 2{
				PrintLogF(fmt.Sprintf("FAIL POST %s", self.userdata), self.Thread)
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	self.client.CloseIdleConnections()

	return true
}

func main() {
	
	threads := CurrentConfig.Threads // Define the number of threads
	//threads = 1

	var wg sync.WaitGroup
	wg.Add(threads)

	for i := 1; i <= threads; i++ {
		go func(threadNum int) {
			defer wg.Done()
			run(threadNum)
		}(i)
		time.Sleep(500 * time.Millisecond)
	}
	wg.Wait()
	PrintLog("DONE", 0)
}

The text was updated successfully, but these errors were encountered:

bogdanfinn · 2023-08-15T05:34:47Z

@iamat8080 the code does not reproduce your issue.

iamat8080 · 2023-08-15T05:50:54Z

updated code it crashed after serval hours of running not able to figure out why and where the bug @bogdanfinn

iamat8080 · 2023-08-17T05:45:52Z

0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc003469fa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc006b13a00)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

goroutine 774853 [IO wait, 1 minutes]:
internal/poll.runtime_pollWait(0x1ad6700f8e0, 0x72)
        C:/Program Files/Go/src/runtime/netpoll.go:306 +0x89
internal/poll.(*pollDesc).wait(0xd81159c9a0d3a647?, 0xc002c976c0?, 0x0)
        C:/Program Files/Go/src/internal/poll/fd_poll_runtime.go:84 +0x32
internal/poll.execIO(0xc000ca0018, 0x8a4540)
        C:/Program Files/Go/src/internal/poll/fd_windows.go:175 +0xf7
internal/poll.(*FD).Read(0xc000ca0000, {0xc00547b500, 0x1500, 0x1500})
        C:/Program Files/Go/src/internal/poll/fd_windows.go:436 +0x2b8
net.(*netFD).Read(0xc000ca0000, {0xc00547b500?, 0xc00547b505?, 0x36?})
        C:/Program Files/Go/src/net/fd_posix.go:55 +0x29
net.(*conn).Read(0xc0005dc378, {0xc00547b500?, 0x0?, 0x0?})
        C:/Program Files/Go/src/net/net.go:183 +0x45
github.com/bogdanfinn/utls.(*atLeastReader).Read(0xc004cc36e0, {0xc00547b500?, 0xc004cc36e0?, 0x0?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:791 +0x3d
bytes.(*Buffer).ReadFrom(0xc003027eb0, {0x914600, 0xc004cc36e0})
        C:/Program Files/Go/src/bytes/buffer.go:202 +0x98
github.com/bogdanfinn/utls.(*Conn).readFromUntil(0xc003027c00, {0x1ad40a440d8?, 0xc0005dc378}, 0xc00504f000?)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:813 +0xe5
github.com/bogdanfinn/utls.(*Conn).readRecordOrCCS(0xc003027c00, 0x0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:620 +0x116
github.com/bogdanfinn/utls.(*Conn).readRecord(...)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:586
github.com/bogdanfinn/utls.(*Conn).Read(0xc003027c00, {0xc006ee9000, 0x1000, 0xc009dcd200?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/utls/conn.go:1300 +0x16f
bufio.(*Reader).Read(0xc0090fed20, {0xc0059be8f8, 0x9, 0x30?})
        C:/Program Files/Go/src/bufio/bufio.go:237 +0x1bb
io.ReadAtLeast({0x914080, 0xc0090fed20}, {0xc0059be8f8, 0x9, 0x9}, 0x9)
        C:/Program Files/Go/src/io/io.go:332 +0x9a
io.ReadFull(...)
        C:/Program Files/Go/src/io/io.go:351
github.com/bogdanfinn/fhttp/http2.readFrameHeader({0xc0059be8f8?, 0x9?, 0x771ca2?}, {0x914080?, 0xc0090fed20?})
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:238 +0x6e
github.com/bogdanfinn/fhttp/http2.(*Framer).ReadFrame(0xc0059be8c0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/frame.go:494 +0x95
github.com/bogdanfinn/fhttp/http2.(*clientConnReadLoop).run(0xc002c97fa0)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2100 +0x170
github.com/bogdanfinn/fhttp/http2.(*ClientConn).readLoop(0xc0091fd040)
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:2021 +0x79
created by github.com/bogdanfinn/fhttp/http2.(*Transport).newClientConn
        D:/Newfolder/PROJECT/GolandProjects/Apitest/vendor/github.com/bogdanfinn/fhttp/http2/transport.go:860 +0xc92

this time it's different

0x676e67 · 2023-08-21T00:54:52Z

My guess is: Resource descriptors are exhausted. In Linux, you can limit resource concurrency by setting ulimit -n 65536

iamat8080 · 2023-08-22T06:13:47Z

hello @gngpp I am using windows

KhushC-03 · 2023-08-26T11:13:57Z

updated code it crashed after serval hours of running not able to figure out why and where the bug @bogdanfinn

Getting the same exact issue, works perfectly fine but crashes after a few hours of running.

github.com/bogdanfinn/fhttp.setRequestCancel.func4()
C:/Users/Administrator/go/pkg/mod/github.com/bogdanfinn/[email protected]/client.go:398 +0x8b
created by github.com/bogdanfinn/fhttp.setRequestCancel
C:/Users/Administrator/go/pkg/mod/github.com/bogdanfinn/[email protected]/client.go:397 +0x44a

bogdanfinn · 2023-08-28T07:42:57Z

Still no code provided which reproduces the issue and can be compiled.

iamat8080 · 2023-08-30T06:56:15Z

package main

import (
	"fmt"
	"io"
	"sync"
	"time"

	http "github.com/bogdanfinn/fhttp"
	tls_client "github.com/bogdanfinn/tls-client"
)

type re struct {
	client       tls_client.HttpClient
	Thread       int
	userdata     string
	data string
}

func init__(thread int) re {
	tl, _ := tls_client.NewHttpClient(nil, option()...)
	return re{
		client:       tl,
		Thread:       thread,
		userdata:     userdataitter(),
	}
}

var gg = http.Header{
	"accept":             {"application/json"},
	"Cache-Control":      {"no-cache"},
	"user-agent":         {"android/8.1.7.1053/NapsterGlobal"},
	"Accept-Encoding":    {"gzip, deflate"},
	"Origin": {"https://www.ssense.com"},
	"Referer":       {"https://www.ssense.com/"},
	"Connection":         {"Close"},
	http.HeaderOrderKey: {
		"accept",
		"Accept-Encoding",
		"Cache-Control",
		"Origin",
		"Referer",
		"User-Agent",
		}}

func (ms *re) Today() bool {
	url := "https://www.ssense.com/en-us/men"
	req, err := http.NewRequest(http.MethodGet, url, nil)
	if err != nil {
		return false
	}

	req.Header = gg
	resp, err := ms.client.Do(req)
	if err != nil {
		return false
	}
	defer resp.Body.Close()

	readBytes, err := io.ReadAll(resp.Body)
	if err != nil {
		return false
	}
	fmt.Println(readBytes)

	return true

}

func run(Threads int) bool {
	self := init__(Threads)
	for i := 0; i < 3; i++{
		anid := self.Today()
		if !anid{
			if i == 2{
				self.client.CloseIdleConnections()
				return false
			}
			continue
		}
		break
	}
	self.client.CloseIdleConnections()

	return true
}

func main() {

	threads := CurrentConfig.Threads // Define the number of threads
	//threads = 1

	var wg sync.WaitGroup
	wg.Add(threads)

	for i := 1; i <= threads; i++ {
		go func(threadNum int) {
			defer wg.Done()
			run(threadNum)
		}(i)
		time.Sleep(500 * time.Millisecond)
	}
	wg.Wait()

}

it's sample code I didn't add another post request where I scarpe and post to DB using API

If you format any bot like this with a high number of threads and in an endless loop running for a while it might crash
or it would be great if you guide me on what to do I'll test debug on my side and provide you with all logs
else I have to write different code using different domains and APIs which can be similar to this one

PS: I believe running in an endless loop with few posts and get requests can reproduce the crash
@bogdanfinn

huaibaoyy1 · 2024-02-16T11:34:48Z

Brother, I also encountered the same problem as you, have you solved it? It will crash after reaching millions of requests.

iamat8080 added the bug Something isn't working label Aug 15, 2023

Provide feedback

Saved searches

Use saved searches to filter your results more quickly

[Bug]: Crash after servals hour on concurrency #71

[Bug]: Crash after servals hour on concurrency #71

iamat8080 commented Aug 15, 2023 •

edited

Loading

bogdanfinn commented Aug 15, 2023

iamat8080 commented Aug 15, 2023

iamat8080 commented Aug 17, 2023

0x676e67 commented Aug 21, 2023 •

edited

Loading

iamat8080 commented Aug 22, 2023

KhushC-03 commented Aug 26, 2023

bogdanfinn commented Aug 28, 2023 •

edited

Loading

iamat8080 commented Aug 30, 2023 •

edited

Loading

huaibaoyy1 commented Feb 16, 2024

[Bug]: Crash after servals hour on concurrency #71

[Bug]: Crash after servals hour on concurrency #71

Comments

iamat8080 commented Aug 15, 2023 • edited Loading

TLS client version

System information

Issue description

Steps to reproduce / Code Sample

bogdanfinn commented Aug 15, 2023

iamat8080 commented Aug 15, 2023

iamat8080 commented Aug 17, 2023

0x676e67 commented Aug 21, 2023 • edited Loading

iamat8080 commented Aug 22, 2023

KhushC-03 commented Aug 26, 2023

bogdanfinn commented Aug 28, 2023 • edited Loading

iamat8080 commented Aug 30, 2023 • edited Loading

huaibaoyy1 commented Feb 16, 2024

iamat8080 commented Aug 15, 2023 •

edited

Loading

0x676e67 commented Aug 21, 2023 •

edited

Loading

bogdanfinn commented Aug 28, 2023 •

edited

Loading

iamat8080 commented Aug 30, 2023 •

edited

Loading