From 074aaf1d8cbad499589628ea9564255f5361e6aa Mon Sep 17 00:00:00 2001 From: "lzyme.dev@gmail.com" Date: Sat, 18 Nov 2023 16:16:32 +0800 Subject: [PATCH] fix choose the best available site url --- config/config.go | 141 +++++++++++++++++++++++++++++++++++++++--- config/config_test.go | 11 ++++ utils/utils.go | 24 ++++++- utils/utils_test.go | 93 ++++++++++++++++++++++++++++ 4 files changed, 260 insertions(+), 9 deletions(-) create mode 100644 config/config_test.go diff --git a/config/config.go b/config/config.go index 3afecc1..499a844 100644 --- a/config/config.go +++ b/config/config.go @@ -6,7 +6,10 @@ import ( "net/http" "os" "path/filepath" + "regexp" "strconv" + "strings" + "sync" "go.uber.org/zap" @@ -19,27 +22,149 @@ const MetaDataDb = "asmr.db" // AsmroneStartPageUrl https://api.asmr.one/api/works?order=create_date&sort=desc&page=1&seed=92&subtitle=0 // const AsmrOneStartPageUrl = "https://api.asmr.one" -const Asmr100StartPageUrl = "https://api.asmr-100.com" -const Asmr200StartPageUrl = "https://api.asmr-200.com" +//const Asmr100StartPageUrl = "https://api.asmr-100.com" +//const Asmr200StartPageUrl = "https://api.asmr-200.com" var AsmrBaseApiUrl = "" func init() { //访问asmr.one + url := GetRespFastestSiteUrl() + AsmrBaseApiUrl = url +} + +// GetAsmrLatestUrls +// +// @Description: 获取asmr.one最新域名列表 +// @return []string +// @return error +func GetAsmrLatestUrls() ([]string, error) { + //访问asmr.one 最新域名发布页 + // official : https://as.mr + // cf worker proxy: https://as.131433.xyz + var officialPublishSite = "https://as.mr" + var cfProxyPublishSite = "https://as.131433.xyz" + var latestPublishSite = "" client := utils.Client.Get().(*http.Client) - req, _ := http.NewRequest("GET", "https://asmr.one", nil) + req, _ := http.NewRequest("GET", officialPublishSite, nil) req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") resp, err := client.Do(req) if err != nil || resp.StatusCode != 200 { - log.AsmrLog.Error("尝试访问asmr.one失败: ", zap.String("error", err.Error())) - log.AsmrLog.Info("当前使用asmr-200.com访问") - AsmrBaseApiUrl = Asmr200StartPageUrl + log.AsmrLog.Info("尝试访问asmr.one最新站点发布页as.mr失败: ", zap.String("error", err.Error())) + log.AsmrLog.Info("当前使用as.131433.xyz代理访问最新站点发布页") + latestPublishSite = cfProxyPublishSite } else { - log.AsmrLog.Info("当前使用asmr-100.com访问...") - AsmrBaseApiUrl = Asmr100StartPageUrl + log.AsmrLog.Info("当前使用as.mr访问最新站点发布页...") + latestPublishSite = officialPublishSite + } + utils.Client.Put(client) + defer resp.Body.Close() + + client = utils.Client.Get().(*http.Client) + req, _ = http.NewRequest("GET", latestPublishSite, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + resp, err = client.Do(req) + if err != nil || resp.StatusCode != 200 { + log.AsmrLog.Error("访问asmr.one最新域名发布页出现错误: ", zap.String("error", err.Error())) + return nil, err + } + utils.Client.Put(client) + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + log.AsmrLog.Error("Error reading response body:", zap.String("error", err.Error())) + return nil, err + } + + // Convert the response body to a string and print it + bodyText := string(body) + //fmt.Println("Response Text:", bodyText) + + pattern := `` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(bodyText) + + var jsFilePath = "" + if len(match) > 1 { + jsFilePath = match[1] + //fmt.Println("JavaScript file path:", jsFilePath) + } else { + //fmt.Println("JavaScript file path not found.") + } + + jsContentUrl := latestPublishSite + jsFilePath + client = utils.Client.Get().(*http.Client) + req, _ = http.NewRequest("GET", jsContentUrl, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + resp, err = client.Do(req) + if err != nil || resp.StatusCode != 200 { + log.AsmrLog.Error("访问asmr.one最新域名发布页js resource出现错误: ", zap.String("error", err.Error())) + return nil, err } utils.Client.Put(client) defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + //fmt.Println("Error reading response body:", err) + return nil, err + } + jsText := string(body) + //fmt.Println("Response Text:", jsText) + + //extract rapid resp site url from js file text + sitePattern := `link:\s*"([^"]+)"` + re = regexp.MustCompile(sitePattern) + matches := re.FindAllStringSubmatch(jsText, -1) + var result []string + for _, match := range matches { + if len(match) > 1 { + link := match[1] + if strings.HasPrefix(link, "https://") { + result = append(result, link) + } + //fmt.Println("Link:", link) + } + } + return result, nil +} + +func GetRespFastestSiteUrl() string { + latestUrls, err := GetAsmrLatestUrls() + if err != nil { + log.AsmrLog.Error("获取最新域名列表失败: ", zap.String("error", err.Error())) + //as the default + return "https://api.asmr.one" + } + var wg sync.WaitGroup + ch := make(chan string, len(latestUrls)) + + for _, url := range latestUrls { + wg.Add(1) + go utils.FastFetch(url, &wg, ch) + } + + // Use a goroutine to wait for all fetches to complete and close the channel + go func() { + wg.Wait() + close(ch) + }() + + // Wait for the fastest response + var fastestResponse string + for response := range ch { + if fastestResponse == "" || len(response) < len(fastestResponse) { + fastestResponse = response + } + log.AsmrLog.Info("Checking Fast Response:", zap.String("response", response)) + } + + log.AsmrLog.Info("Fastest Response is:", zap.String("response", fastestResponse)) + fastUrls := strings.Split(fastestResponse, "|") + url := fastUrls[0] + url = strings.Trim(url, "/") + // convert to api + apiUrl := strings.Replace(url, "https://", "https://api.", 1) + return apiUrl } // Config diff --git a/config/config_test.go b/config/config_test.go new file mode 100644 index 0000000..5247387 --- /dev/null +++ b/config/config_test.go @@ -0,0 +1,11 @@ +package config + +import ( + "fmt" + "testing" +) + +func TestGetRespFastestSiteUrl(t *testing.T) { + url := GetRespFastestSiteUrl() + fmt.Println(url) +} diff --git a/utils/utils.go b/utils/utils.go index de881ba..e04d3e4 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -40,7 +40,8 @@ var Client = sync.Pool{ Transport: &http.Transport{ Proxy: http.ProxyFromEnvironment, TLSClientConfig: &tls.Config{ - MaxVersion: tls.VersionTLS12, // Cloudflare 会杀 + //update tls version,version 12 may cause error on cf worker + MaxVersion: tls.VersionTLS13, }, }, } @@ -382,3 +383,24 @@ func CopyFile(src, dst string) (err error) { } return err } + +func FastFetch(url string, wg *sync.WaitGroup, ch chan<- string) { + defer wg.Done() + + startTime := time.Now() + resp, err := http.Get(url) + if err != nil { + fmt.Printf("Error fetching %s: %v\n", url, err) + return + } + defer resp.Body.Close() + + _, err = io.ReadAll(resp.Body) + if err != nil { + fmt.Printf("Error reading response body from %s: %v\n", url, err) + return + } + + duration := time.Since(startTime) + ch <- fmt.Sprintf("%s|%s", url, duration) +} diff --git a/utils/utils_test.go b/utils/utils_test.go index 0caf867..34ff6bf 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -2,7 +2,12 @@ package utils import ( "bufio" + "fmt" + "io" + "net/http" "os" + "regexp" + "strings" "testing" ) @@ -30,3 +35,91 @@ func TestWriteErrorFile(t *testing.T) { func TestFixBrokenDownloadFile(t *testing.T) { FixBrokenDownloadFile(3) } + +func TestGetRapidRespSiteUrl(t *testing.T) { + //访问asmr.one 最新域名发布页 + // official : https://as.mr + // cf worker proxy: https://as.131433.xyz + var officialPublishSite = "https://as.mr" + var cfProxyPublishSite = "https://as.131433.xyz" + var latestPublishSite = "" + client := Client.Get().(*http.Client) + req, _ := http.NewRequest("GET", "https://as.mr", nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + resp, err := client.Do(req) + if err != nil || resp.StatusCode != 200 { + println("尝试访问asmr.one最新站点发布页as.mr失败: ", err.Error()) + println("当前使用as.131433.xyz访问") + latestPublishSite = cfProxyPublishSite + } else { + println("当前使用as.131433.xyz访问...") + latestPublishSite = officialPublishSite + } + Client.Put(client) + defer resp.Body.Close() + + client = Client.Get().(*http.Client) + req, _ = http.NewRequest("GET", latestPublishSite, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + resp, err = client.Do(req) + if err != nil || resp.StatusCode != 200 { + println("访问asmr.one最新域名发布页出现错误: ", err.Error()) + return + } + defer resp.Body.Close() + body, err := io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return + } + + // Convert the response body to a string and print it + bodyText := string(body) + fmt.Println("Response Text:", bodyText) + Client.Put(client) + + pattern := `` + re := regexp.MustCompile(pattern) + match := re.FindStringSubmatch(bodyText) + + var jsFilePath = "" + if len(match) > 1 { + jsFilePath = match[1] + fmt.Println("JavaScript file path:", jsFilePath) + } else { + fmt.Println("JavaScript file path not found.") + } + + jsContentUrl := latestPublishSite + jsFilePath + client = Client.Get().(*http.Client) + req, _ = http.NewRequest("GET", jsContentUrl, nil) + req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36") + resp, err = client.Do(req) + if err != nil || resp.StatusCode != 200 { + println("访问asmr.one最新域名发布页js resource出现错误: ", err.Error()) + return + } + defer resp.Body.Close() + body, err = io.ReadAll(resp.Body) + if err != nil { + fmt.Println("Error reading response body:", err) + return + } + jsText := string(body) + fmt.Println("Response Text:", jsText) + + //extract rapid resp site url from js file text + sitePattern := `link:\s*"([^"]+)"` + re = regexp.MustCompile(sitePattern) + matches := re.FindAllStringSubmatch(jsText, -1) + result := []string{} + for _, match := range matches { + if len(match) > 1 { + link := match[1] + if strings.HasPrefix("https://", link) { + result = append(result, link) + } + //fmt.Println("Link:", link) + } + } +}