Skip to content

Commit

Permalink
fix choose the best available site url
Browse files Browse the repository at this point in the history
  • Loading branch information
fireinrain committed Nov 18, 2023
1 parent bd3b329 commit 074aaf1
Show file tree
Hide file tree
Showing 4 changed files with 260 additions and 9 deletions.
141 changes: 133 additions & 8 deletions config/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,10 @@ import (
"net/http"
"os"
"path/filepath"
"regexp"
"strconv"
"strings"
"sync"

"go.uber.org/zap"

Expand All @@ -19,27 +22,149 @@ const MetaDataDb = "asmr.db"

// AsmroneStartPageUrl https://api.asmr.one/api/works?order=create_date&sort=desc&page=1&seed=92&subtitle=0
// const AsmrOneStartPageUrl = "https://api.asmr.one"
const Asmr100StartPageUrl = "https://api.asmr-100.com"
const Asmr200StartPageUrl = "https://api.asmr-200.com"
//const Asmr100StartPageUrl = "https://api.asmr-100.com"
//const Asmr200StartPageUrl = "https://api.asmr-200.com"

var AsmrBaseApiUrl = ""

func init() {
//访问asmr.one
url := GetRespFastestSiteUrl()
AsmrBaseApiUrl = url
}

// GetAsmrLatestUrls
//
// @Description: 获取asmr.one最新域名列表
// @return []string
// @return error
func GetAsmrLatestUrls() ([]string, error) {
//访问asmr.one 最新域名发布页
// official : https://as.mr
// cf worker proxy: https://as.131433.xyz
var officialPublishSite = "https://as.mr"
var cfProxyPublishSite = "https://as.131433.xyz"
var latestPublishSite = ""
client := utils.Client.Get().(*http.Client)
req, _ := http.NewRequest("GET", "https://asmr.one", nil)
req, _ := http.NewRequest("GET", officialPublishSite, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
resp, err := client.Do(req)
if err != nil || resp.StatusCode != 200 {
log.AsmrLog.Error("尝试访问asmr.one失败: ", zap.String("error", err.Error()))
log.AsmrLog.Info("当前使用asmr-200.com访问")
AsmrBaseApiUrl = Asmr200StartPageUrl
log.AsmrLog.Info("尝试访问asmr.one最新站点发布页as.mr失败: ", zap.String("error", err.Error()))
log.AsmrLog.Info("当前使用as.131433.xyz代理访问最新站点发布页")
latestPublishSite = cfProxyPublishSite
} else {
log.AsmrLog.Info("当前使用asmr-100.com访问...")
AsmrBaseApiUrl = Asmr100StartPageUrl
log.AsmrLog.Info("当前使用as.mr访问最新站点发布页...")
latestPublishSite = officialPublishSite
}
utils.Client.Put(client)
defer resp.Body.Close()

client = utils.Client.Get().(*http.Client)
req, _ = http.NewRequest("GET", latestPublishSite, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
resp, err = client.Do(req)
if err != nil || resp.StatusCode != 200 {
log.AsmrLog.Error("访问asmr.one最新域名发布页出现错误: ", zap.String("error", err.Error()))
return nil, err
}
utils.Client.Put(client)
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
log.AsmrLog.Error("Error reading response body:", zap.String("error", err.Error()))
return nil, err
}

// Convert the response body to a string and print it
bodyText := string(body)
//fmt.Println("Response Text:", bodyText)

pattern := `<script type="module" crossorigin src="(/assets/index\.[a-f0-9]+\.js)"></script>`
re := regexp.MustCompile(pattern)
match := re.FindStringSubmatch(bodyText)

var jsFilePath = ""
if len(match) > 1 {
jsFilePath = match[1]
//fmt.Println("JavaScript file path:", jsFilePath)
} else {
//fmt.Println("JavaScript file path not found.")
}

jsContentUrl := latestPublishSite + jsFilePath
client = utils.Client.Get().(*http.Client)
req, _ = http.NewRequest("GET", jsContentUrl, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
resp, err = client.Do(req)
if err != nil || resp.StatusCode != 200 {
log.AsmrLog.Error("访问asmr.one最新域名发布页js resource出现错误: ", zap.String("error", err.Error()))
return nil, err
}
utils.Client.Put(client)
defer resp.Body.Close()
body, err = io.ReadAll(resp.Body)
if err != nil {
//fmt.Println("Error reading response body:", err)
return nil, err
}
jsText := string(body)
//fmt.Println("Response Text:", jsText)

//extract rapid resp site url from js file text
sitePattern := `link:\s*"([^"]+)"`
re = regexp.MustCompile(sitePattern)
matches := re.FindAllStringSubmatch(jsText, -1)
var result []string
for _, match := range matches {
if len(match) > 1 {
link := match[1]
if strings.HasPrefix(link, "https://") {
result = append(result, link)
}
//fmt.Println("Link:", link)
}
}
return result, nil
}

func GetRespFastestSiteUrl() string {
latestUrls, err := GetAsmrLatestUrls()
if err != nil {
log.AsmrLog.Error("获取最新域名列表失败: ", zap.String("error", err.Error()))
//as the default
return "https://api.asmr.one"
}
var wg sync.WaitGroup
ch := make(chan string, len(latestUrls))

for _, url := range latestUrls {
wg.Add(1)
go utils.FastFetch(url, &wg, ch)
}

// Use a goroutine to wait for all fetches to complete and close the channel
go func() {
wg.Wait()
close(ch)
}()

// Wait for the fastest response
var fastestResponse string
for response := range ch {
if fastestResponse == "" || len(response) < len(fastestResponse) {
fastestResponse = response
}
log.AsmrLog.Info("Checking Fast Response:", zap.String("response", response))
}

log.AsmrLog.Info("Fastest Response is:", zap.String("response", fastestResponse))
fastUrls := strings.Split(fastestResponse, "|")
url := fastUrls[0]
url = strings.Trim(url, "/")
// convert to api
apiUrl := strings.Replace(url, "https://", "https://api.", 1)
return apiUrl
}

// Config
Expand Down
11 changes: 11 additions & 0 deletions config/config_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
package config

import (
"fmt"
"testing"
)

func TestGetRespFastestSiteUrl(t *testing.T) {
url := GetRespFastestSiteUrl()
fmt.Println(url)
}
24 changes: 23 additions & 1 deletion utils/utils.go
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,8 @@ var Client = sync.Pool{
Transport: &http.Transport{
Proxy: http.ProxyFromEnvironment,
TLSClientConfig: &tls.Config{
MaxVersion: tls.VersionTLS12, // Cloudflare 会杀
//update tls version,version 12 may cause error on cf worker
MaxVersion: tls.VersionTLS13,
},
},
}
Expand Down Expand Up @@ -382,3 +383,24 @@ func CopyFile(src, dst string) (err error) {
}
return err
}

func FastFetch(url string, wg *sync.WaitGroup, ch chan<- string) {
defer wg.Done()

startTime := time.Now()
resp, err := http.Get(url)
if err != nil {
fmt.Printf("Error fetching %s: %v\n", url, err)
return
}
defer resp.Body.Close()

_, err = io.ReadAll(resp.Body)
if err != nil {
fmt.Printf("Error reading response body from %s: %v\n", url, err)
return
}

duration := time.Since(startTime)
ch <- fmt.Sprintf("%s|%s", url, duration)
}
93 changes: 93 additions & 0 deletions utils/utils_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,12 @@ package utils

import (
"bufio"
"fmt"
"io"
"net/http"
"os"
"regexp"
"strings"
"testing"
)

Expand Down Expand Up @@ -30,3 +35,91 @@ func TestWriteErrorFile(t *testing.T) {
func TestFixBrokenDownloadFile(t *testing.T) {
FixBrokenDownloadFile(3)
}

func TestGetRapidRespSiteUrl(t *testing.T) {
//访问asmr.one 最新域名发布页
// official : https://as.mr
// cf worker proxy: https://as.131433.xyz
var officialPublishSite = "https://as.mr"
var cfProxyPublishSite = "https://as.131433.xyz"
var latestPublishSite = ""
client := Client.Get().(*http.Client)
req, _ := http.NewRequest("GET", "https://as.mr", nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
resp, err := client.Do(req)
if err != nil || resp.StatusCode != 200 {
println("尝试访问asmr.one最新站点发布页as.mr失败: ", err.Error())
println("当前使用as.131433.xyz访问")
latestPublishSite = cfProxyPublishSite
} else {
println("当前使用as.131433.xyz访问...")
latestPublishSite = officialPublishSite
}
Client.Put(client)
defer resp.Body.Close()

client = Client.Get().(*http.Client)
req, _ = http.NewRequest("GET", latestPublishSite, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
resp, err = client.Do(req)
if err != nil || resp.StatusCode != 200 {
println("访问asmr.one最新域名发布页出现错误: ", err.Error())
return
}
defer resp.Body.Close()
body, err := io.ReadAll(resp.Body)
if err != nil {
fmt.Println("Error reading response body:", err)
return
}

// Convert the response body to a string and print it
bodyText := string(body)
fmt.Println("Response Text:", bodyText)
Client.Put(client)

pattern := `<script type="module" crossorigin src="(/assets/index\.[a-f0-9]+\.js)"></script>`
re := regexp.MustCompile(pattern)
match := re.FindStringSubmatch(bodyText)

var jsFilePath = ""
if len(match) > 1 {
jsFilePath = match[1]
fmt.Println("JavaScript file path:", jsFilePath)
} else {
fmt.Println("JavaScript file path not found.")
}

jsContentUrl := latestPublishSite + jsFilePath
client = Client.Get().(*http.Client)
req, _ = http.NewRequest("GET", jsContentUrl, nil)
req.Header.Set("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36")
resp, err = client.Do(req)
if err != nil || resp.StatusCode != 200 {
println("访问asmr.one最新域名发布页js resource出现错误: ", err.Error())
return
}
defer resp.Body.Close()
body, err = io.ReadAll(resp.Body)
if err != nil {
fmt.Println("Error reading response body:", err)
return
}
jsText := string(body)
fmt.Println("Response Text:", jsText)

//extract rapid resp site url from js file text
sitePattern := `link:\s*"([^"]+)"`
re = regexp.MustCompile(sitePattern)
matches := re.FindAllStringSubmatch(jsText, -1)
result := []string{}
for _, match := range matches {
if len(match) > 1 {
link := match[1]
if strings.HasPrefix("https://", link) {
result = append(result, link)
}
//fmt.Println("Link:", link)
}
}
}

0 comments on commit 074aaf1

Please sign in to comment.