Skip to content

Commit

Permalink
feat: 优化订阅流量获取, 启用共享缓存(默认一分钟) 并优先尝试 HEAD 方法
Browse files Browse the repository at this point in the history
  • Loading branch information
xream committed Jan 14, 2024
1 parent 5cbcf4f commit 5584225
Show file tree
Hide file tree
Showing 6 changed files with 203 additions and 22 deletions.
2 changes: 1 addition & 1 deletion backend/package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "sub-store",
"version": "2.14.160",
"version": "2.14.161",
"description": "Advanced Subscription Manager for QX, Loon, Surge, Stash and ShadowRocket.",
"main": "src/main.js",
"scripts": {
Expand Down
2 changes: 2 additions & 0 deletions backend/src/constants.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,8 @@ export const GIST_BACKUP_KEY = 'Auto Generated Sub-Store Backup';
export const GIST_BACKUP_FILE_NAME = 'Sub-Store';
export const ARTIFACT_REPOSITORY_KEY = 'Sub-Store Artifacts Repository';
export const RESOURCE_CACHE_KEY = '#sub-store-cached-resource';
export const HEADERS_RESOURCE_CACHE_KEY = '#sub-store-cached-headers-resource';
export const CHR_EXPIRATION_TIME_KEY = '#sub-store-chr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 1 min
export const CACHE_EXPIRATION_TIME_MS = 60 * 60 * 1000; // 1 hour
export const SCRIPT_RESOURCE_CACHE_KEY = '#sub-store-cached-script-resource'; // cached-script-resource CSR
export const CSR_EXPIRATION_TIME_KEY = '#sub-store-csr-expiration-time'; // Custom expiration time key; (Loon|Surge) Default write 48 hour
14 changes: 12 additions & 2 deletions backend/src/core/proxy-utils/processors/index.js
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,12 @@ import { ProxyUtils } from '@/core/proxy-utils';
import { produceArtifact } from '@/restful/sync';

import env from '@/utils/env';
import { getFlowHeaders, parseFlowHeaders, flowTransfer } from '@/utils/flow';
import {
getFlowField,
getFlowHeaders,
parseFlowHeaders,
flowTransfer,
} from '@/utils/flow';

/**
The rule "(name CONTAINS "🇨🇳") AND (port IN [80, 443])" can be expressed as follows:
Expand Down Expand Up @@ -781,7 +786,12 @@ function removeFlag(str) {
}

function createDynamicFunction(name, script, $arguments) {
const flowUtils = { getFlowHeaders, parseFlowHeaders, flowTransfer };
const flowUtils = {
getFlowField,
getFlowHeaders,
parseFlowHeaders,
flowTransfer,
};
if ($.env.isLoon) {
return new Function(
'$arguments',
Expand Down
10 changes: 9 additions & 1 deletion backend/src/utils/download.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@ import { findByName } from '@/utils/database';
import { HTTP, ENV } from '@/vendor/open-api';
import { hex_md5 } from '@/vendor/md5';
import resourceCache from '@/utils/resource-cache';
import headersResourceCache from '@/utils/headers-resource-cache';
import { getFlowField } from '@/utils/flow';
import $ from '@/core/app';

const tasks = new Map();
Expand Down Expand Up @@ -71,7 +73,13 @@ export default async function download(url, ua, timeout) {
);
http.get(url)
.then((resp) => {
const body = resp.body;
const { body, headers } = resp;
if (headers) {
const flowInfo = getFlowField(headers);
if (flowInfo) {
headersResourceCache.set(url, flowInfo);
}
}
if (body.replace(/\s/g, '').length === 0)
reject(new Error('远程资源内容为空!'));
else {
Expand Down
90 changes: 72 additions & 18 deletions backend/src/utils/flow.js
Original file line number Diff line number Diff line change
@@ -1,30 +1,84 @@
import { SETTINGS_KEY } from '@/constants';
import { HTTP } from '@/vendor/open-api';
import $ from '@/core/app';
import headersResourceCache from '@/utils/headers-resource-cache';

export async function getFlowHeaders(url, ua, timeout) {
const { defaultFlowUserAgent, defaultTimeout } = $.read(SETTINGS_KEY);
const userAgent =
ua ||
defaultFlowUserAgent ||
'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)';
const requestTimeout = timeout || defaultTimeout;
const http = HTTP();
const { headers } = await http.get({
url: url
.split(/[\r\n]+/)
.map((i) => i.trim())
.filter((i) => i.length)[0],
headers: {
'User-Agent': userAgent,
},
timeout: requestTimeout,
});
export function getFlowField(headers) {
const subkey = Object.keys(headers).filter((k) =>
/SUBSCRIPTION-USERINFO/i.test(k),
)[0];
return headers[subkey];
}
export async function getFlowHeaders(url, ua, timeout) {
let $arguments = {};
const rawArgs = url.split('#');
if (rawArgs.length > 1) {
try {
// 支持 `#${encodeURIComponent(JSON.stringify({arg1: "1"}))}`
$arguments = JSON.parse(decodeURIComponent(rawArgs[1]));
} catch (e) {
for (const pair of rawArgs[1].split('&')) {
const key = pair.split('=')[0];
const value = pair.split('=')[1];
// 部分兼容之前的逻辑 const value = pair.split('=')[1] || true;
$arguments[key] =
value == null || value === ''
? true
: decodeURIComponent(value);
}
}
}
const cached = headersResourceCache.get(url);
let flowInfo;
if (!$arguments?.noCache && cached) {
$.info(`使用缓存的流量信息: ${url}`);
flowInfo = cached;
} else {
const { defaultFlowUserAgent, defaultTimeout } = $.read(SETTINGS_KEY);
const userAgent =
ua ||
defaultFlowUserAgent ||
'Quantumult%20X/1.0.30 (iPhone14,2; iOS 15.6)';
const requestTimeout = timeout || defaultTimeout;
const http = HTTP();
try {
$.info(`使用 HEAD 方法获取流量信息: ${url}`);
const { headers } = await http.head({
url: url
.split(/[\r\n]+/)
.map((i) => i.trim())
.filter((i) => i.length)[0],
headers: {
'User-Agent': userAgent,
},
timeout: requestTimeout,
});
flowInfo = getFlowField(headers);
} catch (e) {
$.error(
`使用 HEAD 方法获取流量信息失败: ${url}: ${e.message ?? e}`,
);
}
if (!flowInfo) {
$.info(`使用 GET 方法获取流量信息: ${url}`);
const { headers } = await http.get({
url: url
.split(/[\r\n]+/)
.map((i) => i.trim())
.filter((i) => i.length)[0],
headers: {
'User-Agent': userAgent,
},
timeout: requestTimeout,
});
flowInfo = getFlowField(headers);
}
}
if (flowInfo) {
headersResourceCache.set(url, flowInfo);
}
return flowInfo;
}
export function parseFlowHeaders(flowHeaders) {
if (!flowHeaders) return;
// unit is KB
Expand Down
107 changes: 107 additions & 0 deletions backend/src/utils/headers-resource-cache.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
import $ from '@/core/app';
import {
HEADERS_RESOURCE_CACHE_KEY,
CHR_EXPIRATION_TIME_KEY,
} from '@/constants';

class ResourceCache {
constructor() {
this.expires = getExpiredTime();
if (!$.read(HEADERS_RESOURCE_CACHE_KEY)) {
$.write('{}', HEADERS_RESOURCE_CACHE_KEY);
}
this.resourceCache = JSON.parse($.read(HEADERS_RESOURCE_CACHE_KEY));
this._cleanup();
}

_cleanup() {
// clear obsolete cached resource
let clear = false;
Object.entries(this.resourceCache).forEach((entry) => {
const [id, updated] = entry;
if (!updated.time) {
// clear old version cache
delete this.resourceCache[id];
$.delete(`#${id}`);
clear = true;
}
if (new Date().getTime() - updated.time > this.expires) {
delete this.resourceCache[id];
clear = true;
}
});
if (clear) this._persist();
}

revokeAll() {
this.resourceCache = {};
this._persist();
}

_persist() {
$.write(JSON.stringify(this.resourceCache), HEADERS_RESOURCE_CACHE_KEY);
}

get(id) {
const updated = this.resourceCache[id] && this.resourceCache[id].time;
if (updated && new Date().getTime() - updated <= this.expires) {
return this.resourceCache[id].data;
}
return null;
}

gettime(id) {
const updated = this.resourceCache[id] && this.resourceCache[id].time;
if (updated && new Date().getTime() - updated <= this.expires) {
return this.resourceCache[id].time;
}
return null;
}

set(id, value) {
this.resourceCache[id] = { time: new Date().getTime(), data: value };
this._persist();
}
}

function getExpiredTime() {
// console.log($.read(CHR_EXPIRATION_TIME_KEY));
if (!$.read(CHR_EXPIRATION_TIME_KEY)) {
$.write('6e4', CHR_EXPIRATION_TIME_KEY); // 1分钟
}
let expiration = 6e4;
if ($.env.isLoon) {
const loont = {
// Loon 插件自义定
'1\u5206\u949f': 6e4,
'5\u5206\u949f': 3e5,
'10\u5206\u949f': 6e5,
'30\u5206\u949f': 18e5, // "30分钟"
'1\u5c0f\u65f6': 36e5,
'2\u5c0f\u65f6': 72e5,
'3\u5c0f\u65f6': 108e5,
'6\u5c0f\u65f6': 216e5,
'12\u5c0f\u65f6': 432e5,
'24\u5c0f\u65f6': 864e5,
'48\u5c0f\u65f6': 1728e5,
'72\u5c0f\u65f6': 2592e5, // "72小时"
'\u53c2\u6570\u4f20\u5165': 'readcachets', // "参数输入"
};
let intimed = $.read(
'#\u54cd\u5e94\u5934\u7f13\u5b58\u6709\u6548\u671f',
); // Loon #响应头缓存有效期
// console.log(intimed);
if (intimed in loont) {
expiration = loont[intimed];
if (expiration === 'readcachets') {
expiration = intimed;
}
}
return expiration;
} else {
expiration = $.read(CHR_EXPIRATION_TIME_KEY);
return expiration;
}
}

export default new ResourceCache();

0 comments on commit 5584225

Please sign in to comment.