From 4a8b9386b3452459886370c03cdb8b6004137eea Mon Sep 17 00:00:00 2001 From: diginote Date: Fri, 9 Jun 2023 18:08:45 +0200 Subject: [PATCH 1/2] extend crux request with proxy handling --- lib/plugins/crux/send.js | 154 +++++++++++++++++++++++++++++++-------- 1 file changed, 122 insertions(+), 32 deletions(-) diff --git a/lib/plugins/crux/send.js b/lib/plugins/crux/send.js index fb5b0c928a..70efe5059f 100644 --- a/lib/plugins/crux/send.js +++ b/lib/plugins/crux/send.js @@ -1,46 +1,136 @@ import { request as _request } from 'node:https'; +import * as http from 'node:http'; +import * as https from 'node:https'; + import intel from 'intel'; const log = intel.getLogger('plugin.crux'); +//use proxy? +let proxy = false; +if(process.env.http_proxy) { + proxy = new URL(process.env.http_proxy); +} + +if(process.env.https_proxy) { + proxy = new URL(process.env.https_proxy); +} + +//chromeuxreport target config +const cruxtHost = 'chromeuxreport.googleapis.com'; +const cruxPort = 443; +const cruxHTTPMethod = 'POST'; + + export async function send(url, key, formFactor, shouldWeTestTheURL) { let data = shouldWeTestTheURL ? { url } : { origin: url }; if (formFactor !== 'ALL') { data.formFactor = formFactor; } data = JSON.stringify(data); - // Return new promise - return new Promise(function (resolve, reject) { - // Do async job - const request = _request( - { - host: 'chromeuxreport.googleapis.com', - port: 443, - path: `/v1/records:queryRecord?key=${key}`, - headers: { - 'Content-Type': 'application/json', - 'Content-Length': Buffer.byteLength(data, 'utf8') + + const cruxPath = `/v1/records:queryRecord?key=${key}`; + const cruxHeaders = { + 'Content-Type': 'application/json', + 'Content-Length': Buffer.byteLength(data, 'utf8') + }; + + //no proxy + if(proxy === false) { + // Return new promise + return new Promise(function (resolve, reject) { + // Do async job + const request = _request( + { + host: cruxtHost, + port: cruxPort, + path: cruxPath, + headers: cruxHeaders, + method: cruxHTTPMethod }, - method: 'POST' - }, - function (res) { - if (res.statusCode >= 499) { - log.error( - `Got error from CrUx. Error Code: ${res.statusCode} Message: ${res.statusMessage}` + (res) => { + if (res.statusCode >= 499) { + log.error( + `Got error from CrUx. Error Code: ${res.statusCode} Message: ${res.statusMessage}` + ); + return reject(new Error(`Status Code: ${res.statusCode}`)); + } + const data = []; + + res.on('data', chunk => { + data.push(chunk); + }); + + res.on('end', () => + resolve(JSON.parse(Buffer.concat(data).toString())) ); - return reject(new Error(`Status Code: ${res.statusCode}`)); } - const data = []; - - res.on('data', chunk => { - data.push(chunk); - }); - - res.on('end', () => - resolve(JSON.parse(Buffer.concat(data).toString())) - ); - } - ); - request.write(data); - request.end(); - }); + ); + request.write(data); + request.end(); + }); + //use proxy + } else { + return proxyRequest(`https://${cruxtHost}${cruxPath}`, cruxHeaders, data); + } +} + + + + +//do request with proxy +async function proxyRequest(url, headers, body) { + return new Promise((resolve, reject) => { + const urlParsed = new URL(url); + http.request({ + host: proxy.hostname, + port: proxy.port, + method: 'CONNECT', + path: `${urlParsed.hostname}:${cruxPort}`, + headers + }).on('connect', (res, socket) => { + // To avoid leaking the header since it could enable someone to detect you! + delete headers['Proxy-Authorization'] + if (res.statusCode === 200) { + const agent = new https.Agent({ socket }); + + const req = https.request( + { + host: urlParsed.hostname, + path: urlParsed.pathname + urlParsed.search, + agent, + headers: headers, + method: cruxHTTPMethod + }, + (res) => { + if (res.statusCode >= 499) { + log.error( + `Got error from CrUx. Error Code: ${res.statusCode} Message: ${res.statusMessage}` + ); + return reject(new Error(`Status Code: ${res.statusCode}`)); + } + const data = []; + + res.on('data', chunk => { + data.push(chunk); + }); + + res.on('end', () => { + resolve(JSON.parse(Buffer.concat(data).toString())); + }); + } + ); + req.write(body); + req.end(); + + req.on('error', (err) => { + reject(err.message); + }) + } else { + reject('Could not connect to proxy!') + } + + }).on('error', (err) => { + reject(err.message); + }).end(); + }) } From 78e69677fce67bb206a02c7147b0bc474b5c9502 Mon Sep 17 00:00:00 2001 From: diginote Date: Wed, 14 Jun 2023 18:51:44 +0200 Subject: [PATCH 2/2] Update crux-test.yml add test for crux request over publicly available proxy --- .github/workflows/crux-test.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/crux-test.yml b/.github/workflows/crux-test.yml index 7e64f637a2..8010e01600 100644 --- a/.github/workflows/crux-test.yml +++ b/.github/workflows/crux-test.yml @@ -18,5 +18,10 @@ jobs: - name: Install sitespeed.io run: npm ci - name: Run tests with CruX + run: bin/sitespeed.js -b chrome -n 1 --crux.key ${{ secrets.CRUX_KEY }} https://en.wikipedia.org/wiki/Main_Page --plugins.remove browsertime + - name: Run tests with CruX via proxy + env: + http_proxy: http://164.90.212.28:8080 + https_proxy: http://164.90.212.28:8080 run: bin/sitespeed.js -b chrome -n 1 --crux.key ${{ secrets.CRUX_KEY }} https://en.wikipedia.org/wiki/Main_Page --plugins.remove browsertime - \ No newline at end of file +