From ae50fe8bd2b837efa83d0e8f5a14bd3db46b88f3 Mon Sep 17 00:00:00 2001 From: Wildan M Date: Thu, 15 Feb 2024 12:36:54 +0700 Subject: [PATCH] Use cached metadata --- jsconfig.json | 3 +- package.json | 1 + src/binaries/metadata.cjs | 1 + src/binaries/metadata.json | 44 ++++++++++++++++ src/binaries/update.js | 104 +++++++++++++++++++++++++++++++++++++ src/util.js | 83 ++++------------------------- 6 files changed, 161 insertions(+), 75 deletions(-) create mode 100644 src/binaries/metadata.cjs create mode 100644 src/binaries/metadata.json create mode 100644 src/binaries/update.js diff --git a/jsconfig.json b/jsconfig.json index c5b8b23..252b2f1 100644 --- a/jsconfig.json +++ b/jsconfig.json @@ -2,6 +2,7 @@ "compilerOptions": { "module": "ESNext", "moduleResolution": "Node", - "checkJs": true + "checkJs": true, + "resolveJsonModule": true } } \ No newline at end of file diff --git a/package.json b/package.json index 3701719..e63e93d 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "node": ">=16.0.0" }, "scripts": { + "update-binaries": "node ./src/binaries/update.js", "build": "sh ./tools-init.sh", "start": "node app.js" }, diff --git a/src/binaries/metadata.cjs b/src/binaries/metadata.cjs new file mode 100644 index 0000000..ecb09fd --- /dev/null +++ b/src/binaries/metadata.cjs @@ -0,0 +1 @@ +module.exports = require('./metadata.json'); diff --git a/src/binaries/metadata.json b/src/binaries/metadata.json new file mode 100644 index 0000000..fbffbf8 --- /dev/null +++ b/src/binaries/metadata.json @@ -0,0 +1,44 @@ +{ + "rubyVersionsList": [ + "3.1.3", + "3.1.2", + "3.0.5", + "3.0.4", + "3.0.3", + "2.7.7", + "2.7.6", + "2.7.5" + ], + "pythonVersionsList": [ + "3.12.1", + "3.11.7", + "3.10.13", + "3.9.18" + ], + "javaVersionsList": [ + "21.0.2+13.0.LTS", + "20.0.2+9", + "19.0.2+7", + "18.0.2+101", + "17.0.10+7", + "16.0.2+7", + "11.0.22+7", + "8.0.402+6" + ], + "pythonVersionsMap": { + "3.10.13": "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.10.13+20240107-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst", + "3.11.7": "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.11.7+20240107-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst", + "3.12.1": "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.12.1+20240107-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst", + "3.9.18": "https://github.com/indygreg/python-build-standalone/releases/download/20240107/cpython-3.9.18+20240107-x86_64_v3-unknown-linux-gnu-pgo+lto-full.tar.zst" + }, + "javaVersionsMap": { + "8.0.402+6": "https://github.com/adoptium/temurin8-binaries/releases/download/jdk8u402-b06/OpenJDK8U-jdk_x64_linux_hotspot_8u402b06.tar.gz", + "11.0.22+7": "https://github.com/adoptium/temurin11-binaries/releases/download/jdk-11.0.22%2B7/OpenJDK11U-jdk_x64_linux_hotspot_11.0.22_7.tar.gz", + "16.0.2+7": "https://github.com/adoptium/temurin16-binaries/releases/download/jdk-16.0.2%2B7/OpenJDK16U-jdk_x64_linux_hotspot_16.0.2_7.tar.gz", + "17.0.10+7": "https://github.com/adoptium/temurin17-binaries/releases/download/jdk-17.0.10%2B7/OpenJDK17U-jdk_x64_linux_hotspot_17.0.10_7.tar.gz", + "18.0.2+101": "https://github.com/adoptium/temurin18-binaries/releases/download/jdk-18.0.2.1%2B1/OpenJDK18U-jdk_x64_linux_hotspot_18.0.2.1_1.tar.gz", + "19.0.2+7": "https://github.com/adoptium/temurin19-binaries/releases/download/jdk-19.0.2%2B7/OpenJDK19U-jdk_x64_linux_hotspot_19.0.2_7.tar.gz", + "20.0.2+9": "https://github.com/adoptium/temurin20-binaries/releases/download/jdk-20.0.2%2B9/OpenJDK20U-jdk_x64_linux_hotspot_20.0.2_9.tar.gz", + "21.0.2+13.0.LTS": "https://github.com/adoptium/temurin21-binaries/releases/download/jdk-21.0.2%2B13/OpenJDK21U-jdk_x64_linux_hotspot_21.0.2_13.tar.gz" + } +} \ No newline at end of file diff --git a/src/binaries/update.js b/src/binaries/update.js new file mode 100644 index 0000000..36ea45d --- /dev/null +++ b/src/binaries/update.js @@ -0,0 +1,104 @@ +import axios from 'axios'; +import fs from 'fs'; +import { dirname } from 'path'; +import { fileURLToPath } from 'url'; +const __dirname = dirname(fileURLToPath(import.meta.url)); + +// https://packagist.org/php-statistics +let rubyVersionsList = []; +let pythonVersionsList = []; +let javaVersionsList = []; +/** + * @type {Record} + */ +let pythonVersionsMap = {}; +/** + * @type {Record} + */ +let javaVersionsMap = {}; +const pythonConstants = { + // https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json + tag: "20240107", + // NOTE: x86_64_v3 requires AVX2 CPU support + match: /cpython-(\d+\.\d+\.\d+)\+\d+-x86_64_v3-unknown-linux-gnu-pgo\+lto-full\.tar\.zst/g, + index() { + return `https://github.com/indygreg/python-build-standalone/releases/expanded_assets/${this.tag}` + }, + latestTagUrl() { + return 'https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json'; + }, + /** + * @param {string} filename + */ + asset_url(filename) { + return `https://github.com/indygreg/python-build-standalone/releases/download/${this.tag}/${filename}`; + }, +} +export const initUtils = async () => { + // TODO: detect OS/arch? + await axios.get('https://rvm_io.global.ssl.fastly.net/binaries/centos/9/x86_64/').then(res => { + // @ts-ignore + var matches = [...("" + res.data).matchAll(/href="ruby-([.\d]+).tar.bz2"/g)] + for (const match of matches) { + if (!rubyVersionsList.includes(match[1])) { + rubyVersionsList.push(match[1]); + } + } + rubyVersionsList = sortSemver(rubyVersionsList).reverse(); + }).catch(err => { + console.error('error fetching Ruby releases', err); + }); + + await axios.get(pythonConstants.latestTagUrl()).then(res => { + if (res.data && res.data.tag) { + pythonConstants.tag = res.data.tag; + } else { + console.warn('unable get latest python tag'); + } + }) + await axios.get(pythonConstants.index()).then(res => { + // @ts-ignore + var matches = [...("" + res.data).matchAll(pythonConstants.match)] + for (const match of matches) { + if (!pythonVersionsMap[match[1]]) { + pythonVersionsMap[match[1]] = pythonConstants.asset_url(match[0]); + pythonVersionsList.push(match[1]); + } + } + pythonVersionsList = sortSemver(pythonVersionsList).reverse(); + }).catch(err => { + console.error('error fetching Python releases', err); + }); + await axios.get('https://api.adoptium.net/v3/info/available_releases').then(async res => { + for (const ver of res.data.available_releases) { + await axios.get(`https://api.adoptium.net/v3/assets/latest/${ver}/hotspot?architecture=x64&image_type=jdk&os=linux&vendor=eclipse`).then(x => { + for (const binary of x.data) { + javaVersionsMap[binary.version.semver] = binary.binary.package.link; + } + }) + } + javaVersionsList = sortSemver(Object.keys(javaVersionsMap)).reverse(); + }) + + + fs.writeFileSync(__dirname + '/metadata.json', JSON.stringify({ + rubyVersionsList, + pythonVersionsList, + javaVersionsList, + pythonVersionsMap, + javaVersionsMap, + }, null, 2)) +} + + +// https://stackoverflow.com/a/40201629/3908409 +/** + * @param {string[]} arr + */ +export function sortSemver(arr) { + return arr.map(a => a.replace(/\d+/g, n => +n + 100000 + '')).sort() + .map(a => a.replace(/\d+/g, n => +n - 100000 + '')); +} + + +initUtils().then(() => console.log('metadata written')); diff --git a/src/util.js b/src/util.js index 8432cbc..eb291e5 100644 --- a/src/util.js +++ b/src/util.js @@ -1,47 +1,25 @@ import path from 'path'; import { spawn } from 'child_process'; import { lock } from 'proper-lockfile'; -import axios from 'axios'; import fs from 'fs'; +import binaries from './binaries/metadata.cjs'; +const { + javaVersionsList, + javaVersionsMap, + pythonVersionsList, + pythonVersionsMap, + rubyVersionsList +} = binaries; let tokenSecret, allowIps, sudoutil, version, revision; // https://packagist.org/php-statistics let phpVersionsList = []; -let rubyVersionsList = []; -let pythonVersionsList = []; -let javaVersionsList = []; -/** - * @type {Record} - */ -let pythonVersionsMap = {}; -/** - * @type {Record} - */ -let javaVersionsMap = {}; /** * @type {Record} */ let sslWildcardsMap = {}; -const pythonConstants = { - // https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json - tag: "20240107", - // NOTE: x86_64_v3 requires AVX2 CPU support - match: /cpython-(\d+\.\d+\.\d+)\+\d+-x86_64_v3-unknown-linux-gnu-pgo\+lto-full\.tar\.zst/g, - index() { - return `https://github.com/indygreg/python-build-standalone/releases/expanded_assets/${this.tag}` - }, - latestTagUrl() { - return 'https://raw.githubusercontent.com/indygreg/python-build-standalone/latest-release/latest-release.json'; - }, - /** - * @param {string} filename - */ - asset_url(filename) { - return `https://github.com/indygreg/python-build-standalone/releases/download/${this.tag}/${filename}`; - }, -} -export const initUtils = async () => { +export const initUtils = () => { tokenSecret = `Bearer ${process.env.SECRET}`; allowIps = process.env.ALLOW_IP ? process.env.ALLOW_IP.split(',').reduce((a, b) => { a[b] = true; @@ -69,50 +47,7 @@ export const initUtils = async () => { } catch (error) { phpVersionsList = []; } - // TODO: detect OS/arch? - await axios.get('https://rvm_io.global.ssl.fastly.net/binaries/centos/9/x86_64/').then(res => { - // @ts-ignore - var matches = [...("" + res.data).matchAll(/href="ruby-([.\d]+).tar.bz2"/g)] - for (const match of matches) { - if (!rubyVersionsList.includes(match[1])) { - rubyVersionsList.push(match[1]); - } - } - rubyVersionsList = sortSemver(rubyVersionsList).reverse(); - }).catch(err => { - console.error('error fetching Ruby releases', err); - }); - await axios.get(pythonConstants.latestTagUrl()).then(res => { - if (res.data && res.data.tag) { - pythonConstants.tag = res.data.tag; - } else { - console.warn('unable get latest python tag'); - } - }) - await axios.get(pythonConstants.index()).then(res => { - // @ts-ignore - var matches = [...("" + res.data).matchAll(pythonConstants.match)] - for (const match of matches) { - if (!pythonVersionsMap[match[1]]) { - pythonVersionsMap[match[1]] = pythonConstants.asset_url(match[0]); - pythonVersionsList.push(match[1]); - } - } - pythonVersionsList = sortSemver(pythonVersionsList).reverse(); - }).catch(err => { - console.error('error fetching Python releases', err); - }); - await axios.get('https://api.adoptium.net/v3/info/available_releases').then(async res => { - for (const ver of res.data.available_releases) { - await axios.get(`https://api.adoptium.net/v3/assets/latest/${ver}/hotspot?architecture=x64&image_type=jdk&os=linux&vendor=eclipse`).then(x => { - for (const binary of x.data) { - javaVersionsMap[binary.version.semver] = binary.binary.package.link; - } - }) - } - javaVersionsList = sortSemver(Object.keys(javaVersionsMap)).reverse(); - }) } export const getLtsPhp = (/** @type {string} */ major) => {