From cd8d16d95b2f339aa4e6e8c3b42877b4e49b24b6 Mon Sep 17 00:00:00 2001 From: mario4tier Date: Sun, 1 Sep 2024 03:39:30 -0400 Subject: [PATCH] Implement backend metrics route --- backend/cache/src/controllers/blob.ts | 181 ++++++++++++++++++++++ backend/cache/src/controllers/metrics.ts | 77 +++++++++ backend/cache/src/routes/blobRoutes.ts | 12 ++ backend/cache/src/routes/metricsRoutes.ts | 12 ++ backend/cache/src/utils/strings.ts | 14 ++ backend/cache/src/utils/validation.ts | 21 +++ turbo.json | 4 + 7 files changed, 321 insertions(+) create mode 100644 backend/cache/src/controllers/blob.ts create mode 100644 backend/cache/src/controllers/metrics.ts create mode 100644 backend/cache/src/routes/blobRoutes.ts create mode 100644 backend/cache/src/routes/metricsRoutes.ts create mode 100644 backend/cache/src/utils/strings.ts create mode 100644 backend/cache/src/utils/validation.ts diff --git a/backend/cache/src/controllers/blob.ts b/backend/cache/src/controllers/blob.ts new file mode 100644 index 0000000..b935ba9 --- /dev/null +++ b/backend/cache/src/controllers/blob.ts @@ -0,0 +1,181 @@ +import { spawn } from 'child_process'; +import { Request, Response } from 'express'; +import * as fs from 'fs'; +import path from 'path'; +import { getIdPrefixes } from '@/utils/strings'; +import { isValidBlobId } from '@/utils/validation'; + +// Test Blob Info +// +// MIME type: image/png +// +// Blob ID: 0nzvRVLeF0I5kbWO3s_VDa-ixYZ_nhkp4J2EubJUtjo +// Unencoded size: 165 KiB +// Encoded size (including metadata): 62.0 MiB +// Sui object ID: 0x0ebad3b13ee9bc64f8d6370e71d3408a43febaa017a309d2367117afe144ae8c + +// Cache-Control value initialized once +const blobCacheControl = process.env.BLOB_CACHE_CONTROL || 'public, max-age=10'; +const SIZE_LIMIT = 209715200; + +export const getBlob = async (req: Request, res: Response) => { + const { id = '' } = req.params; + + // Request validation + try { + await isValidBlobId(id); + } catch (error) { + if (error instanceof Error) { + res.status(400).send(error.message); + } else { + res.status(400).send('Unknown error'); + } + } + + const { prefix_1, prefix_2 } = getIdPrefixes(id); + if (!prefix_1 || !prefix_2) { + // Should never happen because id was validated. + return res.status(500).send('Internal Server Error (prefix)'); + } + + const homePath = process.env.HOME || ''; + + // First attempt at reading meta information + // from ~/cache/prefix1/prefix2/.json + // + // The meta file confirms: + // - the blob is locally available. + // - get the blob MIME type. + // + // If the meta file does not exists, then do an async shell + // call to "load-blob ". If this call is successful, then + // try again to read the meta file. + // + // Once meta are confirmed, then the blob to return + // for this request is stored in: + // $HOME/cache/prefix1/prefix2/.blob + const jsonPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.json`); + + // Read the meta JSON file and extract the MIME type and size. + let mime_parsed: string | undefined = undefined; + let blob_size: number | undefined = undefined; + let attempts = 0; + + while (!mime_parsed && attempts < 3) { + try { + await fs.promises.access(jsonPath, fs.constants.F_OK); + + // Read the meta JSON file and extract the MIME type + const meta = await fs.promises.readFile(jsonPath, 'utf8'); + + try { + const parsedMeta = JSON.parse(meta); + mime_parsed = parsedMeta.mime; + blob_size = parsedMeta.size; + } catch (parseError) { + console.error(`Error meta parsing: ${parseError} id: ${id}`); + return res.status(500).send('Internal Server Error (meta parsing)'); + } + + if (blob_size) { + if (blob_size > SIZE_LIMIT) { + console.error(`Error: Blob size exceeds limit ${blob_size} > ${SIZE_LIMIT}`); + return res + .status(404) + .send(`Blob size ${blob_size} not supported by Suiftly (${SIZE_LIMIT} limit)`); + } + } + } catch (error) { + let return_server_error = true; + if (error instanceof Error) { + if ((error as NodeJS.ErrnoException).code === 'ENOENT') { + return_server_error = false; + } + } + if (return_server_error) { + console.error('Error accessing or reading meta file:', error); + return res.status(500).send('Internal Server Error (cache access)'); + } + } + + if (!mime_parsed) { + // Do an async shell call to "load-blob ". + // Note: load-blob handles safe concurrent loading (at blob granularity). + // + // Status code are: + // 0: Success + // 1: Does not exists on Walrus + // 2: Does not exists on Suiftly + // 3: Unsupported MIME type + // 4: Unsupported Blob Size + // 5: Unknown error + // + // In all error cases, the client should fallback to Walrus directly. + let status_code = 255; + let starting_shell_process_failed = false; + + const loadBlob = path.resolve(homePath, 'suiftly-ops/scripts/load-blob'); + const loadBlobProcess = spawn(loadBlob, [id]); + + await new Promise(resolve => { + loadBlobProcess.once('close', code => { + if (code !== null) { + status_code = code; + } + resolve(); + }); + + loadBlobProcess.once('error', err => { + console.error('Failed to start load-blob process:', err); + starting_shell_process_failed = true; + resolve(); + }); + }); + + if (status_code === 0) { + // Success. + } else if (status_code === 1) { + return res.status(404).send('Blob is not stored on Walrus'); + } else if (status_code === 2) { + return res.status(404).send('Blob is not stored on Suiftly'); + } else if (status_code === 3) { + return res.status(404).send('Blob MIME type not supported by Suiftly'); + } else if (status_code === 4) { + return res.status(404).send(`Blob size not supported by Suiftly (${SIZE_LIMIT} limit)`); + } else if (starting_shell_process_failed) { + return res.status(500).send('Internal Server Error (shell call)'); + } else { + return res.status(500).send(`Internal Server Error (${status_code})`); + } + } + attempts++; + } + + if (!mime_parsed) { + console.error('Error meta parsing: mime not found'); + return res.status(500).send('Internal Server Error (timeout)'); + } + + if (!blob_size) { + console.error('Error meta parsing: size not found'); + return res.status(500).send('Internal Server Error (size missing)'); + } + + const blobPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.blob`); + + // Set headers and options + + const options = { + headers: { + 'Cache-Control': blobCacheControl, + 'Content-Type': mime_parsed, + }, + }; + + // Stream the binary as response. + res.sendFile(blobPath, options, err => { + if (err) { + res.status(500).send('Error streaming blob'); + } + }); +}; diff --git a/backend/cache/src/controllers/metrics.ts b/backend/cache/src/controllers/metrics.ts new file mode 100644 index 0000000..a16a4f5 --- /dev/null +++ b/backend/cache/src/controllers/metrics.ts @@ -0,0 +1,77 @@ +import { spawn } from 'child_process'; +import { Request, Response } from 'express'; +import * as fs from 'fs'; +import path from 'path'; +import { getIdPrefixes } from '@/utils/strings'; +import { isValidBlobId } from '@/utils/validation'; + +// Test Blob Info +// +// MIME type: image/png +// +// Blob ID: 0nzvRVLeF0I5kbWO3s_VDa-ixYZ_nhkp4J2EubJUtjo +// Unencoded size: 165 KiB +// Encoded size (including metadata): 62.0 MiB +// Sui object ID: 0x0ebad3b13ee9bc64f8d6370e71d3408a43febaa017a309d2367117afe144ae8c + +// Cache-Control value initialized once +const metricsCacheControl = process.env.METRICS_CACHE_CONTROL || 'public, max-age=10'; + +export const getMetrics = async (req: Request, res: Response) => { + const { id = '' } = req.params; + + // Request validation + try { + await isValidBlobId(id); + } catch (error) { + if (error instanceof Error) { + res.status(400).send(error.message); + } else { + res.status(400).send('Unknown error'); + } + } + + const { prefix_1, prefix_2 } = getIdPrefixes(id); + if (!prefix_1 || !prefix_2) { + // Should never happen because id was validated. + return res.status(500).send('Internal Server Error (prefix)'); + } + + // Attempt reading metrics information + // from ~/cache/prefix1/prefix2/.metrics + // + // If it is not created yet, then there is no metrics. + // Return empty metrics JSON: + // { + // "status": "no_metrics", + // "message": "No metrics accumulated yet for this item. Metrics updated every ~24 hours.", + // } + // + // If the .metrics file exists, return it as JSON. + const homePath = process.env.HOME || ''; + const jsonPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.metrics`); + + const options = { + headers: { + 'Cache-Control': metricsCacheControl, + 'Content-Type': 'application/json', + }, + }; + + try { + await fs.promises.access(jsonPath); + // Stream the .metrics as response. + res.sendFile(jsonPath, options, err => { + if (err) { + res.status(500).send('Error streaming metrics'); + } + }); + } catch (err) { + // File does not exist, return "no_metrics" JSON response + res.json({ + blobId: id, + status: 'no_metrics', + message: 'No metrics accumulated yet for this blob. Metrics updated every ~24 hours.', + }); + } +}; diff --git a/backend/cache/src/routes/blobRoutes.ts b/backend/cache/src/routes/blobRoutes.ts new file mode 100644 index 0000000..61af9e4 --- /dev/null +++ b/backend/cache/src/routes/blobRoutes.ts @@ -0,0 +1,12 @@ +import express from 'express'; + +import { getBlob } from '@/controllers/blob'; + +export const blobRoutes = express.Router(); + +blobRoutes.get('/', (req, res) => { + res.set('Cache-Control', 'public, max-age=86400'); + res.send('Blob ID missing.

Useage: https://cdn.suiftly.io/blob/<your_blob_id>'); +}); + +blobRoutes.get('/:id', getBlob); diff --git a/backend/cache/src/routes/metricsRoutes.ts b/backend/cache/src/routes/metricsRoutes.ts new file mode 100644 index 0000000..4da15f4 --- /dev/null +++ b/backend/cache/src/routes/metricsRoutes.ts @@ -0,0 +1,12 @@ +import express from 'express'; + +import { getMetrics } from '@/controllers/metrics'; + +export const metricsRoutes = express.Router(); + +metricsRoutes.get('/', (req, res) => { + res.set('Cache-Control', 'public, max-age=86400'); + res.send('Blob ID missing.

Useage: https://cdn.suiftly.io/metrics/<your_blob_id>'); +}); + +metricsRoutes.get('/:id', getMetrics); diff --git a/backend/cache/src/utils/strings.ts b/backend/cache/src/utils/strings.ts new file mode 100644 index 0000000..d92f430 --- /dev/null +++ b/backend/cache/src/utils/strings.ts @@ -0,0 +1,14 @@ +// Simple string utilities + +export function getIdPrefixes(id: string | undefined): { prefix_1: string; prefix_2: string } { + // Extract two short prefix from the blob ID. + // Used for file system partitioning + if (!id || id.length < 4) { + return { prefix_1: '', prefix_2: '' }; + } + + const prefix_1 = id.slice(0, 2); + const prefix_2 = id.slice(2, 4); + + return { prefix_1, prefix_2 }; +} diff --git a/backend/cache/src/utils/validation.ts b/backend/cache/src/utils/validation.ts new file mode 100644 index 0000000..43916f5 --- /dev/null +++ b/backend/cache/src/utils/validation.ts @@ -0,0 +1,21 @@ +export function isValidBlobId(id: string | undefined): Promise { + return new Promise((resolve, reject) => { + if (typeof id !== 'string' || id.trim().length === 0) { + return reject(new Error('Blob ID is required')); + } + + // Verify that the blob ID is a URL-safe base64 string + const base64Pattern = /^[a-zA-Z0-9-_]+$/; + if (!base64Pattern.test(id)) { + reject(new Error('Blob ID invalid')); + } + + // Fast sanity check (enough characters for u256). + // TODO Calculate more precisely, this should be 44!? + if (id.length < 42) { + reject(new Error('Blob ID too short')); + } + + resolve(); + }); +} diff --git a/turbo.json b/turbo.json index 814149e..dfd9070 100644 --- a/turbo.json +++ b/turbo.json @@ -7,6 +7,10 @@ "HOME", "USERPROFILE", "BLOB_CACHE_CONTROL", + "METRICS_CACHE_CONTROL", + "VIEW_CACHE_CONTROL", + "META_CACHE_CONTROL", + "ICONS_CACHE_CONTROL", "PORT", "CDN_METRICS_ACCESS_KEY", "CDN_METRICS_PREFIX_URL",