Implement backend metrics route

backend/cache/src/controllers/blob.ts

@@ -0,0 +1,181 @@
+import { spawn } from 'child_process';
+import { Request, Response } from 'express';
+import * as fs from 'fs';
+import path from 'path';
+import { getIdPrefixes } from '@/utils/strings';
+import { isValidBlobId } from '@/utils/validation';
+
+// Test Blob Info
+//
+// MIME type: image/png
+//
+// Blob ID: 0nzvRVLeF0I5kbWO3s_VDa-ixYZ_nhkp4J2EubJUtjo
+// Unencoded size: 165 KiB
+// Encoded size (including metadata): 62.0 MiB
+// Sui object ID: 0x0ebad3b13ee9bc64f8d6370e71d3408a43febaa017a309d2367117afe144ae8c
+
+// Cache-Control value initialized once
+const blobCacheControl = process.env.BLOB_CACHE_CONTROL || 'public, max-age=10';
+const SIZE_LIMIT = 209715200;
+
+export const getBlob = async (req: Request, res: Response) => {
+    const { id = '' } = req.params;
+
+    // Request validation
+    try {
+        await isValidBlobId(id);
+    } catch (error) {
+        if (error instanceof Error) {
+            res.status(400).send(error.message);
+        } else {
+            res.status(400).send('Unknown error');
+        }
+    }
+
+    const { prefix_1, prefix_2 } = getIdPrefixes(id);
+    if (!prefix_1 || !prefix_2) {
+        // Should never happen because id was validated.
+        return res.status(500).send('Internal Server Error (prefix)');
+    }
+
+    const homePath = process.env.HOME || '';
+
+    // First attempt at reading meta information
+    // from ~/cache/prefix1/prefix2/<id>.json
+    //
+    // The meta file confirms:
+    //  - the blob is locally available.
+    //  - get the blob MIME type.
+    //
+    // If the meta file does not exists, then do an async shell
+    // call to "load-blob <id>". If this call is successful, then
+    // try again to read the meta file.
+    //
+    // Once meta are confirmed, then the blob to return
+    // for this request is stored in:
+    //    $HOME/cache/prefix1/prefix2/<id>.blob
+    const jsonPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.json`);
+
+    // Read the meta JSON file and extract the MIME type and size.
+    let mime_parsed: string | undefined = undefined;
+    let blob_size: number | undefined = undefined;
+    let attempts = 0;
+
+    while (!mime_parsed && attempts < 3) {
+        try {
+            await fs.promises.access(jsonPath, fs.constants.F_OK);
+
+            // Read the meta JSON file and extract the MIME type
+            const meta = await fs.promises.readFile(jsonPath, 'utf8');
+
+            try {
+                const parsedMeta = JSON.parse(meta);
+                mime_parsed = parsedMeta.mime;
+                blob_size = parsedMeta.size;
+            } catch (parseError) {
+                console.error(`Error meta parsing: ${parseError} id: ${id}`);
+                return res.status(500).send('Internal Server Error (meta parsing)');
+            }
+
+            if (blob_size) {
+                if (blob_size > SIZE_LIMIT) {
+                    console.error(`Error: Blob size exceeds limit ${blob_size} > ${SIZE_LIMIT}`);
+                    return res
+                        .status(404)
+                        .send(`Blob size ${blob_size} not supported by Suiftly (${SIZE_LIMIT} limit)`);
+                }
+            }
+        } catch (error) {
+            let return_server_error = true;
+            if (error instanceof Error) {
+                if ((error as NodeJS.ErrnoException).code === 'ENOENT') {
+                    return_server_error = false;
+                }
+            }
+            if (return_server_error) {
+                console.error('Error accessing or reading meta file:', error);
+                return res.status(500).send('Internal Server Error (cache access)');
+            }
+        }
+
+        if (!mime_parsed) {
+            // Do an async shell call to "load-blob <id>".
+            // Note: load-blob handles safe concurrent loading (at blob granularity).
+            //
+            // Status code are:
+            // 0: Success
+            // 1: Does not exists on Walrus
+            // 2: Does not exists on Suiftly
+            // 3: Unsupported MIME type
+            // 4: Unsupported Blob Size
+            // 5: Unknown error
+            //
+            // In all error cases, the client should fallback to Walrus directly.
+            let status_code = 255;
+            let starting_shell_process_failed = false;
+
+            const loadBlob = path.resolve(homePath, 'suiftly-ops/scripts/load-blob');
+            const loadBlobProcess = spawn(loadBlob, [id]);
+
+            await new Promise<void>(resolve => {
+                loadBlobProcess.once('close', code => {
+                    if (code !== null) {
+                        status_code = code;
+                    }
+                    resolve();
+                });
+
+                loadBlobProcess.once('error', err => {
+                    console.error('Failed to start load-blob process:', err);
+                    starting_shell_process_failed = true;
+                    resolve();
+                });
+            });
+
+            if (status_code === 0) {
+                // Success.
+            } else if (status_code === 1) {
+                return res.status(404).send('Blob is not stored on Walrus');
+            } else if (status_code === 2) {
+                return res.status(404).send('Blob is not stored on Suiftly');
+            } else if (status_code === 3) {
+                return res.status(404).send('Blob MIME type not supported by Suiftly');
+            } else if (status_code === 4) {
+                return res.status(404).send(`Blob size not supported by Suiftly (${SIZE_LIMIT} limit)`);
+            } else if (starting_shell_process_failed) {
+                return res.status(500).send('Internal Server Error (shell call)');
+            } else {
+                return res.status(500).send(`Internal Server Error (${status_code})`);
+            }
+        }
+        attempts++;
+    }
+
+    if (!mime_parsed) {
+        console.error('Error meta parsing: mime not found');
+        return res.status(500).send('Internal Server Error (timeout)');
+    }
+
+    if (!blob_size) {
+        console.error('Error meta parsing: size not found');
+        return res.status(500).send('Internal Server Error (size missing)');
+    }
+
+    const blobPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.blob`);
+
+    // Set headers and options
+
+    const options = {
+        headers: {
+            'Cache-Control': blobCacheControl,
+            'Content-Type': mime_parsed,
+        },
+    };
+
+    // Stream the binary as response.
+    res.sendFile(blobPath, options, err => {
+        if (err) {
+            res.status(500).send('Error streaming blob');
+        }
+    });
+};

backend/cache/src/controllers/metrics.ts

@@ -0,0 +1,77 @@
+import { spawn } from 'child_process';
+import { Request, Response } from 'express';
+import * as fs from 'fs';
+import path from 'path';
+import { getIdPrefixes } from '@/utils/strings';
+import { isValidBlobId } from '@/utils/validation';
+
+// Test Blob Info
+//
+// MIME type: image/png
+//
+// Blob ID: 0nzvRVLeF0I5kbWO3s_VDa-ixYZ_nhkp4J2EubJUtjo
+// Unencoded size: 165 KiB
+// Encoded size (including metadata): 62.0 MiB
+// Sui object ID: 0x0ebad3b13ee9bc64f8d6370e71d3408a43febaa017a309d2367117afe144ae8c
+
+// Cache-Control value initialized once
+const metricsCacheControl = process.env.METRICS_CACHE_CONTROL || 'public, max-age=10';
+
+export const getMetrics = async (req: Request, res: Response) => {
+    const { id = '' } = req.params;
+
+    // Request validation
+    try {
+        await isValidBlobId(id);
+    } catch (error) {
+        if (error instanceof Error) {
+            res.status(400).send(error.message);
+        } else {
+            res.status(400).send('Unknown error');
+        }
+    }
+
+    const { prefix_1, prefix_2 } = getIdPrefixes(id);
+    if (!prefix_1 || !prefix_2) {
+        // Should never happen because id was validated.
+        return res.status(500).send('Internal Server Error (prefix)');
+    }
+
+    // Attempt reading metrics information
+    // from ~/cache/prefix1/prefix2/<id>.metrics
+    //
+    // If it is not created yet, then there is no metrics.
+    // Return empty metrics JSON:
+    // {
+    //   "status": "no_metrics",
+    //   "message": "No metrics accumulated yet for this item. Metrics updated every ~24 hours.",
+    // }
+    //
+    // If the .metrics file exists, return it as JSON.
+    const homePath = process.env.HOME || '';
+    const jsonPath = path.resolve(homePath, 'cache', prefix_1, prefix_2, `${id}.metrics`);
+
+    const options = {
+        headers: {
+            'Cache-Control': metricsCacheControl,
+            'Content-Type': 'application/json',
+        },
+    };
+
+    try {
+        await fs.promises.access(jsonPath);
+        // Stream the .metrics as response.
+        res.sendFile(jsonPath, options, err => {
+            if (err) {
+                res.status(500).send('Error streaming metrics');
+            }
+        });
+    } catch (err) {
+        // File does not exist, return "no_metrics" JSON response
+        res.json({
+            blobId: id,
+            status: 'no_metrics',
+            message: 'No metrics accumulated yet for this blob. Metrics updated every ~24 hours.',
+        });
+    }
+};

backend/cache/src/routes/blobRoutes.ts

@@ -0,0 +1,12 @@
+import express from 'express';
+
+import { getBlob } from '@/controllers/blob';
+
+export const blobRoutes = express.Router();
+
+blobRoutes.get('/', (req, res) => {
+    res.set('Cache-Control', 'public, max-age=86400');
+    res.send('Blob ID missing.<br><br> Useage: <b>https://cdn.suiftly.io/blob/<i>&lt;your_blob_id&gt;</i></b>');
+});
+
+blobRoutes.get('/:id', getBlob);

backend/cache/src/routes/metricsRoutes.ts

@@ -0,0 +1,12 @@
+import express from 'express';
+
+import { getMetrics } from '@/controllers/metrics';
+
+export const metricsRoutes = express.Router();
+
+metricsRoutes.get('/', (req, res) => {
+    res.set('Cache-Control', 'public, max-age=86400');
+    res.send('Blob ID missing.<br><br> Useage: <b>https://cdn.suiftly.io/metrics/<i>&lt;your_blob_id&gt;</i></b>');
+});
+
+metricsRoutes.get('/:id', getMetrics);

backend/cache/src/utils/strings.ts

@@ -0,0 +1,14 @@
+// Simple string utilities
+
+export function getIdPrefixes(id: string | undefined): { prefix_1: string; prefix_2: string } {
+    // Extract two short prefix from the blob ID.
+    // Used for file system partitioning
+    if (!id || id.length < 4) {
+        return { prefix_1: '', prefix_2: '' };
+    }
+
+    const prefix_1 = id.slice(0, 2);
+    const prefix_2 = id.slice(2, 4);
+
+    return { prefix_1, prefix_2 };
+}

backend/cache/src/utils/validation.ts

@@ -0,0 +1,21 @@
+export function isValidBlobId(id: string | undefined): Promise<void> {
+    return new Promise((resolve, reject) => {
+        if (typeof id !== 'string' || id.trim().length === 0) {
+            return reject(new Error('Blob ID is required'));
+        }
+
+        // Verify that the blob ID is a URL-safe base64 string
+        const base64Pattern = /^[a-zA-Z0-9-_]+$/;
+        if (!base64Pattern.test(id)) {
+            reject(new Error('Blob ID invalid'));
+        }
+
+        // Fast sanity check (enough characters for u256).
+        // TODO Calculate more precisely, this should be 44!?
+        if (id.length < 42) {
+            reject(new Error('Blob ID too short'));
+        }
+
+        resolve();
+    });
+}

turbo.json

@@ -7,6 +7,10 @@
         "HOME",
         "USERPROFILE",
         "BLOB_CACHE_CONTROL",
+        "METRICS_CACHE_CONTROL",
+        "VIEW_CACHE_CONTROL",
+        "META_CACHE_CONTROL",
+        "ICONS_CACHE_CONTROL",
         "PORT",
         "CDN_METRICS_ACCESS_KEY",
         "CDN_METRICS_PREFIX_URL",